Files
element-call/src/tracks/BackgroundEffectTransformer.ts
T
fkwpandClaude Fable 5.1 d3eefdbcf2 Attach background effects through one switchable pipeline
Blur runs through one pipeline, shared by the pre-join preview and the
call: attached the first time an effect is wanted and switched in place
after, never detached. The logic is BackgroundEffects in src/state;
BackgroundEffectsContext only builds it, from the app's view model, so
the standalone app and the component hand the same instance to every
camera. Disabled drops the blur radius, so no effect costs no
segmentation. The SDK gains fromLivekitProcessor beside the adapter,
now toLivekitProcessor.

Ported from spec/background-effects/2-pipeline (#4282): a91dab8f with
its review tidy-ups 445e010c, bbc73d07, 51729bcf, f52d2b79, 813f1e6f
and 584bbaa0. Left out as already solved here: OneStepPipeline (the SDK
runs a processor's steps in turn) and the LiveKit-track sync helpers
(the lobby's camera is an SDK track, synced by followProcessor).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-11-04 19:00:37 +01:00

96 lines
3.3 KiB
TypeScript

/*
Copyright 2024-2025 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
BackgroundTransformer,
VideoTransformer,
type VideoTransformerInitOptions,
type BackgroundOptions,
} from "@livekit/track-processors";
import { ImageSegmenter } from "@mediapipe/tasks-vision";
import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url";
interface WasmFileset {
/** The path to the Wasm loader script. */
wasmLoaderPath: string;
/** The path to the Wasm binary. */
wasmBinaryPath: string;
}
// The MediaPipe package, by default, ships some alternative versions of the
// WASM files which avoid SIMD for compatibility with older browsers. But SIMD
// in WASM is actually fine by our support policy, so we include just the SIMD
// versions.
// It's really not ideal that we have to reference these internal files from
// MediaPipe and depend on node_modules having this specific structure. It's
// easy to see this breaking if our dependencies changed and MediaPipe were
// no longer hoisted, or if we switched to another dependency loader such as
// yarn PnP.
// https://github.com/google-ai-edge/mediapipe/issues/5961
const wasmFileset: WasmFileset = {
wasmLoaderPath: new URL(
"../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js",
import.meta.url,
).href,
wasmBinaryPath: new URL(
"../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm",
import.meta.url,
).href,
};
/**
* Track processor that applies effects such as blurring to a user's background.
*
* This is just like LiveKit's prebuilt BackgroundTransformer except that it
* loads the segmentation models from our own bundle rather than as an external
* resource fetched from the public internet.
*/
export class BackgroundEffectTransformer extends BackgroundTransformer {
/**
* As the library's, except that disabling also drops the blur radius: kept,
* it has every frame segmented and the result thrown away, where with
* neither a radius nor a picture frames are passed on untouched.
*/
public override async update(opts: BackgroundOptions): Promise<void> {
await super.update(
opts.backgroundDisabled ? { ...opts, blurRadius: undefined } : opts,
);
}
public async init({
outputCanvas,
inputElement: inputVideo,
}: VideoTransformerInitOptions): Promise<void> {
// Call super.super.init() since we're totally replacing the init method of
// BackgroundTransformer here, rather than extending it
await VideoTransformer.prototype.init.call(this, {
outputCanvas,
inputElement: inputVideo,
});
this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, {
baseOptions: {
modelAssetPath,
delegate: "GPU",
...this.options.segmenterOptions,
},
canvas: this.canvas,
runningMode: "VIDEO",
outputCategoryMask: true,
outputConfidenceMasks: false,
});
// BackgroundTransformer's own init applies these, and this one replaces it.
if (this.options.blurRadius) {
this.gl?.setBlurRadius(this.options.blurRadius);
}
this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false);
}
}