mirror of
https://github.com/element-hq/element-call.git
synced 2026-11-22 12:17:37 +00:00
Blur runs through one pipeline, shared by the pre-join preview and the call: attached the first time an effect is wanted and switched in place after, never detached. The logic is BackgroundEffects in src/state; BackgroundEffectsContext only builds it, from the app's view model, so the standalone app and the component hand the same instance to every camera. Disabled drops the blur radius, so no effect costs no segmentation. The SDK gains fromLivekitProcessor beside the adapter, now toLivekitProcessor. Ported from spec/background-effects/2-pipeline (#4282):a91dab8fwith its review tidy-ups445e010c,bbc73d07,51729bcf,f52d2b79,813f1e6fand584bbaa0. Left out as already solved here: OneStepPipeline (the SDK runs a processor's steps in turn) and the LiveKit-track sync helpers (the lobby's camera is an SDK track, synced by followProcessor). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
96 lines
3.3 KiB
TypeScript
96 lines
3.3 KiB
TypeScript
/*
|
|
Copyright 2024-2025 New Vector Ltd.
|
|
Copyright 2026 Element Creations Ltd.
|
|
|
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
|
Please see LICENSE in the repository root for full details.
|
|
*/
|
|
|
|
import {
|
|
BackgroundTransformer,
|
|
VideoTransformer,
|
|
type VideoTransformerInitOptions,
|
|
type BackgroundOptions,
|
|
} from "@livekit/track-processors";
|
|
import { ImageSegmenter } from "@mediapipe/tasks-vision";
|
|
|
|
import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url";
|
|
|
|
interface WasmFileset {
|
|
/** The path to the Wasm loader script. */
|
|
wasmLoaderPath: string;
|
|
/** The path to the Wasm binary. */
|
|
wasmBinaryPath: string;
|
|
}
|
|
|
|
// The MediaPipe package, by default, ships some alternative versions of the
|
|
// WASM files which avoid SIMD for compatibility with older browsers. But SIMD
|
|
// in WASM is actually fine by our support policy, so we include just the SIMD
|
|
// versions.
|
|
// It's really not ideal that we have to reference these internal files from
|
|
// MediaPipe and depend on node_modules having this specific structure. It's
|
|
// easy to see this breaking if our dependencies changed and MediaPipe were
|
|
// no longer hoisted, or if we switched to another dependency loader such as
|
|
// yarn PnP.
|
|
// https://github.com/google-ai-edge/mediapipe/issues/5961
|
|
const wasmFileset: WasmFileset = {
|
|
wasmLoaderPath: new URL(
|
|
"../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js",
|
|
import.meta.url,
|
|
).href,
|
|
wasmBinaryPath: new URL(
|
|
"../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm",
|
|
import.meta.url,
|
|
).href,
|
|
};
|
|
|
|
/**
|
|
* Track processor that applies effects such as blurring to a user's background.
|
|
*
|
|
* This is just like LiveKit's prebuilt BackgroundTransformer except that it
|
|
* loads the segmentation models from our own bundle rather than as an external
|
|
* resource fetched from the public internet.
|
|
*/
|
|
export class BackgroundEffectTransformer extends BackgroundTransformer {
|
|
/**
|
|
* As the library's, except that disabling also drops the blur radius: kept,
|
|
* it has every frame segmented and the result thrown away, where with
|
|
* neither a radius nor a picture frames are passed on untouched.
|
|
*/
|
|
public override async update(opts: BackgroundOptions): Promise<void> {
|
|
await super.update(
|
|
opts.backgroundDisabled ? { ...opts, blurRadius: undefined } : opts,
|
|
);
|
|
}
|
|
|
|
public async init({
|
|
outputCanvas,
|
|
inputElement: inputVideo,
|
|
}: VideoTransformerInitOptions): Promise<void> {
|
|
// Call super.super.init() since we're totally replacing the init method of
|
|
// BackgroundTransformer here, rather than extending it
|
|
await VideoTransformer.prototype.init.call(this, {
|
|
outputCanvas,
|
|
inputElement: inputVideo,
|
|
});
|
|
|
|
this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, {
|
|
baseOptions: {
|
|
modelAssetPath,
|
|
delegate: "GPU",
|
|
...this.options.segmenterOptions,
|
|
},
|
|
canvas: this.canvas,
|
|
runningMode: "VIDEO",
|
|
outputCategoryMask: true,
|
|
outputConfidenceMasks: false,
|
|
});
|
|
|
|
// BackgroundTransformer's own init applies these, and this one replaces it.
|
|
if (this.options.blurRadius) {
|
|
this.gl?.setBlurRadius(this.options.blurRadius);
|
|
}
|
|
this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false);
|
|
}
|
|
}
|