diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index 83a58b053..9f80e4476 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -9,12 +9,13 @@ import { combineLatest, map, type Observable, switchMap } from "rxjs"; import { supportsAudioOutputSelection } from "livekit-client"; import { supportsBackgroundProcessors } from "../livekit/backgroundProcessing"; +import { parseEffect } from "../livekit/backgroundEffects"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { type MenuOptions } from "./MediaMuteAndSwitchButton"; import { type MediaDevices } from "../state/MediaDevices"; import { - backgroundBlur as backgroundBlurSettings, + backgroundEffect as backgroundEffectSetting, debugTileLayout as debugTileLayoutSetting, } from "../settings/settings"; import { type Behavior, constant } from "../state/Behavior"; @@ -93,6 +94,11 @@ function buildDeviceBehaviors( ), ); + const blurOn$ = scope.behavior( + backgroundEffectSetting.value$.pipe( + map((raw) => parseEffect(raw).kind === "blur"), + ), + ); return { audioOptions$: scope.behavior(options$(mediaDevices.audioInput.available$)), selectedAudio$: scope.behavior( @@ -118,17 +124,17 @@ function buildDeviceBehaviors( ), selectVideoButtonOption$: constant(mediaDevices.videoInput.select), toggleBlur$: scope.behavior( - combineLatest([backgroundBlurSettings.value$, disableSwitcher$]).pipe( + combineLatest([blurOn$, disableSwitcher$]).pipe( map(([current, switcherDisabled]) => { return !switcherDisabled && supportsBackgroundProcessors() ? (): void => { - backgroundBlurSettings.setValue(!current); + backgroundEffectSetting.setValue(current ? "none" : "blur"); } : undefined; }), ), ), - videoBlurEnabled$: backgroundBlurSettings.value$, + videoBlurEnabled$: blurOn$, }; } diff --git a/src/graphics/background-arc.jpg b/src/graphics/background-arc.jpg new file mode 100644 index 000000000..7d709f819 Binary files /dev/null and b/src/graphics/background-arc.jpg differ diff --git a/src/graphics/background-glow.jpg b/src/graphics/background-glow.jpg new file mode 100644 index 000000000..91baebc55 Binary files /dev/null and b/src/graphics/background-glow.jpg differ diff --git a/src/livekit/BackgroundEffectTransformer.test.ts b/src/livekit/BackgroundEffectTransformer.test.ts index b8a26b86b..fae7a2af1 100644 --- a/src/livekit/BackgroundEffectTransformer.test.ts +++ b/src/livekit/BackgroundEffectTransformer.test.ts @@ -5,11 +5,39 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + BackgroundTransformer, + VideoTransformer, + type VideoTransformerInitOptions, +} from "@livekit/track-processors"; +import { ImageSegmenter } from "@mediapipe/tasks-vision"; import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; describe("BackgroundEffectTransformer", () => { + afterEach(() => vi.restoreAllMocks()); + + it("loads a background chosen before it was built", async () => { + vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue(); + vi.spyOn(ImageSegmenter, "createFromOptions").mockResolvedValue( + {} as ImageSegmenter, + ); + const load = vi + .spyOn(BackgroundTransformer.prototype, "loadAndSetBackground") + .mockResolvedValue(); + const transformer = new BackgroundEffectTransformer({ + backgroundDisabled: true, + }); + // Chosen before the camera is processed, as the first effect always is: + // loaded then, there is nothing yet to draw it with. + await transformer.update({ imagePath: "/background.jpg" }); + load.mockClear(); + + await transformer.init({} as VideoTransformerInitOptions); + expect(load).toHaveBeenCalledWith("/background.jpg"); + }); + it("passes frames on untouched once disabled, even after blurring", async () => { const transformer = new BackgroundEffectTransformer({}); await transformer.update({ blurRadius: 15, backgroundDisabled: false }); diff --git a/src/livekit/BackgroundEffectTransformer.ts b/src/livekit/BackgroundEffectTransformer.ts index 965359211..75c99cf75 100644 --- a/src/livekit/BackgroundEffectTransformer.ts +++ b/src/livekit/BackgroundEffectTransformer.ts @@ -12,6 +12,7 @@ import { type VideoTransformerInitOptions, type BackgroundOptions, } from "@livekit/track-processors"; +import { logger } from "matrix-js-sdk/lib/logger"; import { ImageSegmenter } from "@mediapipe/tasks-vision"; import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url"; @@ -87,6 +88,11 @@ export class BackgroundEffectTransformer extends BackgroundTransformer { }); // BackgroundTransformer's own init applies these, and this one replaces it. + if (this.options.imagePath) { + await this.loadAndSetBackground(this.options.imagePath).catch((e) => + logger.warn("Failed to load the background image", e), + ); + } if (this.options.blurRadius) { this.gl?.setBlurRadius(this.options.blurRadius); } diff --git a/src/livekit/BackgroundEffectsContext.test.ts b/src/livekit/BackgroundEffectsContext.test.ts index 4fd4b67b9..7196c457f 100644 --- a/src/livekit/BackgroundEffectsContext.test.ts +++ b/src/livekit/BackgroundEffectsContext.test.ts @@ -24,7 +24,7 @@ import { createAppBackgroundEffects, } from "./BackgroundEffectsContext"; import { type BackgroundEffects } from "../state/BackgroundEffects"; -import { backgroundBlur } from "../settings/settings"; +import { backgroundEffect } from "../settings/settings"; import { constant } from "../state/Behavior"; import { flushPromises, testScope } from "../utils/test"; @@ -168,7 +168,7 @@ describe("BackgroundEffectsProvider", () => { ): ReturnType => createElement(Surfaces, { tracks }); const blur = async (on: boolean): Promise => { await act(async () => { - backgroundBlur.setValue(on); + backgroundEffect.setValue(on ? "blur" : "none"); await flushPromises(); }); }; @@ -181,9 +181,9 @@ describe("BackgroundEffectsProvider", () => { pipelines.destroyed = 0; pipelines.switches = []; platformMock.platform = "desktop"; - backgroundBlur.setValue(false); + backgroundEffect.setValue("none"); }); - afterEach(() => backgroundBlur.setValue(false)); + afterEach(() => backgroundEffect.setValue("none")); // A pipeline primes itself when it is built and when it is destroyed, and a // primed pipeline lets one frame through untouched: so the frame is spent diff --git a/src/livekit/BackgroundEffectsContext.tsx b/src/livekit/BackgroundEffectsContext.tsx index 05c8d4f20..55cb40cb0 100644 --- a/src/livekit/BackgroundEffectsContext.tsx +++ b/src/livekit/BackgroundEffectsContext.tsx @@ -15,7 +15,7 @@ import { type LocalVideoTrack } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { combineLatest } from "rxjs"; -import { backgroundBlur as backgroundBlurSettings } from "../settings/settings"; +import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; import { OneStepPipeline } from "./OneStepPipeline"; import { supportsBackgroundProcessors } from "./backgroundProcessing"; @@ -117,7 +117,7 @@ export function createAppBackgroundEffects( ): BackgroundEffects { return createBackgroundEffects(scope, { supported: supportsBackgroundProcessors(), - blur$: backgroundBlurSettings.value$, + effect$: backgroundEffectSetting.value$, pipeline: new OneStepPipeline( new BackgroundEffectTransformer({ backgroundDisabled: true }), "background-effect", diff --git a/src/livekit/backgroundEffects.test.ts b/src/livekit/backgroundEffects.test.ts new file mode 100644 index 000000000..cac6901cc --- /dev/null +++ b/src/livekit/backgroundEffects.test.ts @@ -0,0 +1,53 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { readFile } from "node:fs/promises"; +import { describe, expect, test } from "vitest"; + +import { + type BackgroundEffect, + imagePathFor, + parseEffect, + serializeEffect, + shippedBackgrounds, +} from "./backgroundEffects"; + +describe("the chosen background effect", () => { + test("round-trips through its stored form", () => { + const effects: BackgroundEffect[] = [ + { kind: "none" }, + { kind: "blur" }, + ...shippedBackgrounds.map(({ id }) => ({ kind: "shipped" as const, id })), + ]; + for (const effect of effects) + expect(parseEffect(serializeEffect(effect))).toEqual(effect); + }); + + test("falls back to no effect when the stored form names nothing we ship", () => { + expect(parseEffect("image:gone")).toEqual({ kind: "none" }); + expect(parseEffect("")).toEqual({ kind: "none" }); + }); + + test("reads a stored value that is not a string as no effect", () => { + for (const raw of [null, 1, {}, ["blur"]]) + expect(parseEffect(raw)).toEqual({ kind: "none" }); + }); + + test("gives every shipped background an image to draw", () => { + for (const { id } of shippedBackgrounds) + expect(imagePathFor(id)).toBeTruthy(); + }); + + // A background has to cover what is behind it: JPEG has no alpha channel. + test("ships only opaque backgrounds", async () => { + for (const { imagePath } of shippedBackgrounds) { + const name = imagePath.split("/").pop()!.split("?")[0]; + const bytes = await readFile(`src/graphics/${name}`); + expect([bytes[0], bytes[1], bytes[2]]).toEqual([0xff, 0xd8, 0xff]); + } + }); +}); diff --git a/src/livekit/backgroundEffects.ts b/src/livekit/backgroundEffects.ts new file mode 100644 index 000000000..467c90d08 --- /dev/null +++ b/src/livekit/backgroundEffects.ts @@ -0,0 +1,62 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import arc from "../graphics/background-arc.jpg?url"; +import glow from "../graphics/background-glow.jpg?url"; + +export const blurRadius = 15; + +export interface ShippedBackground { + /** Stored in the user's settings when chosen, so it must not change. */ + id: string; + imagePath: string; +} + +export const shippedBackgrounds: ShippedBackground[] = [ + { id: "arc", imagePath: arc }, + { id: "glow", imagePath: glow }, +]; + +export type BackgroundEffect = + | { kind: "none" } + | { kind: "blur" } + | { kind: "shipped"; id: string }; + +/** An effect as the setting stores it, and as the menu names its option. */ +export type EffectId = "none" | "blur" | `image:${string}`; + +export const noEffect: BackgroundEffect = { kind: "none" }; + +/** + * Reads a stored effect; one we no longer ship reads as no effect. Stored by + * hand or by another build, the value need not even be a string. + */ +export function parseEffect(raw: unknown): BackgroundEffect { + if (typeof raw !== "string") return noEffect; + if (raw === "blur") return { kind: "blur" }; + if (raw.startsWith("image:")) { + const id = raw.slice("image:".length); + if (shippedBackgrounds.some((b) => b.id === id)) + return { kind: "shipped", id }; + } + return noEffect; +} + +export function serializeEffect(effect: BackgroundEffect): EffectId { + switch (effect.kind) { + case "blur": + return "blur"; + case "shipped": + return `image:${effect.id}`; + default: + return "none"; + } +} + +export function imagePathFor(id: string): string | undefined { + return shippedBackgrounds.find((b) => b.id === id)?.imagePath; +} diff --git a/src/settings/SettingsModal.test.tsx b/src/settings/SettingsModal.test.tsx new file mode 100644 index 000000000..348207c5e --- /dev/null +++ b/src/settings/SettingsModal.test.tsx @@ -0,0 +1,93 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { type MatrixClient } from "matrix-js-sdk"; +import { TooltipProvider } from "@vector-im/compound-web"; + +import { SettingsModal } from "./SettingsModal"; +import { backgroundEffect } from "./settings"; +import { MediaDevicesContext } from "../MediaDevicesContext"; +import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext"; +import { mockMediaDevices } from "../utils/test"; +import { constant } from "../state/Behavior"; +import { type MediaDevices } from "../state/MediaDevices"; + +vi.mock("../livekit/BackgroundEffectsContext", () => ({ + useBackgroundEffects: (): BackgroundEffectsState => ({ + supported: true, + processor: undefined, + }), +})); + +// Behaviors, as the device menus read them. +const noDevices = { + available$: constant(new Map()), + selected$: constant(undefined), + select: vi.fn(), +}; + +function renderVideoTab(): void { + render( + + )} + > + "@user:example.org", + getUser: () => null, + } as unknown as MatrixClient + } + /> + + , + ); +} + +const blurControl = (): HTMLElement => + screen.getByRole("checkbox", { name: "Blur the background of the video" }); + +describe("SettingsModal", () => { + afterEach(() => backgroundEffect.setValue("none")); + + it("blur control turns blur on and off", async () => { + const user = userEvent.setup(); + renderVideoTab(); + expect(blurControl()).not.toBeChecked(); + + await user.click(blurControl()); + expect(backgroundEffect.getValue()).toBe("blur"); + expect(blurControl()).toBeChecked(); + + await user.click(blurControl()); + expect(backgroundEffect.getValue()).toBe("none"); + expect(blurControl()).not.toBeChecked(); + }); + + it("blur control reflects and replaces an image background", async () => { + const user = userEvent.setup(); + backgroundEffect.setValue("image:arc"); + renderVideoTab(); + expect(blurControl()).not.toBeChecked(); + + await user.click(blurControl()); + expect(backgroundEffect.getValue()).toBe("blur"); + }); +}); diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 83166bcd6..7dc90c6de 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -22,13 +22,14 @@ import { useHostBridge } from "../HostBridge"; import { useSetting, soundEffectVolume as soundEffectVolumeSetting, - backgroundBlur as backgroundBlurSetting, + backgroundEffect as backgroundEffectSetting, developerMode, } from "./settings"; import { PreferencesSettingsTab } from "./PreferencesSettingsTab"; import { Slider } from "../Slider"; import { DeviceSelection } from "./DeviceSelection"; import { useBackgroundEffects } from "../livekit/BackgroundEffectsContext"; +import { parseEffect, serializeEffect } from "../livekit/backgroundEffects"; import { DeveloperSettingsTab, type DeveloperSettingsSnapshot, @@ -83,7 +84,9 @@ export const SettingsModal: FC = ({ const BlurCheckbox: React.FC = (): ReactNode => { const { supported } = useBackgroundEffects(); - const [blurActive, setBlurActive] = useSetting(backgroundBlurSetting); + // The camera menu's setting, of which this control sees only blur. + const [effect, setEffect] = useSetting(backgroundEffectSetting); + const blurActive = parseEffect(effect).kind === "blur"; return ( <> @@ -97,8 +100,12 @@ export const SettingsModal: FC = ({ supported ? "" : t("settings.blur_not_supported_by_browser") } type="checkbox" - checked={!!blurActive} - onChange={(b): void => setBlurActive(b.target.checked)} + checked={blurActive} + onChange={(b): void => + setEffect( + serializeEffect({ kind: b.target.checked ? "blur" : "none" }), + ) + } disabled={!supported} /> diff --git a/src/settings/settings.ts b/src/settings/settings.ts index f108eb5f9..06d441cbc 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -12,6 +12,7 @@ import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; import { type Behavior } from "../state/Behavior"; import { useBehavior } from "../useBehavior"; import { MatrixRTCMode } from "../config/ConfigOptions"; +import { type EffectId } from "../livekit/backgroundEffects"; export class Setting { public constructor( @@ -115,7 +116,14 @@ export const videoInput = new Setting( undefined, ); -export const backgroundBlur = new Setting("background-blur", false); +// Only read, to carry blur over for anyone who had it on. +const backgroundBlur = new Setting("background-blur", false); + +/** The chosen background effect, in the form `parseEffect` reads. */ +export const backgroundEffect = new Setting( + "background-effect", + backgroundBlur.getValue() ? "blur" : "none", +); export const showHandRaisedTimer = new Setting( "hand-raised-show-timer", diff --git a/src/state/BackgroundEffects.test.ts b/src/state/BackgroundEffects.test.ts index a3c30ce64..ebe1597a5 100644 --- a/src/state/BackgroundEffects.test.ts +++ b/src/state/BackgroundEffects.test.ts @@ -15,6 +15,7 @@ import { type BackgroundEffectsOptions, } from "./BackgroundEffects"; import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext"; +import { shippedBackgrounds } from "../livekit/backgroundEffects"; import { flushPromises, testScope, withTestScheduler } from "../utils/test"; /** A pipeline that records what it is switched to. */ @@ -61,7 +62,7 @@ describe("the pipeline's state", () => { withTestScheduler(({ behavior, expectObservable }) => { const effects = createBackgroundEffects(testScope(), { supported: true, - blur$: behavior(effect, { n: false, b: true }), + effect$: behavior(effect, { n: "none", b: "blur" }), pipeline: fakePipeline().pipeline, }); expectObservable( @@ -70,12 +71,14 @@ describe("the pipeline's state", () => { }); } + it("defaults to no effect", () => testState({ effect: "n", expected: "i" })); + it("attaches on first use and stays attached", () => testState({ effect: "nbn", expected: "ia" })); }); describe("background effects", () => { - let blur$: BehaviorSubject; + let effect$: BehaviorSubject; let fake: ReturnType; function build( @@ -83,21 +86,34 @@ describe("background effects", () => { ): BackgroundEffects { return createBackgroundEffects(testScope(), { supported: true, - blur$, + effect$, pipeline: fake.pipeline, ...options, }); } - const blur = async (on: boolean): Promise => { - blur$.next(on); + const choose = async (raw: string): Promise => { + effect$.next(raw); await flushPromises(); }; + const blur = async (on: boolean): Promise => + choose(on ? "blur" : "none"); beforeEach(() => { - blur$ = new BehaviorSubject(false); + effect$ = new BehaviorSubject("none"); fake = fakePipeline(); }); + it("puts a shipped background on as that picture", async () => { + build(); + await choose(`image:${shippedBackgrounds[0].id}`); + expect(fake.switches).toEqual([ + { + mode: "virtual-background", + imagePath: shippedBackgrounds[0].imagePath, + }, + ]); + }); + it("switches in place rather than reattaching", async () => { const effects = build(); await blur(true); diff --git a/src/state/BackgroundEffects.ts b/src/state/BackgroundEffects.ts index ba4ac114a..68e0cbbf1 100644 --- a/src/state/BackgroundEffects.ts +++ b/src/state/BackgroundEffects.ts @@ -16,14 +16,18 @@ import { deepCompare } from "matrix-js-sdk/lib/utils"; import { type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext"; - -const blurRadius = 15; +import { + type BackgroundEffect, + blurRadius, + imagePathFor, + parseEffect, +} from "../livekit/backgroundEffects"; export interface BackgroundEffectsOptions { /** Whether this browser can run a pipeline at all. */ supported: boolean; - /** Whether blur is chosen. */ - blur$: Behavior; + /** The effect chosen, as the setting stores it. */ + effect$: Behavior; /** The background processor pipeline to be switched between effects. */ pipeline: BackgroundProcessorWrapper; } @@ -36,10 +40,16 @@ export interface BackgroundEffects { /** Switches the pipeline as the choice changes, for as long as the scope lasts. */ export function createBackgroundEffects( scope: ObservableScope, - { supported, blur$, pipeline }: BackgroundEffectsOptions, + { supported, effect$, pipeline }: BackgroundEffectsOptions, ): BackgroundEffects { + const choice$ = effect$.pipe(map(parseEffect)); + const wanted$ = choice$.pipe( + map((effect) => effect.kind !== "none"), + distinctUntilChanged(), + ); + const state$ = scope.behavior( - blur$.pipe( + wanted$.pipe( scan( (previous, wanted) => { // Attached the first time an effect is wanted and never detached @@ -56,13 +66,9 @@ export function createBackgroundEffects( const switchOptions$ = scope.behavior< SwitchBackgroundProcessorOptions | undefined >( - combineLatest([state$, blur$]).pipe( - map(([{ processor }, blur]) => - processor === undefined - ? undefined - : blur - ? { mode: "background-blur", blurRadius } - : { mode: "disabled" }, + combineLatest([state$, choice$]).pipe( + map(([{ processor }, effect]) => + processor === undefined ? undefined : switchOptionsFor(effect), ), distinctUntilChanged(deepCompare), ), @@ -79,3 +85,20 @@ export function createBackgroundEffects( return { state$ }; } + +function switchOptionsFor( + effect: BackgroundEffect, +): SwitchBackgroundProcessorOptions { + switch (effect.kind) { + case "blur": + return { mode: "background-blur", blurRadius }; + case "shipped": { + const imagePath = imagePathFor(effect.id); + return imagePath + ? { mode: "virtual-background", imagePath } + : { mode: "disabled" }; + } + default: + return { mode: "disabled" }; + } +}