Choose a background effect, not only blur on or off

- A catalogue of what can be chosen: no effect, blur, and two shipped
  backgrounds, the app's own gradients flattened onto its canvas colour.
  Their ids are what a saved choice stores, so they are named for the
  pictures.
- One setting for the choice, typed as the ids an effect can have and
  defaulting to blur for anyone who had it on. A stored value we no
  longer ship, or one that isn't even a string, reads as no effect: it
  comes from storage another build or a hand may have written.
- The pipeline follows it, switching in place to a picture as readily as
  to blur, and the camera menu's blur toggle reads and writes it.
- So does Settings' blur checkbox: a picture reads as blur off, and
  checking it replaces the picture with blur. The old on/off setting is
  only read, to carry blur over. First tests for the Settings modal,
  which fail with the checkbox on the old setting.
- The transformer's own init loads a picture chosen before it was built,
  as the base init does and it had not: the first effect always is, and
  the picture was drawn black.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/components/CallFooterViewModel.tsx

@@ -9,12 +9,13 @@ import { combineLatest, map, type Observable, switchMap } from "rxjs";
 import { supportsAudioOutputSelection } from "livekit-client";
 
 import { supportsBackgroundProcessors } from "../livekit/backgroundProcessing";
+import { parseEffect } from "../livekit/backgroundEffects";
 
 import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
 import { type MenuOptions } from "./MediaMuteAndSwitchButton";
 import { type MediaDevices } from "../state/MediaDevices";
 import {
-  backgroundBlur as backgroundBlurSettings,
+  backgroundEffect as backgroundEffectSetting,
   debugTileLayout as debugTileLayoutSetting,
 } from "../settings/settings";
 import { type Behavior, constant } from "../state/Behavior";
@@ -93,6 +94,11 @@ function buildDeviceBehaviors(
       ),
     );
 
+  const blurOn$ = scope.behavior(
+    backgroundEffectSetting.value$.pipe(
+      map((raw) => parseEffect(raw).kind === "blur"),
+    ),
+  );
   return {
     audioOptions$: scope.behavior(options$(mediaDevices.audioInput.available$)),
     selectedAudio$: scope.behavior(
@@ -118,17 +124,17 @@ function buildDeviceBehaviors(
     ),
     selectVideoButtonOption$: constant(mediaDevices.videoInput.select),
     toggleBlur$: scope.behavior(
-      combineLatest([backgroundBlurSettings.value$, disableSwitcher$]).pipe(
+      combineLatest([blurOn$, disableSwitcher$]).pipe(
         map(([current, switcherDisabled]) => {
           return !switcherDisabled && supportsBackgroundProcessors()
             ? (): void => {
-                backgroundBlurSettings.setValue(!current);
+                backgroundEffectSetting.setValue(current ? "none" : "blur");
               }
             : undefined;
         }),
       ),
     ),
-    videoBlurEnabled$: backgroundBlurSettings.value$,
+    videoBlurEnabled$: blurOn$,
   };
 }
 

src/livekit/BackgroundEffectTransformer.test.ts

@@ -5,11 +5,39 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+  BackgroundTransformer,
+  VideoTransformer,
+  type VideoTransformerInitOptions,
+} from "@livekit/track-processors";
+import { ImageSegmenter } from "@mediapipe/tasks-vision";
 
 import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
 
 describe("BackgroundEffectTransformer", () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  it("loads a background chosen before it was built", async () => {
+    vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
+    vi.spyOn(ImageSegmenter, "createFromOptions").mockResolvedValue(
+      {} as ImageSegmenter,
+    );
+    const load = vi
+      .spyOn(BackgroundTransformer.prototype, "loadAndSetBackground")
+      .mockResolvedValue();
+    const transformer = new BackgroundEffectTransformer({
+      backgroundDisabled: true,
+    });
+    // Chosen before the camera is processed, as the first effect always is:
+    // loaded then, there is nothing yet to draw it with.
+    await transformer.update({ imagePath: "/background.jpg" });
+    load.mockClear();
+
+    await transformer.init({} as VideoTransformerInitOptions);
+    expect(load).toHaveBeenCalledWith("/background.jpg");
+  });
+
   it("passes frames on untouched once disabled, even after blurring", async () => {
     const transformer = new BackgroundEffectTransformer({});
     await transformer.update({ blurRadius: 15, backgroundDisabled: false });

src/livekit/BackgroundEffectTransformer.ts

@@ -12,6 +12,7 @@ import {
   type VideoTransformerInitOptions,
   type BackgroundOptions,
 } from "@livekit/track-processors";
+import { logger } from "matrix-js-sdk/lib/logger";
 import { ImageSegmenter } from "@mediapipe/tasks-vision";
 
 import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url";
@@ -87,6 +88,11 @@ export class BackgroundEffectTransformer extends BackgroundTransformer {
     });
 
     // BackgroundTransformer's own init applies these, and this one replaces it.
+    if (this.options.imagePath) {
+      await this.loadAndSetBackground(this.options.imagePath).catch((e) =>
+        logger.warn("Failed to load the background image", e),
+      );
+    }
     if (this.options.blurRadius) {
       this.gl?.setBlurRadius(this.options.blurRadius);
     }

src/livekit/BackgroundEffectsContext.test.ts

@@ -24,7 +24,7 @@ import {
   createAppBackgroundEffects,
 } from "./BackgroundEffectsContext";
 import { type BackgroundEffects } from "../state/BackgroundEffects";
-import { backgroundBlur } from "../settings/settings";
+import { backgroundEffect } from "../settings/settings";
 import { constant } from "../state/Behavior";
 import { flushPromises, testScope } from "../utils/test";
 
@@ -168,7 +168,7 @@ describe("BackgroundEffectsProvider", () => {
   ): ReturnType<typeof createElement> => createElement(Surfaces, { tracks });
   const blur = async (on: boolean): Promise<void> => {
     await act(async () => {
-      backgroundBlur.setValue(on);
+      backgroundEffect.setValue(on ? "blur" : "none");
       await flushPromises();
     });
   };
@@ -181,9 +181,9 @@ describe("BackgroundEffectsProvider", () => {
     pipelines.destroyed = 0;
     pipelines.switches = [];
     platformMock.platform = "desktop";
-    backgroundBlur.setValue(false);
+    backgroundEffect.setValue("none");
   });
-  afterEach(() => backgroundBlur.setValue(false));
+  afterEach(() => backgroundEffect.setValue("none"));
 
   // A pipeline primes itself when it is built and when it is destroyed, and a
   // primed pipeline lets one frame through untouched: so the frame is spent

src/livekit/BackgroundEffectsContext.tsx

@@ -15,7 +15,7 @@ import { type LocalVideoTrack } from "livekit-client";
 import { logger } from "matrix-js-sdk/lib/logger";
 import { combineLatest } from "rxjs";
 
-import { backgroundBlur as backgroundBlurSettings } from "../settings/settings";
+import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
 import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
 import { OneStepPipeline } from "./OneStepPipeline";
 import { supportsBackgroundProcessors } from "./backgroundProcessing";
@@ -117,7 +117,7 @@ export function createAppBackgroundEffects(
 ): BackgroundEffects {
   return createBackgroundEffects(scope, {
     supported: supportsBackgroundProcessors(),
-    blur$: backgroundBlurSettings.value$,
+    effect$: backgroundEffectSetting.value$,
     pipeline: new OneStepPipeline(
       new BackgroundEffectTransformer({ backgroundDisabled: true }),
       "background-effect",

src/livekit/backgroundEffects.test.ts

@@ -0,0 +1,53 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { readFile } from "node:fs/promises";
+import { describe, expect, test } from "vitest";
+
+import {
+  type BackgroundEffect,
+  imagePathFor,
+  parseEffect,
+  serializeEffect,
+  shippedBackgrounds,
+} from "./backgroundEffects";
+
+describe("the chosen background effect", () => {
+  test("round-trips through its stored form", () => {
+    const effects: BackgroundEffect[] = [
+      { kind: "none" },
+      { kind: "blur" },
+      ...shippedBackgrounds.map(({ id }) => ({ kind: "shipped" as const, id })),
+    ];
+    for (const effect of effects)
+      expect(parseEffect(serializeEffect(effect))).toEqual(effect);
+  });
+
+  test("falls back to no effect when the stored form names nothing we ship", () => {
+    expect(parseEffect("image:gone")).toEqual({ kind: "none" });
+    expect(parseEffect("")).toEqual({ kind: "none" });
+  });
+
+  test("reads a stored value that is not a string as no effect", () => {
+    for (const raw of [null, 1, {}, ["blur"]])
+      expect(parseEffect(raw)).toEqual({ kind: "none" });
+  });
+
+  test("gives every shipped background an image to draw", () => {
+    for (const { id } of shippedBackgrounds)
+      expect(imagePathFor(id)).toBeTruthy();
+  });
+
+  // A background has to cover what is behind it: JPEG has no alpha channel.
+  test("ships only opaque backgrounds", async () => {
+    for (const { imagePath } of shippedBackgrounds) {
+      const name = imagePath.split("/").pop()!.split("?")[0];
+      const bytes = await readFile(`src/graphics/${name}`);
+      expect([bytes[0], bytes[1], bytes[2]]).toEqual([0xff, 0xd8, 0xff]);
+    }
+  });
+});

src/livekit/backgroundEffects.ts

@@ -0,0 +1,62 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import arc from "../graphics/background-arc.jpg?url";
+import glow from "../graphics/background-glow.jpg?url";
+
+export const blurRadius = 15;
+
+export interface ShippedBackground {
+  /** Stored in the user's settings when chosen, so it must not change. */
+  id: string;
+  imagePath: string;
+}
+
+export const shippedBackgrounds: ShippedBackground[] = [
+  { id: "arc", imagePath: arc },
+  { id: "glow", imagePath: glow },
+];
+
+export type BackgroundEffect =
+  | { kind: "none" }
+  | { kind: "blur" }
+  | { kind: "shipped"; id: string };
+
+/** An effect as the setting stores it, and as the menu names its option. */
+export type EffectId = "none" | "blur" | `image:${string}`;
+
+export const noEffect: BackgroundEffect = { kind: "none" };
+
+/**
+ * Reads a stored effect; one we no longer ship reads as no effect. Stored by
+ * hand or by another build, the value need not even be a string.
+ */
+export function parseEffect(raw: unknown): BackgroundEffect {
+  if (typeof raw !== "string") return noEffect;
+  if (raw === "blur") return { kind: "blur" };
+  if (raw.startsWith("image:")) {
+    const id = raw.slice("image:".length);
+    if (shippedBackgrounds.some((b) => b.id === id))
+      return { kind: "shipped", id };
+  }
+  return noEffect;
+}
+
+export function serializeEffect(effect: BackgroundEffect): EffectId {
+  switch (effect.kind) {
+    case "blur":
+      return "blur";
+    case "shipped":
+      return `image:${effect.id}`;
+    default:
+      return "none";
+  }
+}
+
+export function imagePathFor(id: string): string | undefined {
+  return shippedBackgrounds.find((b) => b.id === id)?.imagePath;
+}

src/settings/SettingsModal.test.tsx

@@ -0,0 +1,93 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { render, screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { type MatrixClient } from "matrix-js-sdk";
+import { TooltipProvider } from "@vector-im/compound-web";
+
+import { SettingsModal } from "./SettingsModal";
+import { backgroundEffect } from "./settings";
+import { MediaDevicesContext } from "../MediaDevicesContext";
+import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext";
+import { mockMediaDevices } from "../utils/test";
+import { constant } from "../state/Behavior";
+import { type MediaDevices } from "../state/MediaDevices";
+
+vi.mock("../livekit/BackgroundEffectsContext", () => ({
+  useBackgroundEffects: (): BackgroundEffectsState => ({
+    supported: true,
+    processor: undefined,
+  }),
+}));
+
+// Behaviors, as the device menus read them.
+const noDevices = {
+  available$: constant(new Map()),
+  selected$: constant(undefined),
+  select: vi.fn(),
+};
+
+function renderVideoTab(): void {
+  render(
+    <TooltipProvider>
+      <MediaDevicesContext
+        value={mockMediaDevices({
+          requestDeviceNames: vi.fn(),
+          audioInput: noDevices,
+          audioOutput: noDevices,
+          videoInput: noDevices,
+        } as unknown as Partial<MediaDevices>)}
+      >
+        <SettingsModal
+          open
+          onDismiss={vi.fn()}
+          tab="video"
+          onTabChange={vi.fn()}
+          client={
+            {
+              getUserId: () => "@user:example.org",
+              getUser: () => null,
+            } as unknown as MatrixClient
+          }
+        />
+      </MediaDevicesContext>
+    </TooltipProvider>,
+  );
+}
+
+const blurControl = (): HTMLElement =>
+  screen.getByRole("checkbox", { name: "Blur the background of the video" });
+
+describe("SettingsModal", () => {
+  afterEach(() => backgroundEffect.setValue("none"));
+
+  it("blur control turns blur on and off", async () => {
+    const user = userEvent.setup();
+    renderVideoTab();
+    expect(blurControl()).not.toBeChecked();
+
+    await user.click(blurControl());
+    expect(backgroundEffect.getValue()).toBe("blur");
+    expect(blurControl()).toBeChecked();
+
+    await user.click(blurControl());
+    expect(backgroundEffect.getValue()).toBe("none");
+    expect(blurControl()).not.toBeChecked();
+  });
+
+  it("blur control reflects and replaces an image background", async () => {
+    const user = userEvent.setup();
+    backgroundEffect.setValue("image:arc");
+    renderVideoTab();
+    expect(blurControl()).not.toBeChecked();
+
+    await user.click(blurControl());
+    expect(backgroundEffect.getValue()).toBe("blur");
+  });
+});

src/settings/SettingsModal.tsx

@@ -22,13 +22,14 @@ import { useHostBridge } from "../HostBridge";
 import {
   useSetting,
   soundEffectVolume as soundEffectVolumeSetting,
-  backgroundBlur as backgroundBlurSetting,
+  backgroundEffect as backgroundEffectSetting,
   developerMode,
 } from "./settings";
 import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
 import { Slider } from "../Slider";
 import { DeviceSelection } from "./DeviceSelection";
 import { useBackgroundEffects } from "../livekit/BackgroundEffectsContext";
+import { parseEffect, serializeEffect } from "../livekit/backgroundEffects";
 import {
   DeveloperSettingsTab,
   type DeveloperSettingsSnapshot,
@@ -83,7 +84,9 @@ export const SettingsModal: FC<Props> = ({
   const BlurCheckbox: React.FC = (): ReactNode => {
     const { supported } = useBackgroundEffects();
 
-    const [blurActive, setBlurActive] = useSetting(backgroundBlurSetting);
+    // The camera menu's setting, of which this control sees only blur.
+    const [effect, setEffect] = useSetting(backgroundEffectSetting);
+    const blurActive = parseEffect(effect).kind === "blur";
 
     return (
       <>
@@ -97,8 +100,12 @@ export const SettingsModal: FC<Props> = ({
               supported ? "" : t("settings.blur_not_supported_by_browser")
             }
             type="checkbox"
-            checked={!!blurActive}
-            onChange={(b): void => setBlurActive(b.target.checked)}
+            checked={blurActive}
+            onChange={(b): void =>
+              setEffect(
+                serializeEffect({ kind: b.target.checked ? "blur" : "none" }),
+              )
+            }
             disabled={!supported}
           />
         </FieldRow>

src/settings/settings.ts

@@ -12,6 +12,7 @@ import { PosthogAnalytics } from "../analytics/PosthogAnalytics";
 import { type Behavior } from "../state/Behavior";
 import { useBehavior } from "../useBehavior";
 import { MatrixRTCMode } from "../config/ConfigOptions";
+import { type EffectId } from "../livekit/backgroundEffects";
 
 export class Setting<T> {
   public constructor(
@@ -115,7 +116,14 @@ export const videoInput = new Setting<string | undefined>(
   undefined,
 );
 
-export const backgroundBlur = new Setting<boolean>("background-blur", false);
+// Only read, to carry blur over for anyone who had it on.
+const backgroundBlur = new Setting<boolean>("background-blur", false);
+
+/** The chosen background effect, in the form `parseEffect` reads. */
+export const backgroundEffect = new Setting<EffectId>(
+  "background-effect",
+  backgroundBlur.getValue() ? "blur" : "none",
+);
 
 export const showHandRaisedTimer = new Setting<boolean>(
   "hand-raised-show-timer",

src/state/BackgroundEffects.test.ts

@@ -15,6 +15,7 @@ import {
   type BackgroundEffectsOptions,
 } from "./BackgroundEffects";
 import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext";
+import { shippedBackgrounds } from "../livekit/backgroundEffects";
 import { flushPromises, testScope, withTestScheduler } from "../utils/test";
 
 /** A pipeline that records what it is switched to. */
@@ -61,7 +62,7 @@ describe("the pipeline's state", () => {
     withTestScheduler(({ behavior, expectObservable }) => {
       const effects = createBackgroundEffects(testScope(), {
         supported: true,
-        blur$: behavior(effect, { n: false, b: true }),
+        effect$: behavior(effect, { n: "none", b: "blur" }),
         pipeline: fakePipeline().pipeline,
       });
       expectObservable(
@@ -70,12 +71,14 @@ describe("the pipeline's state", () => {
     });
   }
 
+  it("defaults to no effect", () => testState({ effect: "n", expected: "i" }));
+
   it("attaches on first use and stays attached", () =>
     testState({ effect: "nbn", expected: "ia" }));
 });
 
 describe("background effects", () => {
-  let blur$: BehaviorSubject<boolean>;
+  let effect$: BehaviorSubject<string>;
   let fake: ReturnType<typeof fakePipeline>;
 
   function build(
@@ -83,21 +86,34 @@ describe("background effects", () => {
   ): BackgroundEffects {
     return createBackgroundEffects(testScope(), {
       supported: true,
-      blur$,
+      effect$,
       pipeline: fake.pipeline,
       ...options,
     });
   }
-  const blur = async (on: boolean): Promise<void> => {
-    blur$.next(on);
+  const choose = async (raw: string): Promise<void> => {
+    effect$.next(raw);
     await flushPromises();
   };
+  const blur = async (on: boolean): Promise<void> =>
+    choose(on ? "blur" : "none");
 
   beforeEach(() => {
-    blur$ = new BehaviorSubject(false);
+    effect$ = new BehaviorSubject("none");
     fake = fakePipeline();
   });
 
+  it("puts a shipped background on as that picture", async () => {
+    build();
+    await choose(`image:${shippedBackgrounds[0].id}`);
+    expect(fake.switches).toEqual([
+      {
+        mode: "virtual-background",
+        imagePath: shippedBackgrounds[0].imagePath,
+      },
+    ]);
+  });
+
   it("switches in place rather than reattaching", async () => {
     const effects = build();
     await blur(true);

src/state/BackgroundEffects.ts

@@ -16,14 +16,18 @@ import { deepCompare } from "matrix-js-sdk/lib/utils";
 import { type Behavior } from "./Behavior";
 import { type ObservableScope } from "./ObservableScope";
 import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext";
-
-const blurRadius = 15;
+import {
+  type BackgroundEffect,
+  blurRadius,
+  imagePathFor,
+  parseEffect,
+} from "../livekit/backgroundEffects";
 
 export interface BackgroundEffectsOptions {
   /** Whether this browser can run a pipeline at all. */
   supported: boolean;
-  /** Whether blur is chosen. */
-  blur$: Behavior<boolean>;
+  /** The effect chosen, as the setting stores it. */
+  effect$: Behavior<string>;
   /** The background processor pipeline to be switched between effects. */
   pipeline: BackgroundProcessorWrapper;
 }
@@ -36,10 +40,16 @@ export interface BackgroundEffects {
 /** Switches the pipeline as the choice changes, for as long as the scope lasts. */
 export function createBackgroundEffects(
   scope: ObservableScope,
-  { supported, blur$, pipeline }: BackgroundEffectsOptions,
+  { supported, effect$, pipeline }: BackgroundEffectsOptions,
 ): BackgroundEffects {
+  const choice$ = effect$.pipe(map(parseEffect));
+  const wanted$ = choice$.pipe(
+    map((effect) => effect.kind !== "none"),
+    distinctUntilChanged(),
+  );
+
   const state$ = scope.behavior(
-    blur$.pipe(
+    wanted$.pipe(
       scan<boolean, BackgroundEffectsState>(
         (previous, wanted) => {
           // Attached the first time an effect is wanted and never detached
@@ -56,13 +66,9 @@ export function createBackgroundEffects(
   const switchOptions$ = scope.behavior<
     SwitchBackgroundProcessorOptions | undefined
   >(
-    combineLatest([state$, blur$]).pipe(
-      map(([{ processor }, blur]) =>
-        processor === undefined
-          ? undefined
-          : blur
-            ? { mode: "background-blur", blurRadius }
-            : { mode: "disabled" },
+    combineLatest([state$, choice$]).pipe(
+      map(([{ processor }, effect]) =>
+        processor === undefined ? undefined : switchOptionsFor(effect),
       ),
       distinctUntilChanged(deepCompare),
     ),
@@ -79,3 +85,20 @@ export function createBackgroundEffects(
 
   return { state$ };
 }
+
+function switchOptionsFor(
+  effect: BackgroundEffect,
+): SwitchBackgroundProcessorOptions {
+  switch (effect.kind) {
+    case "blur":
+      return { mode: "background-blur", blurRadius };
+    case "shipped": {
+      const imagePath = imagePathFor(effect.id);
+      return imagePath
+        ? { mode: "virtual-background", imagePath }
+        : { mode: "disabled" };
+    }
+    default:
+      return { mode: "disabled" };
+  }
+}