From d916fc33cd7fb40457a8625da2ba2236a491a3f5 Mon Sep 17 00:00:00 2001 From: fkwp Date: Thu, 24 Sep 2026 15:17:21 +0200 Subject: [PATCH] Say where background effects run slowly - Where the browser has no MediaStreamTrackProcessor, effects draw each frame through a canvas: slower, and the first build stalls the page. They stay offered and choosable there, with a notice under the grid that says so and describes the group. - Decided in the view model, beside whether effects can run at all, so the lobby and the call give the same answer. - The notice stays within the menu's width however long the sentence, which the story checks. Co-Authored-By: Claude Opus 5.5 (1M context) --- locales/en/app.json | 1 + src/components/CallFooter.tsx | 6 ++- src/components/CallFooterViewModel.test.ts | 12 ++++++ src/components/CallFooterViewModel.tsx | 12 +++++- .../MediaMuteAndSwitchButton.stories.tsx | 37 +++++++++++++++++++ src/livekit/backgroundProcessing.ts | 14 ++++++- 6 files changed, 77 insertions(+), 5 deletions(-) diff --git a/locales/en/app.json b/locales/en/app.json index c010d5fb1..c3c2dd990 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -26,6 +26,7 @@ "blur": "Blur", "none": "None", "numbered": "Background {{n}}", + "slow": "Background effects run slowly on this platform, which may cause your video to stutter.", "unavailable": "Background effects are not supported on this platform." }, "call_ended_view": { diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index ca64e8bde..3aa8badcb 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -86,7 +86,7 @@ export interface FooterState { /** Every effect on offer, in the order they are shown. */ backgroundEffects: BackgroundEffectChoice[]; /** What the user is told about effects here, if anything. */ - backgroundEffectNotice: "unavailable" | undefined; + backgroundEffectNotice: "unavailable" | "slow" | undefined; showFooter: boolean; /* This is needed for WindowMode = "flat" */ @@ -243,7 +243,9 @@ export const CallFooter: FC = ({ backgroundEffectNotice={ backgroundEffectNotice === "unavailable" ? t("background_effects.unavailable") - : undefined + : backgroundEffectNotice === "slow" + ? t("background_effects.slow") + : undefined } />, ); diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index 287786aae..e84963253 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -31,8 +31,10 @@ vi.mock("../Platform", () => ({ // The SDK's own check needs WebGL and canvas APIs jsdom does not have. const sdkSupportMock = vi.hoisted(() => vi.fn(() => false)); +const modernRouteMock = vi.hoisted(() => vi.fn(() => true)); vi.mock("@livekit/track-processors", () => ({ supportsBackgroundProcessors: (): boolean => sdkSupportMock(), + supportsModernBackgroundProcessors: (): boolean => modernRouteMock(), })); const outputSelectionMock = vi.hoisted(() => vi.fn(() => true)); @@ -280,6 +282,16 @@ describe("createCallFooterViewModel", () => { expect(backgroundEffectSetting.getValue()).toBe("none"); }); + it("says they run slowly where only the slower route exists", () => { + sdkSupportMock.mockReturnValue(true); + modernRouteMock.mockReturnValue(false); + const slow = lobbyOn("desktop"); + expect(slow.backgroundEffectNotice$.value).toBe("slow"); + expect(slow.selectBackgroundEffect$.value).toBeDefined(); + modernRouteMock.mockReturnValue(true); + expect(lobbyOn("desktop").backgroundEffectNotice$.value).toBeUndefined(); + }); + it("offers every effect in order", () => { sdkSupportMock.mockReturnValue(true); expect(lobbyOn("desktop").backgroundEffects$.value).toEqual([ diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index 3df9301ae..50f253842 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -8,7 +8,10 @@ Please see LICENSE in the repository root for full details. import { combineLatest, map, type Observable, switchMap } from "rxjs"; import { supportsAudioOutputSelection } from "livekit-client"; -import { supportsBackgroundProcessors } from "../livekit/backgroundProcessing"; +import { + supportsBackgroundProcessors, + usesFallbackProcessing, +} from "../livekit/backgroundProcessing"; import { parseEffect, serializeEffect, @@ -101,6 +104,7 @@ function buildDeviceBehaviors( ); const supported = supportsBackgroundProcessors(); + const slow = usesFallbackProcessing(); const offered$ = disableSwitcher$.pipe( map((switcherDisabled) => !switcherDisabled && supported), ); @@ -147,7 +151,11 @@ function buildDeviceBehaviors( ), backgroundEffects$: constant(backgroundEffectChoices()), backgroundEffectNotice$: scope.behavior( - offered$.pipe(map((offered) => (offered ? undefined : "unavailable"))), + offered$.pipe( + map((offered) => + !offered ? "unavailable" : slow ? "slow" : undefined, + ), + ), ), }; } diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index 6cfb29352..56cb3a080 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -269,6 +269,43 @@ export const BackgroundEffectsUnavailable: Story = { }, }; +/** Where only the slower route exists: offered and choosable, with the cost said. */ +export const BackgroundEffectsSlowInThisBrowser: Story = { + args: { + ...VideoUnmute.args, + backgroundEffectNotice: + "Background effects run slowly on this platform, which may cause your video to stutter.", + }, + play: async ({ args, canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Camera" })); + const menu = within(document.body); + const section = await menu.findByRole("group", { + name: "Background effects", + }); + await expect(section).toHaveAccessibleDescription( + "Background effects run slowly on this platform, which may cause your video to stutter.", + ); + for (const tile of within(section).getAllByRole("menuitemradio")) + await expect(tile).not.toHaveAttribute("aria-disabled"); + await userEvent.click( + within(section).getByRole("menuitemradio", { name: "Background 1" }), + ); + await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith( + "image:arc", + ); + + const notice = document.body.querySelector(`.${styles.notice}`)!; + const frame = document.body + .querySelector("[role='menu']")! + .getBoundingClientRect(); + await expect(frame.width).toBeLessThanOrEqual(296); + await expect(notice.getBoundingClientRect().right).toBeLessThanOrEqual( + frame.right, + ); + }, +}; + /** In a short call the effects scroll into view with the list. */ export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = { args: { diff --git a/src/livekit/backgroundProcessing.ts b/src/livekit/backgroundProcessing.ts index 3dc32e5ec..c131c78b0 100644 --- a/src/livekit/backgroundProcessing.ts +++ b/src/livekit/backgroundProcessing.ts @@ -5,7 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk } from "@livekit/track-processors"; +import { + supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk, + supportsModernBackgroundProcessors, +} from "@livekit/track-processors"; /** * Whether this browser can run background effects: the one answer for the @@ -14,3 +17,12 @@ import { supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk export function supportsBackgroundProcessors(): boolean { return supportsBackgroundProcessorsLivekitSdk(); } + +/** + * Whether effects, where they run at all, draw every frame through a canvas: + * the slower route, which costs frame rate and stalls the page while the + * first one is built. + */ +export function usesFallbackProcessing(): boolean { + return !supportsModernBackgroundProcessors(); +}