Show the wait for the first background effect on its tile

- The first effect chosen in a session shows a spinner on its own tile,
  in place of the tick, until a frame carrying it is drawn. Later changes
  show nothing, having nothing to prepare.
- Ended by the frame, not the promises, which resolve while the segmenter
  is still being built. The transformer reports the first frame that went
  through the segmenter; one passed through untouched doesn't count.
- The pipeline publishes the wait with its state, and both footer view
  models take it as an input.
- The lobby tests' provider mock gains the observable hook, built once: a
  fresh observable per render rebuilt the footer on every render.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
fkwp
2026-10-08 12:22:44 +02:00
co-authored by Claude Opus 5.5
parent 60b25e205e
commit 2b05bfb31a
16 changed files with 232 additions and 29 deletions
+1
View File
@@ -136,6 +136,7 @@ export const Default: Story = {
})),
],
backgroundEffectNotice: undefined,
backgroundEffectSettling: false,
hangup: fn(),
buttonSize: "lg",
showFooter: true,
+4
View File
@@ -87,6 +87,8 @@ export interface FooterState {
backgroundEffects: BackgroundEffectChoice[];
/** What the user is told about effects here, if anything. */
backgroundEffectNotice: "unavailable" | "slow" | undefined;
/** Whether the first effect chosen is still being prepared. */
backgroundEffectSettling: boolean;
showFooter: boolean;
/* This is needed for WindowMode = "flat" */
@@ -173,6 +175,7 @@ export const CallFooter: FC<FooterProps> = ({
const backgroundEffect = useBehavior(vm.backgroundEffect$);
const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$);
const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$);
const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$);
const backgroundEffects = useBackgroundEffectLabels(
useBehavior(vm.backgroundEffects$),
);
@@ -240,6 +243,7 @@ export const CallFooter: FC<FooterProps> = ({
backgroundEffects={backgroundEffects}
selectedBackgroundEffect={backgroundEffect}
onSelectBackgroundEffect={selectBackgroundEffect}
backgroundEffectSettling={backgroundEffectSettling}
backgroundEffectNotice={
backgroundEffectNotice === "unavailable"
? t("background_effects.unavailable")
@@ -116,6 +116,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(gridLayout),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
constant(false),
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
@@ -158,6 +159,7 @@ describe("createCallFooterViewModel", () => {
select: vi.fn(),
},
}),
constant(false),
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
@@ -176,6 +178,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(layout),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
constant(false),
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
@@ -198,6 +201,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(gridLayout),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
constant(false),
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
@@ -243,6 +247,7 @@ describe("createCallFooterViewModel", () => {
testScope(),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
constant(false),
/* openSettings */ undefined,
/* hangup */ undefined,
/* showLogo */ false,
@@ -312,6 +317,23 @@ describe("createCallFooterViewModel", () => {
);
});
it("passes on the wait for the first effect", () => {
platformMock.mockReturnValue("desktop");
const settling$ = new BehaviorSubject(true);
const vm = createLobbyFooterViewModel(
testScope(),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
settling$,
/* openSettings */ undefined,
/* hangup */ undefined,
/* showLogo */ false,
);
expect(vm.backgroundEffectSettling$.value).toBe(true);
settling$.next(false);
expect(vm.backgroundEffectSettling$.value).toBe(false);
});
it("availability is the same before and during a call", () => {
for (const supported of [true, false]) {
sdkSupportMock.mockReturnValue(supported);
@@ -321,6 +343,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(gridLayout),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
constant(false),
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
+31 -3
View File
@@ -5,7 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { combineLatest, map, type Observable, switchMap } from "rxjs";
import {
combineLatest,
distinctUntilChanged,
map,
type Observable,
switchMap,
} from "rxjs";
import { supportsAudioOutputSelection } from "livekit-client";
import {
@@ -72,6 +78,7 @@ function buildDeviceBehaviors(
mediaDevices: MediaDevices,
/** return empty arrays for audioOptions and videoOptions*/
disableSwitcher$: Behavior<boolean>,
backgroundEffectSettling$: Observable<boolean>,
): Pick<
ViewModel<FooterSnapshot>,
| "audioOptions$"
@@ -87,6 +94,7 @@ function buildDeviceBehaviors(
| "selectBackgroundEffect$"
| "backgroundEffects$"
| "backgroundEffectNotice$"
| "backgroundEffectSettling$"
> {
const options$ = (
available$: Behavior<Map<string, MenuOptions["label"]>>,
@@ -157,6 +165,10 @@ function buildDeviceBehaviors(
),
),
),
backgroundEffectSettling$: scope.behavior(
backgroundEffectSettling$.pipe(distinctUntilChanged()),
false,
),
};
}
@@ -167,6 +179,8 @@ function buildDeviceBehaviors(
* @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc.
* @param muteStates - Audio and video mute state + toggles.
* @param mediaDevices - Available and selected input devices.
* @param backgroundEffectSettling$ - Whether the first effect chosen is still
* being prepared.
* @param reactionIdentifier - The local user's reaction identifier string, or
* undefined when reactions are not supported (hides the reaction button).
* @param options - `showControls`: whether the call controls should be shown.
@@ -177,6 +191,7 @@ export function createCallFooterViewModel(
callModel: CallViewModel,
muteStates: MuteStates,
mediaDevices: MediaDevices,
backgroundEffectSettling$: Observable<boolean>,
reactionIdentifier: string | undefined,
options: { showControls: boolean; header: HeaderStyle },
): ViewModel<FooterSnapshot> {
@@ -191,7 +206,12 @@ export function createCallFooterViewModel(
);
return {
...buildMuteBehaviors(scope, muteStates),
...buildDeviceBehaviors(scope, mediaDevices, disableDeviceSwitcher$),
...buildDeviceBehaviors(
scope,
mediaDevices,
disableDeviceSwitcher$,
backgroundEffectSettling$,
),
// candidat to move into the FooterViewModel
showFooter$: callModel.showFooter$,
hideControls$: constant(!showControls),
@@ -253,6 +273,8 @@ export function createCallFooterViewModel(
* @param scope - ObservableScope that bounds the lifetime of derived behaviors.
* @param muteStates - Audio and video mute state + toggles.
* @param mediaDevices - Available and selected input devices.
* @param backgroundEffectSettling$ - Whether the first effect chosen is still
* being prepared.
* @param openSettings - Callback to open the settings modal, or undefined.
* @param hangup - Callback to leave/cancel, or undefined (hides the button).
* @param showLogo - Whether to show the Element Call logo.
@@ -261,6 +283,7 @@ export function createLobbyFooterViewModel(
scope: ObservableScope,
muteStates: MuteStates,
mediaDevices: MediaDevices,
backgroundEffectSettling$: Observable<boolean>,
openSettings: (() => void) | undefined,
hangup: (() => void) | undefined,
showLogo: boolean,
@@ -304,7 +327,12 @@ export function createLobbyFooterViewModel(
selectVideoButtonOption: undefined,
}),
...buildMuteBehaviors(scope, muteStates),
...buildDeviceBehaviors(scope, mediaDevices, constant(false)),
...buildDeviceBehaviors(
scope,
mediaDevices,
constant(false),
backgroundEffectSettling$,
),
};
}
@@ -305,6 +305,33 @@ export const BackgroundEffectsSlowInThisBrowser: Story = {
},
};
/** The first effect of a session shows its wait on its own tile. */
export const BackgroundEffectsSettling: Story = {
args: {
...VideoUnmute.args,
selectedBackgroundEffect: "blur",
backgroundEffectSettling: true,
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
const menu = within(document.body);
const chosen = await menu.findByRole("menuitemradio", {
name: "Blur",
checked: true,
});
await expect(chosen).toHaveAttribute("aria-busy", "true");
const marks = chosen.querySelectorAll("svg");
await expect(marks).toHaveLength(1);
await expect(getComputedStyle(marks[0]).animationName).not.toBe("none");
const section = menu.getByRole("group", { name: "Background effects" });
for (const other of within(section).getAllByRole("menuitemradio", {
checked: false,
}))
await expect(other).toHaveAttribute("aria-busy", "false");
},
};
/** In a short call the effects scroll into view with the list. */
export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = {
args: {
@@ -77,6 +77,8 @@ export interface MediaMuteAndSwitchButtonProps {
onSelectBackgroundEffect?: (id: string) => void;
/** What the user should know before choosing an effect, if anything. */
backgroundEffectNotice?: string;
/** Whether the first effect chosen is still being prepared. */
backgroundEffectSettling?: boolean;
/**
* For any toggle and option this method will be called.
* So toggles need to be implemented by listening here and setting the right toggle item to `enabled`
@@ -110,6 +112,7 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
selectedBackgroundEffect,
onSelectBackgroundEffect,
backgroundEffectNotice,
backgroundEffectSettling,
onSelect,
}) => {
// Requested but not yet selected. Keyed by kind too, since Chrome uses
@@ -422,6 +425,7 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
effects={backgroundEffects ?? []}
selected={selectedBackgroundEffect}
onSelect={onSelectBackgroundEffect}
settling={backgroundEffectSettling}
describedBy={
backgroundEffectNotice === undefined ? undefined : noticeId
}
@@ -15,6 +15,9 @@ import { ImageSegmenter } from "@mediapipe/tasks-vision";
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
const frame = {} as VideoFrame;
const controller = {} as TransformStreamDefaultController<VideoFrame>;
describe("BackgroundEffectTransformer", () => {
afterEach(() => vi.restoreAllMocks());
@@ -38,6 +41,32 @@ describe("BackgroundEffectTransformer", () => {
expect(load).toHaveBeenCalledWith("/background.jpg");
});
it("reports the first frame carrying an effect, once", async () => {
vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation(
async function (this: BackgroundTransformer) {
this.isFirstFrame = false;
return Promise.resolve();
},
);
const transformer = new BackgroundEffectTransformer({});
transformer.onFirstFrame = vi.fn();
await transformer.transform(frame, controller);
await transformer.transform(frame, controller);
expect(transformer.onFirstFrame).toHaveBeenCalledTimes(1);
});
it("does not report a frame passed through untouched", async () => {
vi.spyOn(BackgroundTransformer.prototype, "transform").mockResolvedValue();
const transformer = new BackgroundEffectTransformer({
backgroundDisabled: true,
});
transformer.onFirstFrame = vi.fn();
await transformer.transform(frame, controller);
expect(transformer.onFirstFrame).not.toHaveBeenCalled();
});
it("passes frames on untouched once disabled, even after blurring", async () => {
const transformer = new BackgroundEffectTransformer({});
await transformer.update({ blurRadius: 15, backgroundDisabled: false });
@@ -53,6 +53,20 @@ const wasmFileset: WasmFileset = {
* resource fetched from the public internet.
*/
export class BackgroundEffectTransformer extends BackgroundTransformer {
/** Called once, when the first frame carrying an effect has been drawn. */
public onFirstFrame: (() => void) | undefined;
public override async transform(
frame: VideoFrame,
controller: TransformStreamDefaultController<VideoFrame>,
): Promise<void> {
const priming = this.isFirstFrame;
await super.transform(frame, controller);
// Cleared only once a frame has been through the segmenter; a frame
// passed through untouched leaves it set.
if (priming && !this.isFirstFrame) this.onFirstFrame?.();
}
/**
* As the library's, except that disabling also drops the blur radius: kept,
* it has every frame segmented and the result thrown away, where with
+7 -4
View File
@@ -41,6 +41,8 @@ import { useBehavior } from "../useBehavior";
export type BackgroundEffectsState = {
supported: boolean | undefined;
processor: undefined | ProcessorWrapper<BackgroundOptions>;
/** From the first effect chosen until a frame carrying it is drawn. */
settling?: boolean;
};
const BackgroundEffectsContext = createContext<BackgroundEffects | undefined>(
@@ -126,14 +128,15 @@ export const BackgroundEffectsProvider: FC<Props> = ({ children }) => {
const [effects, setEffects] = useState<BackgroundEffects | null>(null);
useEffect(() => {
const scope = new ObservableScope();
const transformer = new BackgroundEffectTransformer({
backgroundDisabled: true,
});
setEffects(
createBackgroundEffects(scope, {
supported: supportsBackgroundProcessors(),
effect$: backgroundEffectSetting.value$,
pipeline: new OneStepPipeline(
new BackgroundEffectTransformer({ backgroundDisabled: true }),
"background-effect",
),
pipeline: new OneStepPipeline(transformer, "background-effect"),
transformer,
}),
);
return (): void => scope.end();
+2
View File
@@ -22,6 +22,7 @@ import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import classNames from "classnames";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { useTranslation } from "react-i18next";
import { map } from "rxjs";
import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
import { HeaderStyle, useUrlParams } from "../UrlParams";
@@ -184,6 +185,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
vm,
props.muteStates,
mediaDevices,
backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)),
`${props.client.getUserId()}:${props.client.getDeviceId()}`,
{ showControls: urlParams.showControls, header: urlParams.header },
);
+14 -5
View File
@@ -10,6 +10,7 @@ import { render, screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { TooltipProvider } from "@vector-im/compound-web";
import { type MatrixClient, type RoomSummary } from "matrix-js-sdk";
import { type Observable } from "rxjs";
import { KnockLobbyView } from "./KnockLobbyView";
import { LeaveToHomeProvider } from "../LeaveToHomeContext";
@@ -21,13 +22,21 @@ vi.mock("@livekit/components-react", () => ({
usePreviewTracks: (): unknown[] => [],
}));
vi.mock("../livekit/BackgroundEffectsContext", () => ({
useBackgroundEffects: (): BackgroundEffectsState => ({
vi.mock("../livekit/BackgroundEffectsContext", async () => {
const { of } = await import("rxjs");
const none: BackgroundEffectsState = {
supported: false,
processor: undefined,
}),
useSyncBackgroundEffects: (): void => {},
}));
};
// One observable, as the real hook keeps: a fresh one each render would
// rebuild the footer on every render.
const none$ = of(none);
return {
useBackgroundEffects: (): BackgroundEffectsState => none,
useBackgroundEffectsState$: (): Observable<BackgroundEffectsState> => none$,
useSyncBackgroundEffects: (): void => {},
};
});
vi.mock("react-use-measure", () => ({
default: (): [() => void, object] => [(): void => {}, {}],
+14 -6
View File
@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
import { afterEach, describe, expect, it, vi } from "vitest";
import { render, screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { of } from "rxjs";
import { type Observable, of } from "rxjs";
import { LeaveToHomeProvider } from "../LeaveToHomeContext";
import { TooltipProvider } from "@vector-im/compound-web";
import { type MatrixClient } from "matrix-js-sdk";
@@ -41,13 +41,21 @@ vi.mock("@livekit/components-react", () => ({
usePreviewTracks: (): unknown[] => [],
}));
vi.mock("../livekit/BackgroundEffectsContext", () => ({
useBackgroundEffects: (): BackgroundEffectsState => ({
vi.mock("../livekit/BackgroundEffectsContext", async () => {
const { of } = await import("rxjs");
const none: BackgroundEffectsState = {
supported: false,
processor: undefined,
}),
useSyncBackgroundEffects: (): void => {},
}));
};
// One observable, as the real hook keeps: a fresh one each render would
// rebuild the footer on every render.
const none$ = of(none);
return {
useBackgroundEffects: (): BackgroundEffectsState => none,
useBackgroundEffectsState$: (): Observable<BackgroundEffectsState> => none$,
useSyncBackgroundEffects: (): void => {},
};
});
vi.mock("react-use-measure", () => ({
default: (): [() => void, object] => [(): void => {}, {}],
+12 -1
View File
@@ -24,6 +24,7 @@ import {
type LocalVideoTrack,
Track,
} from "livekit-client";
import { map } from "rxjs";
import inCallStyles from "./InCallView.module.css";
import styles from "./LobbyView.module.css";
@@ -41,6 +42,7 @@ import { ObservableScope } from "../state/ObservableScope";
import { useInitial } from "../useInitial";
import {
useBackgroundEffects,
useBackgroundEffectsState$,
useSyncBackgroundEffects,
} from "../livekit/BackgroundEffectsContext";
import { getValue } from "../utils/observable";
@@ -137,6 +139,7 @@ export const LobbyView: FC<Props> = ({
);
const { processor } = useBackgroundEffects();
const backgroundEffectsState$ = useBackgroundEffectsState$();
const initialProcessor = useInitial(() => processor);
const localTrackOptions = useMemo<CreateLocalTracksOptions>(
@@ -195,6 +198,7 @@ export const LobbyView: FC<Props> = ({
footerScope,
muteStates,
devices,
backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)),
openSettings,
hangup,
// Logo and header are connected: only show the logo in SPA with header.
@@ -204,7 +208,14 @@ export const LobbyView: FC<Props> = ({
return (): void => {
footerScope.end();
};
}, [devices, hangup, hideHeader, muteStates, openSettings]);
}, [
devices,
backgroundEffectsState$,
hangup,
hideHeader,
muteStates,
openSettings,
]);
// TODO: Unify this component with InCallView, so we can get slick joining
// animations and don't have to feel bad about reusing its CSS
+20 -4
View File
@@ -46,25 +46,33 @@ function fakePipeline(): {
};
}
/** One letter per state: idle, attached. */
/** One letter per state: idle, waiting for a frame, attached. */
function letter(state: BackgroundEffectsState): string {
return state.processor === undefined ? "i" : "a";
if (state.processor === undefined) return "i";
return state.settling ? "w" : "a";
}
describe("the pipeline's state", () => {
function testState({
effect,
firstFrame = "",
expected,
}: {
effect: string;
firstFrame?: string;
expected: string;
}): void {
withTestScheduler(({ behavior, expectObservable }) => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
const transformer = {
onFirstFrame: undefined as (() => void) | undefined,
};
const effects = createBackgroundEffects(testScope(), {
supported: true,
effect$: behavior(effect, { n: "none", b: "blur" }),
pipeline: fakePipeline().pipeline,
transformer,
});
schedule(firstFrame, { f: () => transformer.onFirstFrame?.() });
expectObservable(
effects.state$.pipe(map(letter), distinctUntilChanged()),
).toBe(expected);
@@ -73,8 +81,15 @@ describe("the pipeline's state", () => {
it("defaults to no effect", () => testState({ effect: "n", expected: "i" }));
it("waits from the first effect until a frame carries it", () =>
testState({
effect: " nb-n-b",
firstFrame: " --f",
expected: " iwa",
}));
it("attaches on first use and stays attached", () =>
testState({ effect: "nbn", expected: "ia" }));
testState({ effect: "nb-n", firstFrame: "--f", expected: "iwa" }));
});
describe("background effects", () => {
@@ -88,6 +103,7 @@ describe("background effects", () => {
supported: true,
effect$,
pipeline: fake.pipeline,
transformer: { onFirstFrame: undefined },
...options,
});
}
+29 -6
View File
@@ -5,7 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { combineLatest, distinctUntilChanged, filter, map, scan } from "rxjs";
import {
combineLatest,
distinctUntilChanged,
filter,
map,
Observable,
scan,
} from "rxjs";
import {
type BackgroundProcessorWrapper,
type SwitchBackgroundProcessorOptions,
@@ -30,6 +37,8 @@ export interface BackgroundEffectsOptions {
effect$: Behavior<string>;
/** The background processor pipeline to be switched between effects. */
pipeline: BackgroundProcessorWrapper;
/** Tells, once, that a frame carrying an effect has been drawn. */
transformer: { onFirstFrame: (() => void) | undefined };
}
/** The background effect pipeline, as the camera tracks and the menus see it. */
@@ -40,7 +49,7 @@ export interface BackgroundEffects {
/** Switches the pipeline as the choice changes, for as long as the scope lasts. */
export function createBackgroundEffects(
scope: ObservableScope,
{ supported, effect$, pipeline }: BackgroundEffectsOptions,
{ supported, effect$, pipeline, transformer }: BackgroundEffectsOptions,
): BackgroundEffects {
const choice$ = effect$.pipe(map(parseEffect));
const wanted$ = choice$.pipe(
@@ -48,15 +57,29 @@ export function createBackgroundEffects(
distinctUntilChanged(),
);
const drewAFrame$ = scope.behavior(
new Observable<boolean>((subscriber) => {
subscriber.next(false);
transformer.onFirstFrame = (): void => subscriber.next(true);
return (): void => {
transformer.onFirstFrame = undefined;
};
}),
);
const state$ = scope.behavior(
wanted$.pipe(
scan<boolean, BackgroundEffectsState>(
(previous, wanted) => {
combineLatest([wanted$, drewAFrame$]).pipe(
scan<[boolean, boolean], BackgroundEffectsState>(
(previous, [wanted, drewAFrame]) => {
// Attached the first time an effect is wanted and never detached
// after, so someone who never turns one on pays for none of it.
const enable =
previous.processor !== undefined || (supported && wanted);
return { supported, processor: enable ? pipeline : undefined };
return {
supported,
processor: enable ? pipeline : undefined,
settling: enable && !drewAFrame,
};
},
{ supported, processor: undefined },
),
+1
View File
@@ -188,6 +188,7 @@ export function getBasicCallViewModelEnvironment(
vm,
muteStates,
mediaDevices,
constant(false),
"reactionId",
{ showControls: true, header: HeaderStyle.Standard },
);