From f0675f26ab83c821766cc7d33e9c6a724abb0fa2 Mon Sep 17 00:00:00 2001 From: Timo <16718859+toger5@users.noreply.github.com> Date: Fri, 20 Nov 2026 18:07:51 +0100 Subject: [PATCH] Publish late device (#4485) * Publish a camera or microphone listed after the call view opened Without a lobby the call decides what to publish once, as its view mounts. A device the browser listed a moment later switched to its default "on" without the mute handler being asked, so the button showed on and nothing was published: the other side kept waiting for media. The default is now asked of the handler like any other request where one is set, so the call publishes the device. A device that comes back while a handler is set stays off, since the user may have turned it off; the button used to show it as on there too. Without a handler, as in the lobby, the default is assumed at once as before. Follows the patch tiran133 posted on the issue. A stop-gap until one reconciler owns the local media. Fixes https://github.com/element-hq/element-call/issues/4403 --------- Co-authored-by: Claude Fable 5.2 --- playwright/widget/late-device.spec.ts | 95 +++++++++++++++++++++++ src/state/MuteStates.test.ts | 105 ++++++++++++++++++++++++++ src/state/MuteStates.ts | 36 ++++++--- 3 files changed, 226 insertions(+), 10 deletions(-) create mode 100644 playwright/widget/late-device.spec.ts diff --git a/playwright/widget/late-device.spec.ts b/playwright/widget/late-device.spec.ts new file mode 100644 index 000000000..5f7a0ab58 --- /dev/null +++ b/playwright/widget/late-device.spec.ts @@ -0,0 +1,95 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test, type Page } from "@playwright/test"; + +import { widgetTest } from "../fixtures/widget-user.ts"; +import { TestHelpers } from "./test-helpers.ts"; + +widgetTest.use({ callType: "dm" }); + +// The bug from issue #4403, in a real call between two users in Element Web. +// 1. Brooks' browser is made slow: it lists its camera and microphone only +// after the call is on screen. +// 2. Brooks starts a video call in a DM. A DM call has no lobby. +// 3. Whistler picks up. +// 4. Brooks' camera button shows "on". +// 5. Whistler must see two videos: their own and the one brooks sends. +// Before the fix whistler saw only their own video. +// +// What it guards: the unit tests only check the mute button's own logic. This +// test checks the whole chain: button, call, SFU, and what the other person +// really sees. The bug depends on timing (camera listed before or after the +// call opens), so the normal DM call tests pass by luck most of the time. +// This test forces the bad timing every run, so a regression shows up always +// and not only sometimes. +widgetTest( + "Publishes a camera the browser lists only once the call is on screen", + async ({ asWidget, browserName }) => { + test.skip( + browserName === "firefox", + "The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled", + ); + + // Two Element Webs and a real call. + test.slow(); + const { brooks, whistler } = asWidget; + await listDevicesOnlyInTheCall(brooks.page); + + await TestHelpers.startCallInCurrentRoom(brooks.page, false); + await expect(whistler.page.getByText("Incoming video call")).toBeVisible(); + await whistler.page + .getByRole("alert") + .getByRole("button", { name: "Join" }) + .click(); + + const brooksFrame = brooks.page + .locator('iframe[title="Element Call"]') + .contentFrame(); + await expect( + brooksFrame.getByRole("switch", { name: "Stop video", checked: true }), + ).toBeVisible(); + + // Whistler's own camera and the one brooks sends + const whistlerFrame = whistler.page + .locator('iframe[title="Element Call"]') + .contentFrame(); + await TestHelpers.expectVisibleVideoCount(whistlerFrame, 2); + await expect(whistlerFrame.getByText("Waiting for media...")).toHaveCount( + 0, + ); + }, +); + +/** + * Holds the device list back in the widget until the call view shows a tile, + * as a browser that is slow to answer does. Call before the widget loads. + * + * How: it replaces the browser's `enumerateDevices` with one that waits. It + * checks every 50ms whether a video tile is on the page, waits half a second + * more, and only then gives the real list. Element Web itself (the outer + * page) is left alone; only the call inside the iframe is slowed down. + */ +async function listDevicesOnlyInTheCall(page: Page): Promise { + await page.addInitScript(() => { + if (window.parent === window) return; + const devices = navigator.mediaDevices; + const enumerate = devices.enumerateDevices.bind(devices); + const inTheCall = new Promise((resolve) => { + const poll = setInterval(() => { + if (document.querySelector("[data-testid='videoTile']") === null) + return; + clearInterval(poll); + setTimeout(resolve, 500); + }, 50); + }); + devices.enumerateDevices = async (): Promise => { + await inTheCall; + return enumerate(); + }; + }); +} diff --git a/src/state/MuteStates.test.ts b/src/state/MuteStates.test.ts index efccd70be..37ae4848b 100644 --- a/src/state/MuteStates.test.ts +++ b/src/state/MuteStates.test.ts @@ -169,6 +169,111 @@ describe("MuteState", () => { expect(syncingValues).toContain(true); expect(syncingValues.at(-1)).toBe(false); }); + + // The bug from issue #4403: we join a call without a lobby, and the browser + // tells us about the camera only after the call has started. + // 1. Start with no camera, with the camera set to be on by default. + // 2. Set a handler, like the call does when it starts. + // 3. The camera shows up. + // The handler must be called with "on", so the call publishes the camera, + // and the button must show "on" only after the handler is done. + // + // What it guards: the button and the real camera must never disagree. In + // the bug the button jumped to "on" by itself and nobody told the call, so + // the other side saw "Waiting for media". If someone changes how the + // default is applied and forgets the handler again, this test fails. + test("asks the handler for the default of a device listed after it was set", async () => { + const available$ = new BehaviorSubject(new Map()); + const muteState = new MuteState( + testScope, + aDevice(available$), + true, + constant(false), + ); + const handler = vi.fn(async (desired: boolean) => Promise.resolve(desired)); + muteState.setHandler(handler); + expect(muteState.enabled$.value).toBe(false); + + available$.next(aCamera); + // On only once the handler has acted on it + expect(muteState.enabled$.value).toBe(false); + await flushPromises(); + + expect(handler).toHaveBeenLastCalledWith(true); + expect(muteState.enabled$.value).toBe(true); + expect(muteState.syncing$.value).toBe(false); + }); + + // A camera that is unplugged and plugged back in during a call. + // 1. Start with a camera, on by default, and set a handler. + // 2. The camera goes away, then comes back. + // The handler is told "off" when the camera goes away and is not called + // again. The camera stays off: we must not turn on a camera that the user + // may have turned off before. + // + // What it guards: a privacy problem the fix for #4403 could cause. The fix + // sends the default "on" to the handler. If it did that every time a + // camera shows up, replugging a camera (or leaving earpiece mode) would + // start sending video without the user asking. This test fails if the + // default is ever applied a second time during a call. + test("leaves a device that comes back under a handler off", async () => { + const available$ = new BehaviorSubject(aCamera); + const muteState = new MuteState( + testScope, + aDevice(available$), + true, + constant(false), + ); + expect(muteState.enabled$.value).toBe(true); + const handler = vi.fn(async (desired: boolean) => Promise.resolve(desired)); + muteState.setHandler(handler); + + available$.next(new Map()); + available$.next(aCamera); + await flushPromises(); + + expect(handler.mock.calls).toEqual([[false]]); + expect(muteState.enabled$.value).toBe(false); + }); + + // The lobby case, which must keep working as before: no handler is set. + // 1. Start with no camera, with the camera set to be on by default. + // 2. The camera shows up. + // The button shows "on" right away, with no waiting. + // + // What it guards: the lobby and the join read the button's value at once + // to decide which devices to open. If the fix for #4403 made the default + // wait for a handler here too, the value would be "off" for a moment and + // the lobby would open no camera. This test fails if that happens. + test("assumes the default at once where no handler is set", () => { + const available$ = new BehaviorSubject(new Map()); + const muteState = new MuteState( + testScope, + aDevice(available$), + true, + constant(false), + ); + + available$.next(aCamera); + + expect(muteState.enabled$.value).toBe(true); + }); + + // A device list with one camera in it. + const aCamera = new Map([ + ["fbac11", { type: "name", name: "HD Camera" }], + ]); + + // A fake camera whose device list the test can change while it runs. + function aDevice( + available$: BehaviorSubject>, + ): MediaDevice { + return { + available$, + selected$: constant({ id: "fbac11" }), + select(): void {}, + } as unknown as MediaDevice; + } }); describe("MuteStates", () => { diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index 59413b036..53740da6d 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -53,6 +53,8 @@ export class MuteState { this.handler$.next(defaultHandler); } + private devicesConnectedBefore = false; + private readonly canControlDevices$ = combineLatest([ this.device.available$, this.forceMute$, @@ -86,15 +88,26 @@ export class MuteState { }; } - // Assume the default value only once devices are actually connected - let enabled = this.enabledByDefault; + // defaultHander = no-op hanlder = "no actual device handler available" + const handlerAttached = this.handler$.value !== defaultHandler; + const deviceCameBackInCall = + handlerAttached && this.devicesConnectedBefore; + // Set the flag. A marker, that we reached this code before. + this.devicesConnectedBefore = true; + const initialWanted = deviceCameBackInCall + ? false + : this.enabledByDefault; + // With a handler attached we start off and let the sync below actually call + // the handler, so the switch shows "on" only once the track is really there. + // Without one (lobby) enabled = initialWanted -> no call to handler. + let enabled = handlerAttached ? false : initialWanted; const set$ = new Subject(); const toggle$ = new Subject(); const syncing$ = new BehaviorSubject(false); const desired$ = merge(set$, toggle$.pipe(map(() => !enabled))); const enabled$ = new Observable((subscriber) => { subscriber.next(enabled); - let latestDesired = this.enabledByDefault; + let latestDesired = initialWanted; const sync = async (): Promise => { if (enabled === latestDesired) { @@ -120,16 +133,19 @@ export class MuteState { } }; + const startSync = (): void => { + syncing$.next(true); + sync().catch((err) => { + // TODO: better error handling + logger.error("MuteState: handler error", err); + }); + }; + const s = desired$.subscribe((desired) => { latestDesired = desired; - if (syncing$.value === false) { - syncing$.next(true); - sync().catch((err) => { - // TODO: better error handling - logger.error("MuteState: handler error", err); - }); - } + if (syncing$.value === false) startSync(); }); + if (enabled !== latestDesired) startSync(); return (): void => { s.unsubscribe(); syncing$.complete();