From 5d6490cbea44760985edcfdd90fe51355a398b72 Mon Sep 17 00:00:00 2001 From: Timo <16718859+toger5@users.noreply.github.com> Date: Fri, 20 Nov 2026 17:57:43 +0100 Subject: [PATCH] Merge pull request #4486 from element-hq/toger5/4479-exact-device-constraint Exact device constraint (fixes chrome device switches) --- playwright/camera-switch.spec.ts | 78 +++++++++++++++ sdk/src/LocalMedia.test.ts | 56 ++++++++++- sdk/src/LocalMedia.ts | 97 ++++++++++++++++--- sdk/src/media-api.ts | 5 +- .../livekit/LivekitMediaTrack.test.ts | 43 ++++++-- .../livekit/LivekitMediaTrack.ts | 18 +++- 6 files changed, 270 insertions(+), 27 deletions(-) create mode 100644 playwright/camera-switch.spec.ts diff --git a/playwright/camera-switch.spec.ts b/playwright/camera-switch.spec.ts new file mode 100644 index 000000000..d320bae75 --- /dev/null +++ b/playwright/camera-switch.spec.ts @@ -0,0 +1,78 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test, type Page } from "@playwright/test"; + +import { SpaHelpers } from "./spa-helpers.ts"; + +test.use({ + launchOptions: { + args: [ + "--use-fake-ui-for-media-stream", + "--use-fake-device-for-media-stream=device-count=2", + "--mute-audio", + ], + }, +}); + +test.describe("the camera menu", () => { + test.skip( + ({ browserName }) => browserName !== "chromium", + "Only Chromium offers a second fake camera", + ); + + test("moves the preview to the camera picked, and the call after it", async ({ + page, + }) => { + await page.goto("/"); + await SpaHelpers.createCall(page, "Camera user", "Camera switch"); + await expectCamera(page, "fake_device_0"); + + await selectCamera(page, "fake_device_1"); + await expectCamera(page, "fake_device_1"); + + await page.getByTestId("lobby_joinCall").click(); + await expect(page.getByTestId("name_tag")).toContainText("Camera user"); + await expectCamera(page, "fake_device_1"); + + await selectCamera(page, "fake_device_0"); + await expectCamera(page, "fake_device_0"); + + // Turning the camera on again opens it anew + const camera = page.getByTestId("incall_videomute"); + await camera.click(); + await expect(camera).toHaveAttribute("aria-checked", "false"); + await camera.click(); + await expect(camera).toHaveAttribute("aria-checked", "true"); + await expectCamera(page, "fake_device_0"); + }); +}); + +async function selectCamera(page: Page, name: string): Promise { + await page.getByRole("button", { name: "Camera", exact: true }).click(); + await page.getByRole("menuitemradio", { name }).click(); + // Selecting doesn't close the menu, so dismiss it. + await page.keyboard.press("Escape"); + await expect(page.getByRole("menu")).not.toBeVisible(); +} + +/** Every video on the page shows this camera, and there is one that does. */ +async function expectCamera(page: Page, label: string): Promise { + await expect + .poll(async () => + page.evaluate(() => [ + ...new Set( + [...document.querySelectorAll("video")].flatMap((video) => + video.srcObject instanceof MediaStream + ? video.srcObject.getVideoTracks().map((track) => track.label) + : [], + ), + ), + ]), + ) + .toEqual([label]); +} diff --git a/sdk/src/LocalMedia.test.ts b/sdk/src/LocalMedia.test.ts index 8feed1bbb..37fe4015e 100644 --- a/sdk/src/LocalMedia.test.ts +++ b/sdk/src/LocalMedia.test.ts @@ -24,7 +24,7 @@ describe("createLocalMedia", () => { ]); expect(getUserMedia).toHaveBeenCalledExactlyOnceWith({ - audio: { deviceId: "mic" }, + audio: { deviceId: { exact: "mic" } }, video: { width: 640, height: 480, frameRate: undefined }, }); expect(tracks.map((t) => t.source)).toEqual(["microphone", "camera"]); @@ -124,7 +124,7 @@ describe("createLocalMedia", () => { await camera.setDevice("other"); expect(getUserMedia).toHaveBeenLastCalledWith({ - video: { deviceId: "other" }, + video: { deviceId: { exact: "other" } }, }); expect(first.stop).toHaveBeenCalled(); expect(processor.restart).toHaveBeenCalledOnce(); @@ -133,7 +133,53 @@ describe("createLocalMedia", () => { expect(input).toBe(vi.mocked(processor.init).mock.calls[0][1]); expect(tracksOn(element)).toEqual([processor.output]); expect(media.takeTracks().get("camera")?.constraints).toEqual({ - deviceId: "other", + deviceId: { exact: "other" }, + }); + }); + + it("opens the browser's choice where the device asked for is gone", async () => { + const getUserMedia = vi.mocked(fakeGetUserMedia()); + getUserMedia.mockRejectedValueOnce(overconstrained()); + const media = createLocalMedia(getUserMedia); + + const tracks = await media.capture([ + { source: "microphone", deviceId: "unplugged" }, + { source: "camera", deviceId: "cam" }, + ]); + + expect(getUserMedia).toHaveBeenLastCalledWith({ audio: {}, video: {} }); + expect(tracks.map((t) => t.source)).toEqual(["microphone", "camera"]); + }); + + it("keeps the camera it has where another cannot be opened", async () => { + const getUserMedia = vi.mocked(fakeGetUserMedia()); + const media = createLocalMedia(getUserMedia); + const [camera] = await media.capture([ + { source: "camera", deviceId: "cam" }, + ]); + const first = trackBehind(camera); + getUserMedia.mockRejectedValueOnce(overconstrained()); + + await expect(camera.setDevice("unplugged")).rejects.toThrow(); + + expect(first.stop).not.toHaveBeenCalled(); + expect(media.takeTracks().get("camera")).toMatchObject({ + track: first, + constraints: { deviceId: { exact: "cam" } }, + }); + }); + + it("lets go of the device it was opened on to face the other way", async () => { + const getUserMedia = fakeGetUserMedia({ facingMode: "user" }); + const media = createLocalMedia(getUserMedia); + const [camera] = (await media.capture([ + { source: "camera", deviceId: "front" }, + ])) as [LocalVideoMediaTrack]; + + await camera.switchFacingMode(); + + expect(getUserMedia).toHaveBeenLastCalledWith({ + video: { facingMode: "environment" }, }); }); @@ -227,6 +273,10 @@ function fakeGetUserMedia(videoSettings?: MediaTrackSettings): GetUserMedia { }); } +function overconstrained(): Error { + return Object.assign(new Error("deviceId"), { name: "OverconstrainedError" }); +} + function fakeProcessor(): VideoProcessor & { output: MediaStreamTrack } { const output = fakeTrack("video"); return { diff --git a/sdk/src/LocalMedia.ts b/sdk/src/LocalMedia.ts index e767b05ec..2b7bbe96f 100644 --- a/sdk/src/LocalMedia.ts +++ b/sdk/src/LocalMedia.ts @@ -78,11 +78,14 @@ export function createLocalMedia( ); bySource.set(request.source, request); } - const constraints: MediaStreamConstraints = {}; - for (const request of bySource.values()) - constraints[request.source === "microphone" ? "audio" : "video"] = - trackConstraints(request); - const stream = await getUserMedia(constraints); + const { stream, constraints } = await open(getUserMedia, { + microphone: bySource.has("microphone") + ? trackConstraints(bySource.get("microphone")!) + : undefined, + camera: bySource.has("camera") + ? trackConstraints(bySource.get("camera")!) + : undefined, + }); const captured: CapturedTrack[] = []; for (const track of stream.getTracks()) { const source: Source = track.kind === "audio" ? "microphone" : "camera"; @@ -91,7 +94,7 @@ export function createLocalMedia( const captured_ = new CapturedTrack( source, track, - trackConstraints(request), + constraints[source]!, getUserMedia, ); capturedTracks.set(source, captured_); @@ -139,9 +142,50 @@ export function createLocalMedia( }; } +type SourceConstraints = Partial>; + +/** + * Opens the devices asked for. Where one of them is gone, as a remembered + * choice that was unplugged, the browser picks instead. + * + * This is needed because we use `exact` device ids: an exact request for a + * device that no longer exists fails, where a preferred one would have opened + * another device. Without the retry here a stale choice would open nothing. + */ +async function open( + getUserMedia: GetUserMedia, + constraints: SourceConstraints, +): Promise<{ stream: MediaStream; constraints: SourceConstraints }> { + const streamOf = async (wanted: SourceConstraints): Promise => + getUserMedia({ + ...(wanted.microphone && { audio: wanted.microphone }), + ...(wanted.camera && { video: wanted.camera }), + }); + try { + return { stream: await streamOf(constraints), constraints }; + } catch (e) { + if (!(e instanceof Error) || e.name !== "OverconstrainedError") throw e; + const anyDevice: SourceConstraints = { + ...(constraints.microphone && { + microphone: { ...constraints.microphone, deviceId: undefined }, + }), + ...(constraints.camera && { + camera: { ...constraints.camera, deviceId: undefined }, + }), + }; + return { stream: await streamOf(anyDevice), constraints: anyDevice }; + } +} + /** The constraints a request means to the browser. */ function trackConstraints(request: Request): MediaTrackConstraints { - const device = request.deviceId && { deviceId: request.deviceId }; + // THE FIX for "cannot switch cameras in Chrome" (#4479), first capture. + // `exact` makes the browser open this device or fail. A plain + // `deviceId: "abc"` is only a wish, and Chromium ignores the wish and + // opens its default device. + const device = request.deviceId && { + deviceId: { exact: request.deviceId }, + }; if (request.source === "microphone") return { ...device, ...request.capture }; const resolution = request.capture?.resolution; return { @@ -184,7 +228,12 @@ class CapturedTrack { // The device actually in use, so that a backend asked for it again // recognises it rather than restarting the track const { deviceId } = track.getSettings(); - const constraints = { ...this.constraints, ...(deviceId && { deviceId }) }; + // `exact` here too: the call reopens the camera with these constraints + // after a mute, and must get the same device again, not the default one + const constraints = { + ...this.constraints, + ...(deviceId && { deviceId: { exact: deviceId } }), + }; this.finish(); return { track, constraints, muted: this.muted$.value, processor }; } @@ -245,12 +294,28 @@ class CapturedTrack { for (const element of this.elements) play(element, this.shown()); } - private async reacquire(constraints: MediaTrackConstraints): Promise { + /** + * Replace the current with the one from the selector. On fail, keep current. + * + * The old `deviceId` and `facingMode` are removed before the new selector + * is applied. This is needed because we use `exact` device ids: if the old + * exact id stayed, asking for a new facing mode could not leave the deviceId. + * "flip camera" would do nothing. + */ + private async reacquire( + selector: Pick, + ): Promise { this.live(); - this.constraints = { ...this.constraints, ...constraints }; - const stream = await this.getUserMedia({ - [this.track.kind]: this.constraints, - }); + const constraints = { + ...this.constraints, + // One selects the device at a time: a facing mode cannot move a track + // off a device it is pinned to + deviceId: undefined, + facingMode: undefined, + ...selector, + }; + const stream = await this.getUserMedia({ [this.track.kind]: constraints }); + this.constraints = constraints; this.track.stop(); this.track = stream.getTracks()[0]; this.track.enabled = !this.muted$.value; @@ -288,7 +353,11 @@ class CapturedTrack { }, setDevice: async (deviceId: string): Promise => { if (this.track.getSettings().deviceId === deviceId) return; - await this.reacquire({ deviceId }); + // THE FIX for "cannot switch cameras in Chrome" (#4479), switching in + // the lobby. With a plain `{ deviceId }` Chromium reopened the same + // camera, so the preview only flashed. `exact` opens the one picked, + // or rejects and leaves the current camera running. + await this.reacquire({ deviceId: { exact: deviceId } }); }, }; if (this.source === "microphone") diff --git a/sdk/src/media-api.ts b/sdk/src/media-api.ts index 58adac731..571e7f9ec 100644 --- a/sdk/src/media-api.ts +++ b/sdk/src/media-api.ts @@ -143,7 +143,10 @@ export interface LocalMediaTrack { * from the request where the device could not be used. */ setEnabled(enabled: boolean): Promise; - /** Captures from another device. Rejects for a screen share, which has none. */ + /** + * Captures from another device. Rejects where that device cannot be opened, + * leaving the track on the one it had, and for a screen share, which has none. + */ setDevice(deviceId: string): Promise; } diff --git a/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts b/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts index 2927b5baf..b872e7d11 100644 --- a/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts +++ b/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts @@ -210,11 +210,7 @@ describe("createLivekitMediaTrack", () => { await expect(track.setEnabled(false)).resolves.toBe(false); expect(setEnabled).toHaveBeenCalledWith("microphone", false); await track.setDevice("mic-2"); - expect(switchActiveDevice).toHaveBeenCalledWith( - "audioinput", - "mic-2", - false, - ); + expect(switchActiveDevice).toHaveBeenCalledWith("audioinput", "mic-2"); }); // LiveKit's media stream track is the processed one while a processor runs, @@ -225,19 +221,23 @@ describe("createLivekitMediaTrack", () => { ])( "leaves a processed camera alone unless its device changes: $deviceId", async ({ deviceId, switches }) => { + let openDevice = "cam-1"; const camera = Object.assign(Object.create(LocalVideoTrack.prototype), { source: Track.Source.Camera, _mediaStreamTrack: { readyState: "live", label: "", - getSettings: () => ({ deviceId: "cam-1" }), + getSettings: () => ({ deviceId: openDevice }), }, // What LiveKit shows as the track, with no device of its own processor: { processedTrack: { label: "", getSettings: () => ({}) }, }, }) as LocalVideoTrack; - const switchActiveDevice = vi.fn().mockResolvedValue(true); + const switchActiveDevice = vi.fn(async (_kind: string, id: string) => { + openDevice = id; + return Promise.resolve(true); + }); const { participant, publication, room } = fakes({ track: camera as unknown as FakeTrack, }); @@ -253,11 +253,38 @@ describe("createLivekitMediaTrack", () => { await track.setDevice(deviceId); expect(switchActiveDevice.mock.calls).toEqual( - switches ? [["videoinput", deviceId, false]] : [], + switches ? [["videoinput", deviceId]] : [], ); }, ); + it("rejects where the camera did not move to the device asked for", async () => { + const camera = Object.assign(Object.create(LocalVideoTrack.prototype), { + source: Track.Source.Camera, + _mediaStreamTrack: { + readyState: "live", + label: "", + getSettings: () => ({ deviceId: "cam-1" }), + }, + }) as LocalVideoTrack; + const { participant, publication, room } = fakes({ + track: camera as unknown as FakeTrack, + }); + const track = createLocalLivekitMediaTrack( + scope, + Object.assign(participant, { + isLocal: true, + }) as unknown as LocalParticipant, + publication as unknown as TrackPublication, + Object.assign(room, { + switchActiveDevice: vi.fn().mockResolvedValue(true), + }), + vi.fn(), + ); + + await expect(track.setDevice("cam-2")).rejects.toThrow("another device"); + }); + it("takes the facing mode of a processed camera from the camera", async () => { const source = { readyState: "live", diff --git a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts index 4a42ff519..a639af1fc 100644 --- a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts +++ b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts @@ -208,11 +208,27 @@ export function createLocalLivekitMediaTrack( (await current.getDeviceId(false)) === deviceId ) return; + // THE FIX for "cannot switch cameras in Chrome" (#4479), switching in + // the call. No third argument: LiveKit then asks for the device with + // `exact`. Passing `false` there made it a wish, which Chromium ignores + // by reopening its default device. await room.switchActiveDevice( base.kind === "audio" ? "audioinput" : "videoinput", deviceId, - false, ); + // A check that the switch really happened, so that a failure is an + // error and not silence. Asked of the track again rather than taken + // from LiveKit's answer, which compares against the processed track + // too. A muted track is stopped and takes the device on turning on again + const now = publication.track; + const deviceNow = + now instanceof LocalTrack && !now.isMuted + ? await now.getDeviceId(false) + : undefined; + if (deviceNow !== undefined && deviceNow !== deviceId) + throw new MatrixRTCError( + `The ${base.source} is on another device than the one asked for`, + ); }, }; if (base.kind === "audio") return { ...base, ...local };