mirror of
https://github.com/element-hq/element-call.git
synced 2026-11-22 12:17:37 +00:00
Merge pull request #4486 from element-hq/toger5/4479-exact-device-constraint
Exact device constraint (fixes chrome device switches)
This commit is contained in:
1 parent
7cf6cb4fee
commit
5d6490cbea
6 files changed
+270
-27
No files matched your search
@@ -0,0 +1,78 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { expect, test, type Page } from "@playwright/test";
|
||||
|
||||
import { SpaHelpers } from "./spa-helpers.ts";
|
||||
|
||||
test.use({
|
||||
launchOptions: {
|
||||
args: [
|
||||
"--use-fake-ui-for-media-stream",
|
||||
"--use-fake-device-for-media-stream=device-count=2",
|
||||
"--mute-audio",
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
test.describe("the camera menu", () => {
|
||||
test.skip(
|
||||
({ browserName }) => browserName !== "chromium",
|
||||
"Only Chromium offers a second fake camera",
|
||||
);
|
||||
|
||||
test("moves the preview to the camera picked, and the call after it", async ({
|
||||
page,
|
||||
}) => {
|
||||
await page.goto("/");
|
||||
await SpaHelpers.createCall(page, "Camera user", "Camera switch");
|
||||
await expectCamera(page, "fake_device_0");
|
||||
|
||||
await selectCamera(page, "fake_device_1");
|
||||
await expectCamera(page, "fake_device_1");
|
||||
|
||||
await page.getByTestId("lobby_joinCall").click();
|
||||
await expect(page.getByTestId("name_tag")).toContainText("Camera user");
|
||||
await expectCamera(page, "fake_device_1");
|
||||
|
||||
await selectCamera(page, "fake_device_0");
|
||||
await expectCamera(page, "fake_device_0");
|
||||
|
||||
// Turning the camera on again opens it anew
|
||||
const camera = page.getByTestId("incall_videomute");
|
||||
await camera.click();
|
||||
await expect(camera).toHaveAttribute("aria-checked", "false");
|
||||
await camera.click();
|
||||
await expect(camera).toHaveAttribute("aria-checked", "true");
|
||||
await expectCamera(page, "fake_device_0");
|
||||
});
|
||||
});
|
||||
|
||||
async function selectCamera(page: Page, name: string): Promise<void> {
|
||||
await page.getByRole("button", { name: "Camera", exact: true }).click();
|
||||
await page.getByRole("menuitemradio", { name }).click();
|
||||
// Selecting doesn't close the menu, so dismiss it.
|
||||
await page.keyboard.press("Escape");
|
||||
await expect(page.getByRole("menu")).not.toBeVisible();
|
||||
}
|
||||
|
||||
/** Every video on the page shows this camera, and there is one that does. */
|
||||
async function expectCamera(page: Page, label: string): Promise<void> {
|
||||
await expect
|
||||
.poll(async () =>
|
||||
page.evaluate(() => [
|
||||
...new Set(
|
||||
[...document.querySelectorAll("video")].flatMap((video) =>
|
||||
video.srcObject instanceof MediaStream
|
||||
? video.srcObject.getVideoTracks().map((track) => track.label)
|
||||
: [],
|
||||
),
|
||||
),
|
||||
]),
|
||||
)
|
||||
.toEqual([label]);
|
||||
}
|
||||
@@ -24,7 +24,7 @@ describe("createLocalMedia", () => {
|
||||
]);
|
||||
|
||||
expect(getUserMedia).toHaveBeenCalledExactlyOnceWith({
|
||||
audio: { deviceId: "mic" },
|
||||
audio: { deviceId: { exact: "mic" } },
|
||||
video: { width: 640, height: 480, frameRate: undefined },
|
||||
});
|
||||
expect(tracks.map((t) => t.source)).toEqual(["microphone", "camera"]);
|
||||
@@ -124,7 +124,7 @@ describe("createLocalMedia", () => {
|
||||
await camera.setDevice("other");
|
||||
|
||||
expect(getUserMedia).toHaveBeenLastCalledWith({
|
||||
video: { deviceId: "other" },
|
||||
video: { deviceId: { exact: "other" } },
|
||||
});
|
||||
expect(first.stop).toHaveBeenCalled();
|
||||
expect(processor.restart).toHaveBeenCalledOnce();
|
||||
@@ -133,7 +133,53 @@ describe("createLocalMedia", () => {
|
||||
expect(input).toBe(vi.mocked(processor.init).mock.calls[0][1]);
|
||||
expect(tracksOn(element)).toEqual([processor.output]);
|
||||
expect(media.takeTracks().get("camera")?.constraints).toEqual({
|
||||
deviceId: "other",
|
||||
deviceId: { exact: "other" },
|
||||
});
|
||||
});
|
||||
|
||||
it("opens the browser's choice where the device asked for is gone", async () => {
|
||||
const getUserMedia = vi.mocked(fakeGetUserMedia());
|
||||
getUserMedia.mockRejectedValueOnce(overconstrained());
|
||||
const media = createLocalMedia(getUserMedia);
|
||||
|
||||
const tracks = await media.capture([
|
||||
{ source: "microphone", deviceId: "unplugged" },
|
||||
{ source: "camera", deviceId: "cam" },
|
||||
]);
|
||||
|
||||
expect(getUserMedia).toHaveBeenLastCalledWith({ audio: {}, video: {} });
|
||||
expect(tracks.map((t) => t.source)).toEqual(["microphone", "camera"]);
|
||||
});
|
||||
|
||||
it("keeps the camera it has where another cannot be opened", async () => {
|
||||
const getUserMedia = vi.mocked(fakeGetUserMedia());
|
||||
const media = createLocalMedia(getUserMedia);
|
||||
const [camera] = await media.capture([
|
||||
{ source: "camera", deviceId: "cam" },
|
||||
]);
|
||||
const first = trackBehind(camera);
|
||||
getUserMedia.mockRejectedValueOnce(overconstrained());
|
||||
|
||||
await expect(camera.setDevice("unplugged")).rejects.toThrow();
|
||||
|
||||
expect(first.stop).not.toHaveBeenCalled();
|
||||
expect(media.takeTracks().get("camera")).toMatchObject({
|
||||
track: first,
|
||||
constraints: { deviceId: { exact: "cam" } },
|
||||
});
|
||||
});
|
||||
|
||||
it("lets go of the device it was opened on to face the other way", async () => {
|
||||
const getUserMedia = fakeGetUserMedia({ facingMode: "user" });
|
||||
const media = createLocalMedia(getUserMedia);
|
||||
const [camera] = (await media.capture([
|
||||
{ source: "camera", deviceId: "front" },
|
||||
])) as [LocalVideoMediaTrack];
|
||||
|
||||
await camera.switchFacingMode();
|
||||
|
||||
expect(getUserMedia).toHaveBeenLastCalledWith({
|
||||
video: { facingMode: "environment" },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -227,6 +273,10 @@ function fakeGetUserMedia(videoSettings?: MediaTrackSettings): GetUserMedia {
|
||||
});
|
||||
}
|
||||
|
||||
function overconstrained(): Error {
|
||||
return Object.assign(new Error("deviceId"), { name: "OverconstrainedError" });
|
||||
}
|
||||
|
||||
function fakeProcessor(): VideoProcessor & { output: MediaStreamTrack } {
|
||||
const output = fakeTrack("video");
|
||||
return {
|
||||
|
||||
+83
-14
@@ -78,11 +78,14 @@ export function createLocalMedia(
|
||||
);
|
||||
bySource.set(request.source, request);
|
||||
}
|
||||
const constraints: MediaStreamConstraints = {};
|
||||
for (const request of bySource.values())
|
||||
constraints[request.source === "microphone" ? "audio" : "video"] =
|
||||
trackConstraints(request);
|
||||
const stream = await getUserMedia(constraints);
|
||||
const { stream, constraints } = await open(getUserMedia, {
|
||||
microphone: bySource.has("microphone")
|
||||
? trackConstraints(bySource.get("microphone")!)
|
||||
: undefined,
|
||||
camera: bySource.has("camera")
|
||||
? trackConstraints(bySource.get("camera")!)
|
||||
: undefined,
|
||||
});
|
||||
const captured: CapturedTrack[] = [];
|
||||
for (const track of stream.getTracks()) {
|
||||
const source: Source = track.kind === "audio" ? "microphone" : "camera";
|
||||
@@ -91,7 +94,7 @@ export function createLocalMedia(
|
||||
const captured_ = new CapturedTrack(
|
||||
source,
|
||||
track,
|
||||
trackConstraints(request),
|
||||
constraints[source]!,
|
||||
getUserMedia,
|
||||
);
|
||||
capturedTracks.set(source, captured_);
|
||||
@@ -139,9 +142,50 @@ export function createLocalMedia(
|
||||
};
|
||||
}
|
||||
|
||||
type SourceConstraints = Partial<Record<Source, MediaTrackConstraints>>;
|
||||
|
||||
/**
|
||||
* Opens the devices asked for. Where one of them is gone, as a remembered
|
||||
* choice that was unplugged, the browser picks instead.
|
||||
*
|
||||
* This is needed because we use `exact` device ids: an exact request for a
|
||||
* device that no longer exists fails, where a preferred one would have opened
|
||||
* another device. Without the retry here a stale choice would open nothing.
|
||||
*/
|
||||
async function open(
|
||||
getUserMedia: GetUserMedia,
|
||||
constraints: SourceConstraints,
|
||||
): Promise<{ stream: MediaStream; constraints: SourceConstraints }> {
|
||||
const streamOf = async (wanted: SourceConstraints): Promise<MediaStream> =>
|
||||
getUserMedia({
|
||||
...(wanted.microphone && { audio: wanted.microphone }),
|
||||
...(wanted.camera && { video: wanted.camera }),
|
||||
});
|
||||
try {
|
||||
return { stream: await streamOf(constraints), constraints };
|
||||
} catch (e) {
|
||||
if (!(e instanceof Error) || e.name !== "OverconstrainedError") throw e;
|
||||
const anyDevice: SourceConstraints = {
|
||||
...(constraints.microphone && {
|
||||
microphone: { ...constraints.microphone, deviceId: undefined },
|
||||
}),
|
||||
...(constraints.camera && {
|
||||
camera: { ...constraints.camera, deviceId: undefined },
|
||||
}),
|
||||
};
|
||||
return { stream: await streamOf(anyDevice), constraints: anyDevice };
|
||||
}
|
||||
}
|
||||
|
||||
/** The constraints a request means to the browser. */
|
||||
function trackConstraints(request: Request): MediaTrackConstraints {
|
||||
const device = request.deviceId && { deviceId: request.deviceId };
|
||||
// THE FIX for "cannot switch cameras in Chrome" (#4479), first capture.
|
||||
// `exact` makes the browser open this device or fail. A plain
|
||||
// `deviceId: "abc"` is only a wish, and Chromium ignores the wish and
|
||||
// opens its default device.
|
||||
const device = request.deviceId && {
|
||||
deviceId: { exact: request.deviceId },
|
||||
};
|
||||
if (request.source === "microphone") return { ...device, ...request.capture };
|
||||
const resolution = request.capture?.resolution;
|
||||
return {
|
||||
@@ -184,7 +228,12 @@ class CapturedTrack {
|
||||
// The device actually in use, so that a backend asked for it again
|
||||
// recognises it rather than restarting the track
|
||||
const { deviceId } = track.getSettings();
|
||||
const constraints = { ...this.constraints, ...(deviceId && { deviceId }) };
|
||||
// `exact` here too: the call reopens the camera with these constraints
|
||||
// after a mute, and must get the same device again, not the default one
|
||||
const constraints = {
|
||||
...this.constraints,
|
||||
...(deviceId && { deviceId: { exact: deviceId } }),
|
||||
};
|
||||
this.finish();
|
||||
return { track, constraints, muted: this.muted$.value, processor };
|
||||
}
|
||||
@@ -245,12 +294,28 @@ class CapturedTrack {
|
||||
for (const element of this.elements) play(element, this.shown());
|
||||
}
|
||||
|
||||
private async reacquire(constraints: MediaTrackConstraints): Promise<void> {
|
||||
/**
|
||||
* Replace the current with the one from the selector. On fail, keep current.
|
||||
*
|
||||
* The old `deviceId` and `facingMode` are removed before the new selector
|
||||
* is applied. This is needed because we use `exact` device ids: if the old
|
||||
* exact id stayed, asking for a new facing mode could not leave the deviceId.
|
||||
* "flip camera" would do nothing.
|
||||
*/
|
||||
private async reacquire(
|
||||
selector: Pick<MediaTrackConstraints, "deviceId" | "facingMode">,
|
||||
): Promise<void> {
|
||||
this.live();
|
||||
this.constraints = { ...this.constraints, ...constraints };
|
||||
const stream = await this.getUserMedia({
|
||||
[this.track.kind]: this.constraints,
|
||||
});
|
||||
const constraints = {
|
||||
...this.constraints,
|
||||
// One selects the device at a time: a facing mode cannot move a track
|
||||
// off a device it is pinned to
|
||||
deviceId: undefined,
|
||||
facingMode: undefined,
|
||||
...selector,
|
||||
};
|
||||
const stream = await this.getUserMedia({ [this.track.kind]: constraints });
|
||||
this.constraints = constraints;
|
||||
this.track.stop();
|
||||
this.track = stream.getTracks()[0];
|
||||
this.track.enabled = !this.muted$.value;
|
||||
@@ -288,7 +353,11 @@ class CapturedTrack {
|
||||
},
|
||||
setDevice: async (deviceId: string): Promise<void> => {
|
||||
if (this.track.getSettings().deviceId === deviceId) return;
|
||||
await this.reacquire({ deviceId });
|
||||
// THE FIX for "cannot switch cameras in Chrome" (#4479), switching in
|
||||
// the lobby. With a plain `{ deviceId }` Chromium reopened the same
|
||||
// camera, so the preview only flashed. `exact` opens the one picked,
|
||||
// or rejects and leaves the current camera running.
|
||||
await this.reacquire({ deviceId: { exact: deviceId } });
|
||||
},
|
||||
};
|
||||
if (this.source === "microphone")
|
||||
|
||||
@@ -143,7 +143,10 @@ export interface LocalMediaTrack {
|
||||
* from the request where the device could not be used.
|
||||
*/
|
||||
setEnabled(enabled: boolean): Promise<boolean>;
|
||||
/** Captures from another device. Rejects for a screen share, which has none. */
|
||||
/**
|
||||
* Captures from another device. Rejects where that device cannot be opened,
|
||||
* leaving the track on the one it had, and for a screen share, which has none.
|
||||
*/
|
||||
setDevice(deviceId: string): Promise<void>;
|
||||
}
|
||||
|
||||
|
||||
@@ -210,11 +210,7 @@ describe("createLivekitMediaTrack", () => {
|
||||
await expect(track.setEnabled(false)).resolves.toBe(false);
|
||||
expect(setEnabled).toHaveBeenCalledWith("microphone", false);
|
||||
await track.setDevice("mic-2");
|
||||
expect(switchActiveDevice).toHaveBeenCalledWith(
|
||||
"audioinput",
|
||||
"mic-2",
|
||||
false,
|
||||
);
|
||||
expect(switchActiveDevice).toHaveBeenCalledWith("audioinput", "mic-2");
|
||||
});
|
||||
|
||||
// LiveKit's media stream track is the processed one while a processor runs,
|
||||
@@ -225,19 +221,23 @@ describe("createLivekitMediaTrack", () => {
|
||||
])(
|
||||
"leaves a processed camera alone unless its device changes: $deviceId",
|
||||
async ({ deviceId, switches }) => {
|
||||
let openDevice = "cam-1";
|
||||
const camera = Object.assign(Object.create(LocalVideoTrack.prototype), {
|
||||
source: Track.Source.Camera,
|
||||
_mediaStreamTrack: {
|
||||
readyState: "live",
|
||||
label: "",
|
||||
getSettings: () => ({ deviceId: "cam-1" }),
|
||||
getSettings: () => ({ deviceId: openDevice }),
|
||||
},
|
||||
// What LiveKit shows as the track, with no device of its own
|
||||
processor: {
|
||||
processedTrack: { label: "", getSettings: () => ({}) },
|
||||
},
|
||||
}) as LocalVideoTrack;
|
||||
const switchActiveDevice = vi.fn().mockResolvedValue(true);
|
||||
const switchActiveDevice = vi.fn(async (_kind: string, id: string) => {
|
||||
openDevice = id;
|
||||
return Promise.resolve(true);
|
||||
});
|
||||
const { participant, publication, room } = fakes({
|
||||
track: camera as unknown as FakeTrack,
|
||||
});
|
||||
@@ -253,11 +253,38 @@ describe("createLivekitMediaTrack", () => {
|
||||
|
||||
await track.setDevice(deviceId);
|
||||
expect(switchActiveDevice.mock.calls).toEqual(
|
||||
switches ? [["videoinput", deviceId, false]] : [],
|
||||
switches ? [["videoinput", deviceId]] : [],
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects where the camera did not move to the device asked for", async () => {
|
||||
const camera = Object.assign(Object.create(LocalVideoTrack.prototype), {
|
||||
source: Track.Source.Camera,
|
||||
_mediaStreamTrack: {
|
||||
readyState: "live",
|
||||
label: "",
|
||||
getSettings: () => ({ deviceId: "cam-1" }),
|
||||
},
|
||||
}) as LocalVideoTrack;
|
||||
const { participant, publication, room } = fakes({
|
||||
track: camera as unknown as FakeTrack,
|
||||
});
|
||||
const track = createLocalLivekitMediaTrack(
|
||||
scope,
|
||||
Object.assign(participant, {
|
||||
isLocal: true,
|
||||
}) as unknown as LocalParticipant,
|
||||
publication as unknown as TrackPublication,
|
||||
Object.assign(room, {
|
||||
switchActiveDevice: vi.fn().mockResolvedValue(true),
|
||||
}),
|
||||
vi.fn(),
|
||||
);
|
||||
|
||||
await expect(track.setDevice("cam-2")).rejects.toThrow("another device");
|
||||
});
|
||||
|
||||
it("takes the facing mode of a processed camera from the camera", async () => {
|
||||
const source = {
|
||||
readyState: "live",
|
||||
|
||||
@@ -208,11 +208,27 @@ export function createLocalLivekitMediaTrack(
|
||||
(await current.getDeviceId(false)) === deviceId
|
||||
)
|
||||
return;
|
||||
// THE FIX for "cannot switch cameras in Chrome" (#4479), switching in
|
||||
// the call. No third argument: LiveKit then asks for the device with
|
||||
// `exact`. Passing `false` there made it a wish, which Chromium ignores
|
||||
// by reopening its default device.
|
||||
await room.switchActiveDevice(
|
||||
base.kind === "audio" ? "audioinput" : "videoinput",
|
||||
deviceId,
|
||||
false,
|
||||
);
|
||||
// A check that the switch really happened, so that a failure is an
|
||||
// error and not silence. Asked of the track again rather than taken
|
||||
// from LiveKit's answer, which compares against the processed track
|
||||
// too. A muted track is stopped and takes the device on turning on again
|
||||
const now = publication.track;
|
||||
const deviceNow =
|
||||
now instanceof LocalTrack && !now.isMuted
|
||||
? await now.getDeviceId(false)
|
||||
: undefined;
|
||||
if (deviceNow !== undefined && deviceNow !== deviceId)
|
||||
throw new MatrixRTCError(
|
||||
`The ${base.source} is on another device than the one asked for`,
|
||||
);
|
||||
},
|
||||
};
|
||||
if (base.kind === "audio") return { ...base, ...local };
|
||||
|
||||
Reference in new issue
Block a user