From 08abf1a387d8a75831d2785108d73554035e47ca Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Thu, 1 Oct 2026 20:46:35 +0200 Subject: [PATCH] sdk implementation --- playwright/sdk/harness.ts | 133 +++++++-- playwright/sdk/lifecycle.spec.ts | 67 +++++ playwright/sdk/media.spec.ts | 158 +++++++++++ playwright/sdk/smoke.spec.ts | 36 +-- sdk/README.md | 10 +- sdk/dev/index.html | 17 +- sdk/dev/main.ts | 117 ++++++-- sdk/dev/session.ts | 70 ++++- sdk/index.ts | 258 +---------------- sdk/src/api.ts | 241 ++++++++++++++++ sdk/src/media/LivekitMediaTrack.test.ts | 160 +++++++++++ sdk/src/media/LivekitMediaTrack.ts | 200 +++++++++++++ sdk/src/media/LivekitMemberMedia.ts | 148 ++++++++++ sdk/src/session/ConnectionFactory.ts | 133 +++++++++ sdk/src/session/KeyProvider.ts | 38 +++ sdk/src/session/LocalMember.ts | 359 ++++++++++++++++++++++++ sdk/src/session/LocalTransport.ts | 66 +++++ sdk/src/session/Members.ts | 162 +++++++++++ sdk/src/session/Publisher.ts | 226 +++++++++++++++ sdk/src/session/RtcSession.ts | 235 ++++++++++++++++ sdk/src/session/Transports.ts | 93 ++++++ sdk/src/session/joinRtcSession.ts | 68 +++++ sdk/src/session/status.test.ts | 77 +++++ sdk/src/session/status.ts | 37 +++ sdk/src/utils/LazyBehavior.test.ts | 52 ++++ sdk/src/utils/LazyBehavior.ts | 53 ++++ sdk/src/utils/errors.test.ts | 35 +++ sdk/src/utils/errors.ts | 27 ++ sdk/src/utils/mapScoped.test.ts | 37 +++ sdk/src/utils/mapScoped.ts | 35 +++ vite.config.ts | 15 +- 31 files changed, 3015 insertions(+), 348 deletions(-) create mode 100644 playwright/sdk/lifecycle.spec.ts create mode 100644 playwright/sdk/media.spec.ts create mode 100644 sdk/src/api.ts create mode 100644 sdk/src/media/LivekitMediaTrack.test.ts create mode 100644 sdk/src/media/LivekitMediaTrack.ts create mode 100644 sdk/src/media/LivekitMemberMedia.ts create mode 100644 sdk/src/session/ConnectionFactory.ts create mode 100644 sdk/src/session/KeyProvider.ts create mode 100644 sdk/src/session/LocalMember.ts create mode 100644 sdk/src/session/LocalTransport.ts create mode 100644 sdk/src/session/Members.ts create mode 100644 sdk/src/session/Publisher.ts create mode 100644 sdk/src/session/RtcSession.ts create mode 100644 sdk/src/session/Transports.ts create mode 100644 sdk/src/session/joinRtcSession.ts create mode 100644 sdk/src/session/status.test.ts create mode 100644 sdk/src/session/status.ts create mode 100644 sdk/src/utils/LazyBehavior.test.ts create mode 100644 sdk/src/utils/LazyBehavior.ts create mode 100644 sdk/src/utils/errors.test.ts create mode 100644 sdk/src/utils/errors.ts create mode 100644 sdk/src/utils/mapScoped.test.ts create mode 100644 sdk/src/utils/mapScoped.ts diff --git a/playwright/sdk/harness.ts b/playwright/sdk/harness.ts index 999500eb2..9b7db37f0 100644 --- a/playwright/sdk/harness.ts +++ b/playwright/sdk/harness.ts @@ -5,7 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { expect, type Page } from "@playwright/test"; +import { + type Browser, + expect, + type Locator, + type Page, +} from "@playwright/test"; import { SynapseAdmin } from "../utils/synapse-admin.ts"; @@ -18,35 +23,74 @@ export const SDK_HARNESS_URL = "https://localhost:3002"; const HOMESERVER_URL = "https://synapse.m.localhost"; const PASSWORD = "foobarbaz1!"; +/** A registered user, with the session the registration opened. */ +export interface User { + username: string; + displayName: string; + userId: string; + deviceId: string; + accessToken: string; +} + /** * Registers two users through the Synapse admin API and has the first create a - * public room, so that the second can join it by id without an invite. + * public room, so that the second can join it by id without an invite. The + * registration logs each user in, which spares the harness a `/login` call + * that the homeserver rate-limits. */ export async function createUsersAndRoom( name: string, -): Promise<{ usernames: [string, string]; roomId: string }> { +): Promise<{ users: [User, User]; roomId: string }> { const admin = SynapseAdmin.forHomeserver(HOMESERVER_URL); - const usernames: [string, string] = [ - `${name}_a_${Date.now()}`, - `${name}_b_${Date.now()}`, - ]; - const [{ access_token: accessToken }] = await Promise.all( - usernames.map(async (username, index) => - admin.registerUser(username, PASSWORD, `${name} ${"AB"[index]}`), - ), - ); + const users = (await Promise.all( + ["a", "b"].map(async (letter, index): Promise => { + const username = `${name}_${letter}_${Date.now()}`; + const displayName = `${name} ${"AB"[index]}`; + const registration = await admin.registerUser( + username, + PASSWORD, + displayName, + ); + return { + username, + displayName, + userId: registration.user_id, + deviceId: registration.device_id, + accessToken: registration.access_token, + }; + }), + )) as [User, User]; const response = await fetch( `${HOMESERVER_URL}/_matrix/client/v3/createRoom`, { method: "POST", headers: { - Authorization: `Bearer ${accessToken}`, + Authorization: `Bearer ${users[0].accessToken}`, "Content-Type": "application/json", }, body: JSON.stringify({ name: `${name}'s session`, preset: "public_chat", + // Per-participant media keys travel in encrypted to-device messages, + // which only reach devices the sender's crypto tracks, and it tracks + // the members of encrypted rooms + initial_state: [ + { + type: "m.room.encryption", + state_key: "", + content: { algorithm: "m.megolm.v1.aes-sha2" }, + }, + ], + // Every member has to be allowed to write its own membership, which + // a public room does not grant by default; the same levels Element + // Call gives a room it creates for a call + power_level_content_override: { + state_default: 0, + events_default: 0, + users_default: 0, + events: { "org.matrix.msc3401.call.member": 0 }, + }, }), }, ); @@ -56,33 +100,68 @@ export async function createUsersAndRoom( ); const { room_id: roomId } = (await response.json()) as { room_id: string }; - return { usernames, roomId }; + return { users, roomId }; } /** * Opens the harness signed in as the given user and waits until the SDK - * reports the session as joined. The status line carries the SDK's error if - * it does not get there, so the failure says why. + * reports the session as connected. The status line carries the SDK's error + * if it does not get there, so the failure says why. */ export async function startHarness( page: Page, - username: string, + user: User, roomId: string, ): Promise { const query = new URLSearchParams({ homeserver: HOMESERVER_URL, - username, - password: PASSWORD, + accessToken: user.accessToken, + userId: user.userId, + deviceId: user.deviceId, room: roomId, }); await page.goto(`${SDK_HARNESS_URL}/?${query.toString()}`); await page.getByRole("button", { name: "Start" }).click(); - - // A login, a crypto setup and an initial sync happen first. An error is - // final, so it is not worth waiting out the timeout for "Joined" after one. - const status = page.getByTestId("status"); - await expect(status).toHaveText(/^(Joined|Error)/, { timeout: 120_000 }); - const text = await status.textContent(); - if (text !== "Joined") - throw new Error(`The harness did not join the session: ${text}`); + await waitForConnected(page); +} + +/** + * A login, a crypto setup and an initial sync happen before the session can + * connect. An error is final, so it is not worth waiting out the timeout for + * "connected" after one. + */ +export async function waitForConnected(page: Page): Promise { + const status = page.getByTestId("status"); + await expect(status).toHaveText(/^(connected|Error)/, { timeout: 120_000 }); + const text = await status.textContent(); + if (text !== "connected") + throw new Error(`The harness did not connect to the session: ${text}`); +} + +/** + * A context of its own per user, since a browser profile holds one login. No + * permissions to grant: each browser is launched with fake media that is + * handed out without asking (see playwright.config.ts). + */ +export async function newPage(browser: Browser): Promise { + const context = await browser.newContext({ ignoreHTTPSErrors: true }); + return context.newPage(); +} + +/** Two users in one session, each on a page of their own. */ +export async function startPair( + browser: Browser, + name: string, +): Promise<{ pages: [Page, Page]; users: [User, User]; roomId: string }> { + const { users, roomId } = await createUsersAndRoom(name); + const pages = await Promise.all([newPage(browser), newPage(browser)]); + await Promise.all( + pages.map(async (page, i) => startHarness(page, users[i], roomId)), + ); + return { pages: pages as [Page, Page], users, roomId }; +} + +/** The tile a page shows for a user. */ +export function tileOf(page: Page, user: User): Locator { + return page.locator(`[data-testid="member"][data-user-id="${user.userId}"]`); } diff --git a/playwright/sdk/lifecycle.spec.ts b/playwright/sdk/lifecycle.spec.ts new file mode 100644 index 000000000..f28aa9797 --- /dev/null +++ b/playwright/sdk/lifecycle.spec.ts @@ -0,0 +1,67 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test } from "@playwright/test"; + +import { startPair, tileOf, waitForConnected } from "./harness.ts"; + +/** + * Members coming and going: a member that leaves disappears for the peer, one + * that comes back is shown again, and the harness is left in the state its + * status line claims. + */ + +test.describe.configure({ timeout: 300_000 }); + +test("a member that leaves disappears for the peer", async ({ browser }) => { + const { + pages: [leaver, stayer], + users: [leaving, staying], + } = await startPair(browser, "sdkleave"); + await expect(stayer.getByTestId("member")).toHaveCount(2, { + timeout: 60_000, + }); + + await leaver.getByRole("button", { name: "Leave" }).click(); + await expect(leaver.getByTestId("status")).toHaveText("Left"); + await expect(leaver.getByTestId("member")).toHaveCount(0); + + await expect(tileOf(stayer, leaving)).toHaveCount(0, { timeout: 60_000 }); + await expect(tileOf(stayer, staying)).toHaveCount(1); + await expect(stayer.getByTestId("status")).toHaveText("connected"); +}); + +test("a member that reloads is shown again with its media", async ({ + browser, +}) => { + const { + pages: [reloader, watcher], + users: [reloading], + } = await startPair(browser, "sdkreload"); + await expect(watcher.getByTestId("member")).toHaveCount(2, { + timeout: 60_000, + }); + + // The page comes back on the same device, so the new membership replaces + // the old one rather than sitting next to it + await reloader.reload(); + await reloader.getByRole("button", { name: "Start" }).click(); + await waitForConnected(reloader); + + await expect(tileOf(watcher, reloading)).toHaveCount(1, { + timeout: 120_000, + }); + const video = tileOf(watcher, reloading).locator("video"); + await expect + .poll(async () => video.evaluate((v: HTMLVideoElement) => v.videoWidth), { + timeout: 60_000, + }) + .toBeGreaterThan(0); + await expect(reloader.getByTestId("member")).toHaveCount(2, { + timeout: 60_000, + }); +}); diff --git a/playwright/sdk/media.spec.ts b/playwright/sdk/media.spec.ts new file mode 100644 index 000000000..b98a0a24d --- /dev/null +++ b/playwright/sdk/media.spec.ts @@ -0,0 +1,158 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Locator, type Page, test } from "@playwright/test"; + +import { startPair, tileOf, type User } from "./harness.ts"; + +/** + * What a member's media looks like from the other side: it plays, it is + * encrypted, it follows the mute switches, and the SFU sends only as much as + * the tile on screen can show. Each test reads the labels the harness puts on + * its media elements, which come straight from the SDK's track behaviors. + * + * One session serves every test here: logging two browsers in and connecting + * them takes most of a minute, and none of the tests leaves the session in a + * state the next cannot start from. + */ + +test.describe.configure({ mode: "serial", timeout: 300_000 }); + +let pages: [Page, Page]; +let users: [User, User]; + +test.beforeAll(async ({ browser }) => { + ({ pages, users } = await startPair(browser, "sdkmedia")); + for (const page of pages) + await expect(page.getByTestId("member")).toHaveCount(2, { + timeout: 60_000, + }); +}); + +test.afterAll(async () => { + for (const page of pages ?? []) await page.context().close(); +}); + +test("the peer's camera and microphone play", async () => { + const peer = remoteTile(1, 0); + const video = peer.locator("video"); + await expect + .poll(async () => video.evaluate((v: HTMLVideoElement) => v.videoWidth), { + timeout: 60_000, + }) + .toBeGreaterThan(0); + await expect(video).toHaveJSProperty("paused", false); + const audio = peer.locator("audio"); + await expect + .poll(async () => + audio.evaluate((a: HTMLAudioElement) => a.srcObject !== null), + ) + .toBe(true); + await expect(audio).toHaveJSProperty("paused", false); +}); + +test("the peer's tracks are encrypted", async () => { + const peer = remoteTile(1, 0); + await expect(peer.locator("video")).toHaveAttribute("data-encrypted", "true"); + await expect(peer.locator("audio")).toHaveAttribute("data-encrypted", "true"); +}); + +test("muting the microphone and the camera is seen by the peer", async () => { + const [page] = pages; + const peer = remoteTile(1, 0); + const video = peer.locator("video"); + const audio = peer.locator("audio"); + await expect(video).toHaveAttribute("data-muted", "false"); + await expect(audio).toHaveAttribute("data-muted", "false"); + + await page.getByRole("button", { name: "Microphone" }).click(); + await expect(audio).toHaveAttribute("data-muted", "true", { + timeout: 20_000, + }); + await expect(video).toHaveAttribute("data-muted", "false"); + + await page.getByRole("button", { name: "Camera" }).click(); + await expect(video).toHaveAttribute("data-muted", "true", { + timeout: 20_000, + }); + + await page.getByRole("button", { name: "Microphone" }).click(); + await page.getByRole("button", { name: "Camera" }).click(); + await expect(audio).toHaveAttribute("data-muted", "false", { + timeout: 20_000, + }); + await expect(video).toHaveAttribute("data-muted", "false", { + timeout: 20_000, + }); +}); + +test("a larger tile receives a higher resolution", async () => { + const peer = remoteTile(1, 0); + const video = peer.locator("video"); + // The tile starts small enough for the lowest simulcast layer + const small = await receivedWidth(video, (width) => width > 0); + await resize(peer, 1280); + const large = await receivedWidth(video, (width) => width > small); + expect(large).toBeGreaterThan(small); + await resize(peer, 240); + await receivedWidth(video, (width) => width < large); +}); + +test("a hidden tile stops receiving video, and a shown one resumes", async () => { + const peer = remoteTile(1, 0); + const video = peer.locator("video"); + await expect(frames(video)).resolves.toBeGreaterThan(0); + + await peer.evaluate((tile) => (tile.style.display = "none")); + await expect + .poll(async () => framesStill(video), { timeout: 60_000 }) + .toBe(true); + + await peer.evaluate((tile) => (tile.style.display = "")); + await expect + .poll(async () => !(await framesStill(video)), { timeout: 60_000 }) + .toBe(true); +}); + +/** The tile on one page for the user of the other page. */ +function remoteTile(viewer: 0 | 1, shown: 0 | 1): Locator { + return tileOf(pages[viewer], users[shown]); +} + +async function resize(tile: Locator, width: number): Promise { + await tile.evaluate((t, w) => (t.style.width = `${w}px`), width); +} + +/** The width of the video as decoded, once it satisfies the condition. */ +async function receivedWidth( + video: Locator, + until: (width: number) => boolean, +): Promise { + // Switching simulcast layers takes the SFU a few seconds + let width = 0; + await expect + .poll( + async () => { + width = await video.evaluate((v: HTMLVideoElement) => v.videoWidth); + return until(width); + }, + { timeout: 60_000 }, + ) + .toBe(true); + return width; +} + +async function frames(video: Locator): Promise { + return Number(await video.getAttribute("data-frames")); +} + +/** Whether no frame arrived over a couple of seconds. */ +async function framesStill(video: Locator): Promise { + const before = await frames(video); + await new Promise((resolve) => setTimeout(resolve, 2500)); + return (await frames(video)) === before; +} diff --git a/playwright/sdk/smoke.spec.ts b/playwright/sdk/smoke.spec.ts index 70a4b80c0..1a120288f 100644 --- a/playwright/sdk/smoke.spec.ts +++ b/playwright/sdk/smoke.spec.ts @@ -5,17 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type Browser, expect, type Page, test } from "@playwright/test"; +import { expect, test } from "@playwright/test"; -import { createUsersAndRoom, startHarness } from "./harness.ts"; +import { startPair, tileOf } from "./harness.ts"; /** * The MatrixRTC SDK driven through its harness in `sdk/dev`: no Element Call * on the page, only the SDK's public API. - * - * This is the smoke test the implementation is built against. It fails until - * `createRtcSession` does something, and it fails on the status line first, so - * the SDK's own error is what the report shows. */ // Two browsers each log in, set up crypto and sync before anything is on @@ -23,34 +19,14 @@ import { createUsersAndRoom, startHarness } from "./harness.ts"; test.describe.configure({ timeout: 300_000 }); test("two browsers see each other in one session", async ({ browser }) => { - const { usernames, roomId } = await createUsersAndRoom("sdksmoke"); - const [pageA, pageB] = await Promise.all( - usernames.map(async () => newPage(browser)), - ); - - await Promise.all([ - startHarness(pageA, usernames[0], roomId), - startHarness(pageB, usernames[1], roomId), - ]); + const { pages, users } = await startPair(browser, "sdksmoke"); // Each page shows itself and the other, each tile named by its user - for (const page of [pageA, pageB]) { + for (const page of pages) { await expect(page.getByTestId("member")).toHaveCount(2, { timeout: 60_000, }); - for (const username of usernames) - await expect( - page.getByTestId("member").filter({ hasText: username }), - ).toHaveCount(1); + for (const user of users) + await expect(tileOf(page, user)).toContainText(user.displayName); } }); - -/** - * A context of its own per user, since a browser profile holds one login. No - * permissions to grant: each browser is launched with fake media that is - * handed out without asking (see playwright.config.ts). - */ -async function newPage(browser: Browser): Promise { - const context = await browser.newContext({ ignoreHTTPSErrors: true }); - return context.newPage(); -} diff --git a/sdk/README.md b/sdk/README.md index e750513c3..29be57e1a 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -6,9 +6,11 @@ the media of every member, as observables. It has no UI. Element Call's own `CallViewModel` is meant to become one consumer of it; the design is in [`sdk-plan.md`](../sdk-plan.md). -**Status:** interface only. `createRtcSession` returns an object that does nothing. -The development harness and its e2e test exist so the implementation can be built -against them. +**Status:** first implementation. `createRtcSession` joins the session, connects to +the transport, publishes the local media and exposes every member's media; the +development harness and its e2e tests in `playwright/sdk` drive it. The +implementation still imports the building blocks it shares with Element Call from +`src/` (connections, memberships, key provider); moving them here is the next slice. ## Using it @@ -25,7 +27,7 @@ const scope = new ObservableScope(); const session = createRtcSession( scope, client, // a matrix-js-sdk MatrixClient, logged in and syncing - room, // the matrix-js-sdk Room to hold the session in + room, // the matrix-js-sdk Room to hold the session in, from client.getRoom() { microphoneEnabled$: constant(true), cameraEnabled$: constant(true), diff --git a/sdk/dev/index.html b/sdk/dev/index.html index 8496af4ba..b0de49e90 100644 --- a/sdk/dev/index.html +++ b/sdk/dev/index.html @@ -21,9 +21,12 @@ Please see LICENSE in the repository root for full details. flex-wrap: wrap; } #members { - display: grid; - grid-template-columns: repeat(auto-fill, minmax(240px, 1fr)); + display: flex; gap: 0.5rem; + flex-wrap: wrap; + } + #members section { + width: 240px; } #members video { width: 100%; @@ -46,7 +49,17 @@ Please see LICENSE in the repository root for full details. + + + + + +

diff --git a/sdk/dev/main.ts b/sdk/dev/main.ts index 51765fefc..8950861f8 100644 --- a/sdk/dev/main.ts +++ b/sdk/dev/main.ts @@ -7,12 +7,18 @@ Please see LICENSE in the repository root for full details. /** * The smallest consumer of the SDK: log in, join a room, show every member's - * camera and play every remote member's microphone, leave. What a host writes, - * and nothing a host would not. + * camera and play every remote member's microphone, mute, leave. What a host + * writes, and nothing a host would not. */ import { logger } from "matrix-js-sdk/lib/logger"; -import { combineLatest, type Observable, of, switchMap } from "rxjs"; +import { + BehaviorSubject, + combineLatest, + type Observable, + of, + switchMap, +} from "rxjs"; import { constant, createRtcSession, @@ -24,12 +30,16 @@ import { type RtcSession, } from "@element-hq/matrixrtc-sdk"; -import { createSession } from "./session"; +import { createSession, joinRoom, type Login } from "./session"; const form = document.querySelector("form")!; const status = document.getElementById("status")!; const members = document.getElementById("members")!; -const leaveButton = document.getElementById("leave") as HTMLButtonElement; +const buttons = { + microphone: document.getElementById("microphone") as HTMLButtonElement, + camera: document.getElementById("camera") as HTMLButtonElement, + leave: document.getElementById("leave") as HTMLButtonElement, +}; // So that a test, or a bookmark, can fill the form from the URL const params = new URLSearchParams(location.search); @@ -40,33 +50,36 @@ form.addEventListener("submit", (event) => { event.preventDefault(); const fields = new FormData(form); const field = (name: string): string => fields.get(name) as string; - void start( - field("homeserver"), - field("username"), - field("password"), - field("room"), - ); + const login: Login = field("accessToken") + ? { + accessToken: field("accessToken"), + userId: field("userId"), + deviceId: field("deviceId"), + } + : { username: field("username"), password: field("password") }; + void start(field("homeserver"), login, field("room")); }); async function start( homeserver: string, - username: string, - password: string, + login: Login, roomIdOrAlias: string, ): Promise { try { status.textContent = "Logging in"; - const client = await createSession(homeserver, username, password); - const room = await client.joinRoom(roomIdOrAlias); + const client = await createSession(homeserver, login); + const room = await joinRoom(client, roomIdOrAlias); const scope = new ObservableScope(); + const microphoneEnabled$ = new BehaviorSubject(true); + const cameraEnabled$ = new BehaviorSubject(true); const session = createRtcSession( scope, client, room, { - microphoneEnabled$: constant(true), - cameraEnabled$: constant(true), + microphoneEnabled$, + cameraEnabled$, audioInputDeviceId$: constant(undefined), videoInputDeviceId$: constant(undefined), videoProcessor$: constant(undefined), @@ -76,16 +89,23 @@ async function start( matrixRTCMode: MatrixRTCMode.Compatibility, }, ); + session.status$.pipe(scope.bind()).subscribe((s) => { + status.textContent = s; + }); + session.fatalError$.pipe(scope.bind()).subscribe((error) => { + if (error !== null) status.textContent = `Error: ${error.message}`; + }); showMembers(scope, session); session.join(); - status.textContent = "Joined"; - leaveButton.hidden = false; - leaveButton.onclick = (): void => { + toggle(buttons.microphone, microphoneEnabled$); + toggle(buttons.camera, cameraEnabled$); + buttons.leave.hidden = false; + buttons.leave.onclick = (): void => { session.leave(); scope.end(); members.replaceChildren(); - leaveButton.hidden = true; + for (const button of Object.values(buttons)) button.hidden = true; status.textContent = "Left"; }; } catch (e) { @@ -94,6 +114,17 @@ async function start( } } +function toggle( + button: HTMLButtonElement, + enabled$: BehaviorSubject, +): void { + button.hidden = false; + button.onclick = (): void => { + enabled$.next(!enabled$.value); + button.ariaPressed = String(enabled$.value); + }; +} + function showMembers(scope: ObservableScope, session: RtcSession): void { const tiles = new Map(); combineLatest([session.localMember$, session.remoteMembers$]) @@ -118,6 +149,7 @@ function memberTile(scope: ObservableScope, member: RtcMember): HTMLElement { const tile = document.createElement("section"); tile.dataset.testid = "member"; tile.dataset.userId = member.userId; + tile.dataset.local = String(member.local); const name = tile.appendChild(document.createElement("h2")); member.displayName$.pipe(scope.bind()).subscribe((n) => { @@ -142,6 +174,10 @@ function memberTile(scope: ObservableScope, member: RtcMember): HTMLElement { return tile; } +/** + * Plays a track on an element and labels the element with the track's state, + * so that the page shows what the member is sending and how well it arrives. + */ function render( scope: ObservableScope, track$: Observable, @@ -154,4 +190,43 @@ function render( track?.attach(element); }); scope.onEnd(() => attached?.detach(element)); + + const label = ( + key: string, + value$: Observable, + ): void => { + value$.pipe(scope.bind()).subscribe((value) => { + if (value === undefined) delete element.dataset[key]; + else element.dataset[key] = String(value); + }); + }; + const of$ = ( + pick: (track: MediaTrack) => Observable, + ): Observable => + track$.pipe(switchMap((track) => (track ? pick(track) : of(undefined)))); + label( + "muted", + of$((t) => t.muted$), + ); + label( + "encrypted", + of$((t) => t.encrypted$), + ); + const stats$ = of$((t) => t.stats$); + label("frameWidth", stats$.pipe(switchMap((s) => of(frames(s)?.frameWidth)))); + label("frames", stats$.pipe(switchMap((s) => of(frames(s)?.count)))); +} + +/** The frame counters an RTP stream reports, from either end of it. */ +function frames( + stats: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined, +): { frameWidth: number | undefined; count: number | undefined } | undefined { + if (stats === undefined) return undefined; + return { + frameWidth: stats.frameWidth, + count: + stats.type === "inbound-rtp" + ? (stats as RTCInboundRtpStreamStats).framesDecoded + : (stats as RTCOutboundRtpStreamStats).framesEncoded, + }; } diff --git a/sdk/dev/session.ts b/sdk/dev/session.ts index 301c38048..00b578e76 100644 --- a/sdk/dev/session.ts +++ b/sdk/dev/session.ts @@ -10,25 +10,29 @@ import { createClient, type MatrixClient, MemoryStore, + type Room, SyncState, } from "matrix-js-sdk"; +import { KnownMembership } from "matrix-js-sdk/lib/types"; -/** Logs in with a password and returns a client that has finished its first sync. */ +export type Login = + | { username: string; password: string } + | { accessToken: string; userId: string; deviceId: string }; + +/** + * Returns a client that has finished its first sync, logging in with a + * password unless the caller already holds a token. + */ export async function createSession( homeserver: string, - username: string, - password: string, + login: Login, ): Promise { - const login = await createClient({ baseUrl: homeserver }).login( - "m.login.password", - { identifier: { type: "m.id.user", user: username }, password }, - ); + const credentials = + "accessToken" in login ? login : await logIn(homeserver, login); const client = createClient({ baseUrl: homeserver, - accessToken: login.access_token, - userId: login.user_id, - deviceId: login.device_id, + ...credentials, store: new MemoryStore(), useAuthorizationHeader: true, fallbackICEServerAllowed: true, @@ -47,3 +51,49 @@ export async function createSession( return client; } + +/** + * Joins a room and returns it as the sync loop maintains it. The room object + * `joinRoom` itself returns for a room joined just now is a detached copy + * that never receives the state the sync delivers, and a session built on + * it would never see a member. + */ +export async function joinRoom( + client: MatrixClient, + roomIdOrAlias: string, +): Promise { + const { roomId } = await client.joinRoom(roomIdOrAlias); + const joinedRoom = (): Room | undefined => { + const room = client.getRoom(roomId); + return room?.hasMembershipState(client.getUserId()!, KnownMembership.Join) + ? room + : undefined; + }; + return ( + joinedRoom() ?? + new Promise((resolve) => { + const onSync = (): void => { + const room = joinedRoom(); + if (room === undefined) return; + client.off(ClientEvent.Sync, onSync); + resolve(room); + }; + client.on(ClientEvent.Sync, onSync); + }) + ); +} + +async function logIn( + homeserver: string, + { username, password }: { username: string; password: string }, +): Promise<{ accessToken: string; userId: string; deviceId: string }> { + const login = await createClient({ baseUrl: homeserver }).login( + "m.login.password", + { identifier: { type: "m.id.user", user: username }, password }, + ); + return { + accessToken: login.access_token, + userId: login.user_id, + deviceId: login.device_id, + }; +} diff --git a/sdk/index.ts b/sdk/index.ts index 3fbeade15..d392000fb 100644 --- a/sdk/index.ts +++ b/sdk/index.ts @@ -10,262 +10,18 @@ Please see LICENSE in the repository root for full details. * * MatrixRTC sessions with LiveKit media, without Element Call's UI: the call * model Element Call's own view model is built on, for hosts that want to - * build a different one. - * - * Only the interface exists so far. `createRtcSession` returns an object that - * does nothing, so that the development harness in `sdk/dev` and its e2e test - * can be written against the contract before the implementation lands behind - * it. The design and the migration from Element Call's `CallViewModel` are in - * `sdk-plan.md` at the repository root. + * build a different one. The design and the migration from Element Call's + * `CallViewModel` are in `sdk-plan.md` at the repository root. */ -import { type MatrixClient, type Room } from "matrix-js-sdk"; -import { - type CallMembership, - type RTCCallIntent, - type RTCNotificationType, - type Transport, -} from "matrix-js-sdk/lib/matrixrtc"; -import { type Track, type TrackProcessor } from "livekit-client"; -import { type Observable } from "rxjs"; - -import { type Behavior } from "../src/state/Behavior"; -import { type ObservableScope } from "../src/state/ObservableScope"; -import { type EncryptionSystem } from "../src/e2ee/sharedKeyManagement"; -import { type MatrixRTCMode } from "../src/config/ConfigOptions"; - -// Shared with Element Call. They live in `src` until the SDK has an -// implementation to move them with; a consumer gets them from here either way. +// Shared with Element Call. They live in `src` until the view model consumes +// the SDK, at which point they move here; a consumer gets them from this +// module either way. export { type Behavior, constant } from "../src/state/Behavior"; export { ObservableScope } from "../src/state/ObservableScope"; export { E2eeType } from "../src/e2ee/e2eeType"; export { type EncryptionSystem } from "../src/e2ee/sharedKeyManagement"; export { MatrixRTCMode } from "../src/config/ConfigOptions"; -// --------------------------------------------------------------------------- -// Session - -export interface RtcSessionOptions { - encryptionSystem: EncryptionSystem; - /** Resolved by the host; the SDK reads neither config.json nor settings. */ - matrixRTCMode: MatrixRTCMode; - /** - * MSC4075 notification sent with the join. Parameters of the MatrixRTC join - * itself, so they are here even though they are named after calls; reacting - * to a notification (ringing, timeouts, declines) is the application's job. - */ - sendNotificationType?: RTCNotificationType; - callIntent?: RTCCallIntent; -} - -/** What the local member publishes. */ -export interface LocalMediaInputs { - microphoneEnabled$: Behavior; - cameraEnabled$: Behavior; - audioInputDeviceId$: Behavior; - videoInputDeviceId$: Behavior; - /** Background blur and the like. */ - videoProcessor$: Behavior | undefined>; -} - -export type SessionConnectionStatus = - | "waitingForTransport" - | "connecting" - | "connected" - | "reconnecting" - | "disconnected"; - -export class RtcSessionError extends Error { - public constructor(message: string, options?: ErrorOptions) { - super(message, options); - this.name = "RtcSessionError"; - } -} - -export interface RtcSession { - join(): void; - leave(): void; - /** Collapsed view of the local member's state machine. */ - status$: Behavior; - connected$: Behavior; - reconnecting$: Behavior; - /** A transport, Matrix or connection error that stops the session. */ - fatalError$: Behavior; - - localMember$: Behavior; - remoteMembers$: Behavior; - /** `remoteMembers.length`, plus one for the local member once it exists. */ - memberCount$: Behavior; - - /** - * Whether the session has grown large enough that MatrixRTC has stopped - * rotating the media encryption key. - */ - keyRotationSuppressed$: Behavior; - - /** Transports the session currently holds a live connection to. */ - connectedTransports$: Behavior; -} - -// --------------------------------------------------------------------------- -// Transports - -/** - * One transport advertised in a membership. Transport independent: `type` and - * `id` are all the SDK needs; `raw` and `resolved$` are there for a - * backend-specific developer panel and for connection diagnostics. - */ -export interface TransportMetadata { - /** `"livekit"` today. */ - type: string; - /** Stable key, unique per transport in the session. For LiveKit, the service url. */ - id: string; - /** The transport object as it appears in the membership. */ - raw: Transport; - /** - * What the backend had to fetch before it could connect. Undefined until the - * connection has resolved it, and again after the connection stops. - */ - resolved$: Behavior; -} - -export type ResolvedTransport = - | { - type: "livekit"; - /** The SFU websocket url, as opposed to the JWT service url in `raw`. */ - url: string; - /** A secret: fit for a developer panel, never for a log line. */ - token: string; - roomAlias: string; - identity: string; - } - | { type: string; [key: string]: unknown }; - -// --------------------------------------------------------------------------- -// Members - -export interface RtcMember { - local: boolean; - /** `${userId}:${deviceId}` before sticky events, a uuid in Matrix 2.0 mode. The backend identity. */ - id: string; - userId: string; - deviceId: string; - membership$: Behavior; - displayName$: Behavior; - avatarUrl$: Behavior; - /** Which transport this member is on; undefined when the membership has none. */ - transport$: Behavior; - /** - * Null while the member has a transport but no media has arrived on it yet - * ("waiting for media"). - */ - media$: Behavior; -} - -export interface RemoteRtcMember extends RtcMember { - local: false; -} - -export interface LocalRtcMember extends RtcMember { - local: true; - media$: Behavior; - sharingScreen$: Behavior; - /** Null when the platform cannot share a screen. */ - toggleScreenSharing: (() => void) | null; - screenShareError$: Behavior; - dismissScreenShareError(): void; -} - -// --------------------------------------------------------------------------- -// Media - -export type MediaSource = - | "microphone" - | "camera" - | "screenShare" - | "screenShareAudio"; - -export type MediaStreamStats = - | RTCInboundRtpStreamStats - | RTCOutboundRtpStreamStats - | undefined; - -/** One published track of a member. */ -export interface MediaTrack { - source: MediaSource; - kind: "audio" | "video"; - /** Stable for the life of the track. */ - id: string; - muted$: Behavior; - /** False when the SFU reports the track as unencrypted. */ - encrypted$: Behavior; - /** Polled while subscribed. */ - stats$: Behavior; - /** - * Rendering. The view hands its `