From 584bbaa0780f4f07fc270a105fab9542f11de8e9 Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Tue, 3 Nov 2026 16:26:18 +0100 Subject: [PATCH] Keep both LiveKit processor converters in the SDK (fix circular import) - rename convertToLivekitProcessor to toLivekitProcessor - move code to src/tracks -> sdk/src/media-backend/livekit --- sdk/index.ts | 5 +- .../livekit/LivekitMediaTrack.ts | 4 +- .../media-backend/livekit/Publisher.test.ts | 4 +- .../media-backend/livekit/publishOptions.ts | 4 +- .../livekit/videoProcessor.test.ts | 53 +++++++++++++++--- .../media-backend/livekit/videoProcessor.ts | 33 ++++++++++- src/room/KnockLobbyView.test.tsx | 2 +- src/room/LobbyView.test.tsx | 2 +- src/room/LobbyView.tsx | 5 +- src/state/BackgroundEffects.test.ts | 2 +- src/state/BackgroundEffects.ts | 13 +++-- src/tracks/BackgroundEffectsContext.test.ts | 56 +++---------------- src/tracks/BackgroundEffectsContext.tsx | 43 ++------------ 13 files changed, 112 insertions(+), 114 deletions(-) diff --git a/sdk/index.ts b/sdk/index.ts index 471d3862f..b4f7ef0e1 100644 --- a/sdk/index.ts +++ b/sdk/index.ts @@ -62,7 +62,10 @@ export { type LivekitBackendOptions, } from "./src/media-backend/livekit/LivekitBackend"; // For a host that creates LiveKit tracks itself, such as a preview before joining -export { convertToLivekitProcessor } from "./src/media-backend/livekit/videoProcessor"; +export { + fromLivekitProcessor, + toLivekitProcessor, +} from "./src/media-backend/livekit/videoProcessor"; // For a developer panel to check a transport before using it export { getSFUConfig as authenticateWithTransport, diff --git a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts index 3b07d2455..a0a2ddcef 100644 --- a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts +++ b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts @@ -50,7 +50,7 @@ import { type VideoMediaTrack, } from "../../media-api"; import { MatrixRTCError } from "../../errors"; -import { convertToLivekitProcessor } from "./videoProcessor"; +import { toLivekitProcessor } from "./videoProcessor"; export const mediaSources: Record = { [Track.Source.Microphone]: "microphone", @@ -231,7 +231,7 @@ export function createLocalLivekitMediaTrack( // A processor cannot be built on a track that has already ended if (!track || track.mediaStreamTrack.readyState === "ended") return; if (processor) { - const adapter = convertToLivekitProcessor(processor); + const adapter = toLivekitProcessor(processor); if (track.getProcessor() !== adapter) await track.setProcessor(adapter); } else if (track.getProcessor()) await track.stopProcessor(); }, diff --git a/sdk/src/media-backend/livekit/Publisher.test.ts b/sdk/src/media-backend/livekit/Publisher.test.ts index 80ffbede2..319ce7881 100644 --- a/sdk/src/media-backend/livekit/Publisher.test.ts +++ b/sdk/src/media-backend/livekit/Publisher.test.ts @@ -21,7 +21,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { type PublishRequest, type VideoProcessor } from "../../media-api"; import { flushPromises } from "../../utils/test"; import { type DesiredMedia, Publisher } from "./Publisher"; -import { convertToLivekitProcessor } from "./videoProcessor"; +import { toLivekitProcessor } from "./videoProcessor"; describe("Publisher", () => { describe("setAudioOutputDevice", () => { @@ -134,7 +134,7 @@ describe("Publisher", () => { expect(room.options.videoCaptureDefaults).toMatchObject({ deviceId: "front", - processor: convertToLivekitProcessor(processor), + processor: toLivekitProcessor(processor), }); }); diff --git a/sdk/src/media-backend/livekit/publishOptions.ts b/sdk/src/media-backend/livekit/publishOptions.ts index f030c3fd4..cbdfbdc6f 100644 --- a/sdk/src/media-backend/livekit/publishOptions.ts +++ b/sdk/src/media-backend/livekit/publishOptions.ts @@ -16,7 +16,7 @@ import { type PublishRequest, type VideoCaptureSettings, } from "../../media-api"; -import { convertToLivekitProcessor } from "./videoProcessor"; +import { toLivekitProcessor } from "./videoProcessor"; export type PublishRequestFor = Extract< PublishRequest, @@ -40,7 +40,7 @@ export function videoCaptureOptions({ if (!deviceId && !processor && !capture?.resolution) return undefined; return { ...(deviceId && { deviceId }), - ...(processor && { processor: convertToLivekitProcessor(processor) }), + ...(processor && { processor: toLivekitProcessor(processor) }), ...(capture?.resolution && { resolution: capture.resolution }), }; } diff --git a/sdk/src/media-backend/livekit/videoProcessor.test.ts b/sdk/src/media-backend/livekit/videoProcessor.test.ts index b9d26e414..71892aee3 100644 --- a/sdk/src/media-backend/livekit/videoProcessor.test.ts +++ b/sdk/src/media-backend/livekit/videoProcessor.test.ts @@ -5,13 +5,17 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { Track, type VideoProcessorOptions } from "livekit-client"; +import { + Track, + type TrackProcessor, + type VideoProcessorOptions, +} from "livekit-client"; import { describe, expect, it, vi } from "vitest"; import { type VideoProcessor } from "../../media-api"; -import { convertToLivekitProcessor } from "./videoProcessor"; +import { fromLivekitProcessor, toLivekitProcessor } from "./videoProcessor"; -describe("convertToLivekitProcessor", () => { +describe("toLivekitProcessor", () => { const captured = { id: "captured" } as MediaStreamTrack; const processed = { id: "processed" } as MediaStreamTrack; const restarted = { id: "restarted" } as MediaStreamTrack; @@ -33,7 +37,7 @@ describe("convertToLivekitProcessor", () => { it("holds the processed track where LiveKit reads it", async () => { const processor = fakeProcessor(); - const adapter = convertToLivekitProcessor(processor); + const adapter = toLivekitProcessor(processor); expect(adapter.name).toBe("blur"); expect(adapter.processedTrack).toBeUndefined(); @@ -50,8 +54,43 @@ describe("convertToLivekitProcessor", () => { it("gives one processor one adapter", () => { const processor = fakeProcessor(); - expect(convertToLivekitProcessor(processor)).toBe( - convertToLivekitProcessor(processor), - ); + expect(toLivekitProcessor(processor)).toBe(toLivekitProcessor(processor)); + }); +}); + +describe("fromLivekitProcessor", () => { + const captured = { id: "captured" } as MediaStreamTrack; + const processed = { id: "processed" } as MediaStreamTrack; + + function fakeLivekitProcessor( + processedTrack: MediaStreamTrack | undefined, + ): TrackProcessor { + return { + name: "background-blur", + processedTrack, + init: vi.fn().mockResolvedValue(undefined), + restart: vi.fn().mockResolvedValue(undefined), + destroy: vi.fn().mockResolvedValue(undefined), + }; + } + + it("resolves with the track the LiveKit processor produced", async () => { + const livekitProcessor = fakeLivekitProcessor(processed); + const processor = fromLivekitProcessor(livekitProcessor); + expect(processor.name).toBe("background-blur"); + await expect(processor.init(captured)).resolves.toBe(processed); + expect(livekitProcessor.init).toHaveBeenCalledWith({ + kind: Track.Kind.Video, + track: captured, + }); + await expect(processor.restart(captured)).resolves.toBe(processed); + await processor.destroy(); + expect(livekitProcessor.destroy).toHaveBeenCalled(); + }); + + it("rejects where the LiveKit processor produced no track", async () => { + await expect( + fromLivekitProcessor(fakeLivekitProcessor(undefined)).init(captured), + ).rejects.toThrow("produced no track"); }); }); diff --git a/sdk/src/media-backend/livekit/videoProcessor.ts b/sdk/src/media-backend/livekit/videoProcessor.ts index fdf84d4e3..31a761437 100644 --- a/sdk/src/media-backend/livekit/videoProcessor.ts +++ b/sdk/src/media-backend/livekit/videoProcessor.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type Track, type TrackProcessor } from "livekit-client"; +import { Track, type TrackProcessor } from "livekit-client"; import { type VideoProcessor } from "../../media-api"; @@ -15,14 +15,14 @@ const adapters = new WeakMap< >(); /** - * A `VideoProcessor` as livekit-client wants it. LiveKit reads the processed + * A `VideoProcessor` as a livekit-client processor. LiveKit reads the processed * track off the processor after `init` and `restart` rather than taking it as * their result, so the adapter keeps it there. * * One adapter per processor, so a track can tell that it already runs one: a * processor set again would otherwise be torn down and built anew. */ -export function convertToLivekitProcessor( +export function toLivekitProcessor( processor: VideoProcessor, ): TrackProcessor { const existing = adapters.get(processor); @@ -43,3 +43,30 @@ export function convertToLivekitProcessor( adapters.set(processor, adapter); return adapter; } + +/** + * A livekit-client processor, such as one from `@livekit/track-processors`, + * as a `VideoProcessor`: `init` and `restart` resolve with the track it + * leaves on itself. + */ +export function fromLivekitProcessor( + processor: TrackProcessor, +): VideoProcessor { + const processedTrack = (): MediaStreamTrack => { + if (processor.processedTrack === undefined) + throw new Error(`Processor ${processor.name} produced no track`); + return processor.processedTrack; + }; + return { + name: processor.name, + init: async (track, element) => { + await processor.init({ kind: Track.Kind.Video, track, element }); + return processedTrack(); + }, + restart: async (track, element) => { + await processor.restart({ kind: Track.Kind.Video, track, element }); + return processedTrack(); + }, + destroy: async () => processor.destroy(), + }; +} diff --git a/src/room/KnockLobbyView.test.tsx b/src/room/KnockLobbyView.test.tsx index 0bfb76c2c..a99423cde 100644 --- a/src/room/KnockLobbyView.test.tsx +++ b/src/room/KnockLobbyView.test.tsx @@ -14,7 +14,7 @@ import { type MatrixClient, type RoomSummary } from "matrix-js-sdk"; import { KnockLobbyView } from "./KnockLobbyView"; import { LeaveToHomeProvider } from "../LeaveToHomeContext"; import { MediaDevicesContext } from "../MediaDevicesContext"; -import { type BackgroundEffectsState } from "../tracks/BackgroundEffectsContext"; +import { type BackgroundEffectsState } from "../state/BackgroundEffects"; import { mockMediaDevices } from "../utils/test"; vi.mock("@livekit/components-react", () => ({ diff --git a/src/room/LobbyView.test.tsx b/src/room/LobbyView.test.tsx index ab6bc581e..c9a3c5817 100644 --- a/src/room/LobbyView.test.tsx +++ b/src/room/LobbyView.test.tsx @@ -28,7 +28,7 @@ import { } from "../utils/test"; import { type MediaDevices } from "../state/MediaDevices"; import { MediaDevicesContext } from "../MediaDevicesContext"; -import { type BackgroundEffectsState } from "../tracks/BackgroundEffectsContext"; +import { type BackgroundEffectsState } from "../state/BackgroundEffects"; import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; import lobbyStyles from "./LobbyView.module.css"; import headerStyles from "../Header.module.css"; diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index b7c283b7f..32934f13d 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -24,7 +24,7 @@ import { type LocalVideoTrack, Track, } from "livekit-client"; -import { convertToLivekitProcessor } from "@element-hq/matrixrtc-sdk"; +import { toLivekitProcessor } from "@element-hq/matrixrtc-sdk"; import inCallStyles from "./InCallView.module.css"; import styles from "./LobbyView.module.css"; @@ -151,8 +151,7 @@ export const LobbyView: FC = ({ audio: Object.assign({}, initialAudioOptions), video: videoEnabled && { deviceId: videoInputId, - processor: - initialProcessor && convertToLivekitProcessor(initialProcessor), + processor: initialProcessor && toLivekitProcessor(initialProcessor), }, }), [initialAudioOptions, videoEnabled, videoInputId, initialProcessor], diff --git a/src/state/BackgroundEffects.test.ts b/src/state/BackgroundEffects.test.ts index 80671f868..96ae80ba0 100644 --- a/src/state/BackgroundEffects.test.ts +++ b/src/state/BackgroundEffects.test.ts @@ -13,8 +13,8 @@ import { type BackgroundEffects, createBackgroundEffects, type BackgroundEffectsOptions, + type BackgroundEffectsState, } from "./BackgroundEffects"; -import { type BackgroundEffectsState } from "../tracks/BackgroundEffectsContext"; import { flushPromises, testScope, withTestScheduler } from "../utils/test"; /** A pipeline that records what it is switched to. */ diff --git a/src/state/BackgroundEffects.ts b/src/state/BackgroundEffects.ts index ae47f3d48..25c4b2c7d 100644 --- a/src/state/BackgroundEffects.ts +++ b/src/state/BackgroundEffects.ts @@ -12,16 +12,21 @@ import { } from "@livekit/track-processors"; import { logger } from "matrix-js-sdk/lib/logger"; import { deepCompare } from "matrix-js-sdk/lib/utils"; +import { + fromLivekitProcessor, + type VideoProcessor, +} from "@element-hq/matrixrtc-sdk"; import { type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; -import { - fromLivekitProcessor, - type BackgroundEffectsState, -} from "../tracks/BackgroundEffectsContext"; const blurRadius = 15; +export interface BackgroundEffectsState { + supported: boolean | undefined; + processor: undefined | VideoProcessor; +} + export interface BackgroundEffectsOptions { /** Whether this browser can run a pipeline at all. */ supported: boolean; diff --git a/src/tracks/BackgroundEffectsContext.test.ts b/src/tracks/BackgroundEffectsContext.test.ts index 47afdd35f..068db5a54 100644 --- a/src/tracks/BackgroundEffectsContext.test.ts +++ b/src/tracks/BackgroundEffectsContext.test.ts @@ -10,25 +10,22 @@ import { render } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { type LocalVideoTrack } from "livekit-client"; import { - type BackgroundOptions, - type ProcessorWrapper, -} from "@livekit/track-processors"; -import { - convertToLivekitProcessor, + toLivekitProcessor, type VideoProcessor, } from "@element-hq/matrixrtc-sdk"; import { applyProcessor, - fromLivekitProcessor, BackgroundEffectsProvider, - type BackgroundEffectsState, syncBackgroundEffects, useBackgroundEffects, useSyncBackgroundEffects, createAppBackgroundEffects, } from "./BackgroundEffectsContext"; -import { type BackgroundEffects } from "../state/BackgroundEffects"; +import { + type BackgroundEffects, + type BackgroundEffectsState, +} from "../state/BackgroundEffects"; import { backgroundBlur } from "../settings/settings"; import { constant } from "../state/Behavior"; import { flushPromises, testScope } from "../utils/test"; @@ -135,43 +132,6 @@ describe("syncBackgroundEffects", () => { }); }); -describe("fromLivekitProcessor", () => { - const captured = { id: "captured" } as MediaStreamTrack; - const processed = { id: "processed" } as MediaStreamTrack; - - function mockWrapper( - processedTrack: MediaStreamTrack | undefined, - ): ProcessorWrapper { - return { - name: "background-blur", - processedTrack, - init: vi.fn().mockResolvedValue(undefined), - restart: vi.fn().mockResolvedValue(undefined), - destroy: vi.fn().mockResolvedValue(undefined), - } as unknown as ProcessorWrapper; - } - - it("resolves with the track the wrapper produced", async () => { - const wrapper = mockWrapper(processed); - const sdkProcessor = fromLivekitProcessor(wrapper); - expect(sdkProcessor.name).toBe("background-blur"); - await expect(sdkProcessor.init(captured)).resolves.toBe(processed); - expect(wrapper.init).toHaveBeenCalledWith({ - kind: "video", - track: captured, - }); - await expect(sdkProcessor.restart(captured)).resolves.toBe(processed); - await sdkProcessor.destroy(); - expect(wrapper.destroy).toHaveBeenCalled(); - }); - - it("rejects where the wrapper produced no track", async () => { - await expect( - fromLivekitProcessor(mockWrapper(undefined)).init(captured), - ).rejects.toThrow("produced no track"); - }); -}); - describe("BackgroundEffectsProvider", () => { /** A camera track that holds whatever processor it is given. */ function cameraTrack(): LocalVideoTrack { @@ -263,10 +223,10 @@ describe("BackgroundEffectsProvider", () => { const pipeline = latest().processor; expect(preview.setProcessor).toHaveBeenCalledWith( - convertToLivekitProcessor(pipeline!), + toLivekitProcessor(pipeline!), ); expect(call.setProcessor).toHaveBeenCalledWith( - convertToLivekitProcessor(pipeline!), + toLivekitProcessor(pipeline!), ); expect(pipelines.built).toBe(1); }); @@ -286,7 +246,7 @@ describe("BackgroundEffectsProvider", () => { }); expect(track.setProcessor).toHaveBeenCalledTimes(1); expect(track.setProcessor).toHaveBeenCalledWith( - convertToLivekitProcessor(latest().processor!), + toLivekitProcessor(latest().processor!), ); expect(pipelines.built).toBe(1); expect(pipelines.switches).toHaveLength(1); diff --git a/src/tracks/BackgroundEffectsContext.tsx b/src/tracks/BackgroundEffectsContext.tsx index 536c547e2..e1b0ca109 100644 --- a/src/tracks/BackgroundEffectsContext.tsx +++ b/src/tracks/BackgroundEffectsContext.tsx @@ -6,16 +6,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - type ProcessorWrapper, - type BackgroundOptions, -} from "@livekit/track-processors"; import { createContext, type FC, type JSX, use, useEffect } from "react"; -import { type LocalVideoTrack, Track } from "livekit-client"; +import { type LocalVideoTrack } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { combineLatest } from "rxjs"; import { - convertToLivekitProcessor, + toLivekitProcessor, type VideoProcessor, } from "@element-hq/matrixrtc-sdk"; @@ -27,6 +23,7 @@ import { type Behavior } from "../state/Behavior"; import { type ObservableScope } from "../state/ObservableScope"; import { type BackgroundEffects, + type BackgroundEffectsState, createBackgroundEffects, } from "../state/BackgroundEffects"; import { useBehavior } from "../useBehavior"; @@ -35,38 +32,6 @@ import { useBehavior } from "../useBehavior"; // it is a combination of exposing observable and react hooks. // preferably we should not make this a context anymore and instead just a vm? -export type BackgroundEffectsState = { - supported: boolean | undefined; - processor: undefined | VideoProcessor; -}; - -/** - * This allows to use a LiveKit processor for the matrixRTCSDK (VideoProcessor). - * They have slightly different apis so we need to convert them with this - * function before we can use them in the matrixRTCSDK. - */ -export function fromLivekitProcessor( - wrapper: ProcessorWrapper, -): VideoProcessor { - const processedTrack = (): MediaStreamTrack => { - if (wrapper.processedTrack === undefined) - throw new Error(`Processor ${wrapper.name} produced no track`); - return wrapper.processedTrack; - }; - return { - name: wrapper.name, - init: async (track: MediaStreamTrack, element?: HTMLMediaElement) => { - await wrapper.init({ kind: Track.Kind.Video, track, element }); - return processedTrack(); - }, - restart: async (track: MediaStreamTrack, element?: HTMLMediaElement) => { - await wrapper.restart({ kind: Track.Kind.Video, track, element }); - return processedTrack(); - }, - destroy: async () => wrapper.destroy(), - }; -} - const BackgroundEffectsContext = createContext( undefined, ); @@ -102,7 +67,7 @@ export function applyProcessor( return; } const track = videoTrack.mediaStreamTrack; - videoTrack.setProcessor(convertToLivekitProcessor(processor)).catch((e) => { + videoTrack.setProcessor(toLivekitProcessor(processor)).catch((e) => { if (track.readyState === "ended") logger.debug("Video processor not attached: the track ended first"); else logger.warn("Failed to attach video processor", e);