Keep both LiveKit processor converters in the SDK (fix circular import)

- rename convertToLivekitProcessor to toLivekitProcessor
 - move code to src/tracks -> sdk/src/media-backend/livekit

sdk/index.ts

@@ -62,7 +62,10 @@ export {
   type LivekitBackendOptions,
 } from "./src/media-backend/livekit/LivekitBackend";
 // For a host that creates LiveKit tracks itself, such as a preview before joining
-export { convertToLivekitProcessor } from "./src/media-backend/livekit/videoProcessor";
+export {
+  fromLivekitProcessor,
+  toLivekitProcessor,
+} from "./src/media-backend/livekit/videoProcessor";
 // For a developer panel to check a transport before using it
 export {
   getSFUConfig as authenticateWithTransport,

sdk/src/media-backend/livekit/LivekitMediaTrack.ts

@@ -50,7 +50,7 @@ import {
   type VideoMediaTrack,
 } from "../../media-api";
 import { MatrixRTCError } from "../../errors";
-import { convertToLivekitProcessor } from "./videoProcessor";
+import { toLivekitProcessor } from "./videoProcessor";
 
 export const mediaSources: Record<Track.Source, MediaSource> = {
   [Track.Source.Microphone]: "microphone",
@@ -231,7 +231,7 @@ export function createLocalLivekitMediaTrack(
       // A processor cannot be built on a track that has already ended
       if (!track || track.mediaStreamTrack.readyState === "ended") return;
       if (processor) {
-        const adapter = convertToLivekitProcessor(processor);
+        const adapter = toLivekitProcessor(processor);
         if (track.getProcessor() !== adapter) await track.setProcessor(adapter);
       } else if (track.getProcessor()) await track.stopProcessor();
     },

sdk/src/media-backend/livekit/Publisher.test.ts

@@ -21,7 +21,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { type PublishRequest, type VideoProcessor } from "../../media-api";
 import { flushPromises } from "../../utils/test";
 import { type DesiredMedia, Publisher } from "./Publisher";
-import { convertToLivekitProcessor } from "./videoProcessor";
+import { toLivekitProcessor } from "./videoProcessor";
 
 describe("Publisher", () => {
   describe("setAudioOutputDevice", () => {
@@ -134,7 +134,7 @@ describe("Publisher", () => {
 
       expect(room.options.videoCaptureDefaults).toMatchObject({
         deviceId: "front",
-        processor: convertToLivekitProcessor(processor),
+        processor: toLivekitProcessor(processor),
       });
     });
 

sdk/src/media-backend/livekit/publishOptions.ts

@@ -16,7 +16,7 @@ import {
   type PublishRequest,
   type VideoCaptureSettings,
 } from "../../media-api";
-import { convertToLivekitProcessor } from "./videoProcessor";
+import { toLivekitProcessor } from "./videoProcessor";
 
 export type PublishRequestFor<S extends PublishRequest["source"]> = Extract<
   PublishRequest,
@@ -40,7 +40,7 @@ export function videoCaptureOptions({
   if (!deviceId && !processor && !capture?.resolution) return undefined;
   return {
     ...(deviceId && { deviceId }),
-    ...(processor && { processor: convertToLivekitProcessor(processor) }),
+    ...(processor && { processor: toLivekitProcessor(processor) }),
     ...(capture?.resolution && { resolution: capture.resolution }),
   };
 }

sdk/src/media-backend/livekit/videoProcessor.test.ts

@@ -5,13 +5,17 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { Track, type VideoProcessorOptions } from "livekit-client";
+import {
+  Track,
+  type TrackProcessor,
+  type VideoProcessorOptions,
+} from "livekit-client";
 import { describe, expect, it, vi } from "vitest";
 
 import { type VideoProcessor } from "../../media-api";
-import { convertToLivekitProcessor } from "./videoProcessor";
+import { fromLivekitProcessor, toLivekitProcessor } from "./videoProcessor";
 
-describe("convertToLivekitProcessor", () => {
+describe("toLivekitProcessor", () => {
   const captured = { id: "captured" } as MediaStreamTrack;
   const processed = { id: "processed" } as MediaStreamTrack;
   const restarted = { id: "restarted" } as MediaStreamTrack;
@@ -33,7 +37,7 @@ describe("convertToLivekitProcessor", () => {
 
   it("holds the processed track where LiveKit reads it", async () => {
     const processor = fakeProcessor();
-    const adapter = convertToLivekitProcessor(processor);
+    const adapter = toLivekitProcessor(processor);
     expect(adapter.name).toBe("blur");
     expect(adapter.processedTrack).toBeUndefined();
 
@@ -50,8 +54,43 @@ describe("convertToLivekitProcessor", () => {
 
   it("gives one processor one adapter", () => {
     const processor = fakeProcessor();
-    expect(convertToLivekitProcessor(processor)).toBe(
-      convertToLivekitProcessor(processor),
-    );
+    expect(toLivekitProcessor(processor)).toBe(toLivekitProcessor(processor));
+  });
+});
+
+describe("fromLivekitProcessor", () => {
+  const captured = { id: "captured" } as MediaStreamTrack;
+  const processed = { id: "processed" } as MediaStreamTrack;
+
+  function fakeLivekitProcessor(
+    processedTrack: MediaStreamTrack | undefined,
+  ): TrackProcessor<Track.Kind.Video> {
+    return {
+      name: "background-blur",
+      processedTrack,
+      init: vi.fn().mockResolvedValue(undefined),
+      restart: vi.fn().mockResolvedValue(undefined),
+      destroy: vi.fn().mockResolvedValue(undefined),
+    };
+  }
+
+  it("resolves with the track the LiveKit processor produced", async () => {
+    const livekitProcessor = fakeLivekitProcessor(processed);
+    const processor = fromLivekitProcessor(livekitProcessor);
+    expect(processor.name).toBe("background-blur");
+    await expect(processor.init(captured)).resolves.toBe(processed);
+    expect(livekitProcessor.init).toHaveBeenCalledWith({
+      kind: Track.Kind.Video,
+      track: captured,
+    });
+    await expect(processor.restart(captured)).resolves.toBe(processed);
+    await processor.destroy();
+    expect(livekitProcessor.destroy).toHaveBeenCalled();
+  });
+
+  it("rejects where the LiveKit processor produced no track", async () => {
+    await expect(
+      fromLivekitProcessor(fakeLivekitProcessor(undefined)).init(captured),
+    ).rejects.toThrow("produced no track");
   });
 });

sdk/src/media-backend/livekit/videoProcessor.ts

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { type Track, type TrackProcessor } from "livekit-client";
+import { Track, type TrackProcessor } from "livekit-client";
 
 import { type VideoProcessor } from "../../media-api";
 
@@ -15,14 +15,14 @@ const adapters = new WeakMap<
 >();
 
 /**
- * A `VideoProcessor` as livekit-client wants it. LiveKit reads the processed
+ * A `VideoProcessor` as a livekit-client processor. LiveKit reads the processed
  * track off the processor after `init` and `restart` rather than taking it as
  * their result, so the adapter keeps it there.
  *
  * One adapter per processor, so a track can tell that it already runs one: a
  * processor set again would otherwise be torn down and built anew.
  */
-export function convertToLivekitProcessor(
+export function toLivekitProcessor(
   processor: VideoProcessor,
 ): TrackProcessor<Track.Kind.Video> {
   const existing = adapters.get(processor);
@@ -43,3 +43,30 @@ export function convertToLivekitProcessor(
   adapters.set(processor, adapter);
   return adapter;
 }
+
+/**
+ * A livekit-client processor, such as one from `@livekit/track-processors`,
+ * as a `VideoProcessor`: `init` and `restart` resolve with the track it
+ * leaves on itself.
+ */
+export function fromLivekitProcessor(
+  processor: TrackProcessor<Track.Kind.Video>,
+): VideoProcessor {
+  const processedTrack = (): MediaStreamTrack => {
+    if (processor.processedTrack === undefined)
+      throw new Error(`Processor ${processor.name} produced no track`);
+    return processor.processedTrack;
+  };
+  return {
+    name: processor.name,
+    init: async (track, element) => {
+      await processor.init({ kind: Track.Kind.Video, track, element });
+      return processedTrack();
+    },
+    restart: async (track, element) => {
+      await processor.restart({ kind: Track.Kind.Video, track, element });
+      return processedTrack();
+    },
+    destroy: async () => processor.destroy(),
+  };
+}

src/room/KnockLobbyView.test.tsx

@@ -14,7 +14,7 @@ import { type MatrixClient, type RoomSummary } from "matrix-js-sdk";
 import { KnockLobbyView } from "./KnockLobbyView";
 import { LeaveToHomeProvider } from "../LeaveToHomeContext";
 import { MediaDevicesContext } from "../MediaDevicesContext";
-import { type BackgroundEffectsState } from "../tracks/BackgroundEffectsContext";
+import { type BackgroundEffectsState } from "../state/BackgroundEffects";
 import { mockMediaDevices } from "../utils/test";
 
 vi.mock("@livekit/components-react", () => ({

src/room/LobbyView.test.tsx

@@ -28,7 +28,7 @@ import {
 } from "../utils/test";
 import { type MediaDevices } from "../state/MediaDevices";
 import { MediaDevicesContext } from "../MediaDevicesContext";
-import { type BackgroundEffectsState } from "../tracks/BackgroundEffectsContext";
+import { type BackgroundEffectsState } from "../state/BackgroundEffects";
 import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
 import lobbyStyles from "./LobbyView.module.css";
 import headerStyles from "../Header.module.css";

src/room/LobbyView.tsx

@@ -24,7 +24,7 @@ import {
   type LocalVideoTrack,
   Track,
 } from "livekit-client";
-import { convertToLivekitProcessor } from "@element-hq/matrixrtc-sdk";
+import { toLivekitProcessor } from "@element-hq/matrixrtc-sdk";
 
 import inCallStyles from "./InCallView.module.css";
 import styles from "./LobbyView.module.css";
@@ -151,8 +151,7 @@ export const LobbyView: FC<Props> = ({
       audio: Object.assign({}, initialAudioOptions),
       video: videoEnabled && {
         deviceId: videoInputId,
-        processor:
-          initialProcessor && convertToLivekitProcessor(initialProcessor),
+        processor: initialProcessor && toLivekitProcessor(initialProcessor),
       },
     }),
     [initialAudioOptions, videoEnabled, videoInputId, initialProcessor],

src/state/BackgroundEffects.test.ts

@@ -13,8 +13,8 @@ import {
   type BackgroundEffects,
   createBackgroundEffects,
   type BackgroundEffectsOptions,
+  type BackgroundEffectsState,
 } from "./BackgroundEffects";
-import { type BackgroundEffectsState } from "../tracks/BackgroundEffectsContext";
 import { flushPromises, testScope, withTestScheduler } from "../utils/test";
 
 /** A pipeline that records what it is switched to. */

src/state/BackgroundEffects.ts

@@ -12,16 +12,21 @@ import {
 } from "@livekit/track-processors";
 import { logger } from "matrix-js-sdk/lib/logger";
 import { deepCompare } from "matrix-js-sdk/lib/utils";
+import {
+  fromLivekitProcessor,
+  type VideoProcessor,
+} from "@element-hq/matrixrtc-sdk";
 
 import { type Behavior } from "./Behavior";
 import { type ObservableScope } from "./ObservableScope";
-import {
-  fromLivekitProcessor,
-  type BackgroundEffectsState,
-} from "../tracks/BackgroundEffectsContext";
 
 const blurRadius = 15;
 
+export interface BackgroundEffectsState {
+  supported: boolean | undefined;
+  processor: undefined | VideoProcessor;
+}
+
 export interface BackgroundEffectsOptions {
   /** Whether this browser can run a pipeline at all. */
   supported: boolean;

src/tracks/BackgroundEffectsContext.test.ts

@@ -10,25 +10,22 @@ import { render } from "@testing-library/react";
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { type LocalVideoTrack } from "livekit-client";
 import {
-  type BackgroundOptions,
-  type ProcessorWrapper,
-} from "@livekit/track-processors";
-import {
-  convertToLivekitProcessor,
+  toLivekitProcessor,
   type VideoProcessor,
 } from "@element-hq/matrixrtc-sdk";
 
 import {
   applyProcessor,
-  fromLivekitProcessor,
   BackgroundEffectsProvider,
-  type BackgroundEffectsState,
   syncBackgroundEffects,
   useBackgroundEffects,
   useSyncBackgroundEffects,
   createAppBackgroundEffects,
 } from "./BackgroundEffectsContext";
-import { type BackgroundEffects } from "../state/BackgroundEffects";
+import {
+  type BackgroundEffects,
+  type BackgroundEffectsState,
+} from "../state/BackgroundEffects";
 import { backgroundBlur } from "../settings/settings";
 import { constant } from "../state/Behavior";
 import { flushPromises, testScope } from "../utils/test";
@@ -135,43 +132,6 @@ describe("syncBackgroundEffects", () => {
   });
 });
 
-describe("fromLivekitProcessor", () => {
-  const captured = { id: "captured" } as MediaStreamTrack;
-  const processed = { id: "processed" } as MediaStreamTrack;
-
-  function mockWrapper(
-    processedTrack: MediaStreamTrack | undefined,
-  ): ProcessorWrapper<BackgroundOptions> {
-    return {
-      name: "background-blur",
-      processedTrack,
-      init: vi.fn().mockResolvedValue(undefined),
-      restart: vi.fn().mockResolvedValue(undefined),
-      destroy: vi.fn().mockResolvedValue(undefined),
-    } as unknown as ProcessorWrapper<BackgroundOptions>;
-  }
-
-  it("resolves with the track the wrapper produced", async () => {
-    const wrapper = mockWrapper(processed);
-    const sdkProcessor = fromLivekitProcessor(wrapper);
-    expect(sdkProcessor.name).toBe("background-blur");
-    await expect(sdkProcessor.init(captured)).resolves.toBe(processed);
-    expect(wrapper.init).toHaveBeenCalledWith({
-      kind: "video",
-      track: captured,
-    });
-    await expect(sdkProcessor.restart(captured)).resolves.toBe(processed);
-    await sdkProcessor.destroy();
-    expect(wrapper.destroy).toHaveBeenCalled();
-  });
-
-  it("rejects where the wrapper produced no track", async () => {
-    await expect(
-      fromLivekitProcessor(mockWrapper(undefined)).init(captured),
-    ).rejects.toThrow("produced no track");
-  });
-});
-
 describe("BackgroundEffectsProvider", () => {
   /** A camera track that holds whatever processor it is given. */
   function cameraTrack(): LocalVideoTrack {
@@ -263,10 +223,10 @@ describe("BackgroundEffectsProvider", () => {
 
     const pipeline = latest().processor;
     expect(preview.setProcessor).toHaveBeenCalledWith(
-      convertToLivekitProcessor(pipeline!),
+      toLivekitProcessor(pipeline!),
     );
     expect(call.setProcessor).toHaveBeenCalledWith(
-      convertToLivekitProcessor(pipeline!),
+      toLivekitProcessor(pipeline!),
     );
     expect(pipelines.built).toBe(1);
   });
@@ -286,7 +246,7 @@ describe("BackgroundEffectsProvider", () => {
     });
     expect(track.setProcessor).toHaveBeenCalledTimes(1);
     expect(track.setProcessor).toHaveBeenCalledWith(
-      convertToLivekitProcessor(latest().processor!),
+      toLivekitProcessor(latest().processor!),
     );
     expect(pipelines.built).toBe(1);
     expect(pipelines.switches).toHaveLength(1);

src/tracks/BackgroundEffectsContext.tsx

@@ -6,16 +6,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import {
-  type ProcessorWrapper,
-  type BackgroundOptions,
-} from "@livekit/track-processors";
 import { createContext, type FC, type JSX, use, useEffect } from "react";
-import { type LocalVideoTrack, Track } from "livekit-client";
+import { type LocalVideoTrack } from "livekit-client";
 import { logger } from "matrix-js-sdk/lib/logger";
 import { combineLatest } from "rxjs";
 import {
-  convertToLivekitProcessor,
+  toLivekitProcessor,
   type VideoProcessor,
 } from "@element-hq/matrixrtc-sdk";
 
@@ -27,6 +23,7 @@ import { type Behavior } from "../state/Behavior";
 import { type ObservableScope } from "../state/ObservableScope";
 import {
   type BackgroundEffects,
+  type BackgroundEffectsState,
   createBackgroundEffects,
 } from "../state/BackgroundEffects";
 import { useBehavior } from "../useBehavior";
@@ -35,38 +32,6 @@ import { useBehavior } from "../useBehavior";
 // it is a combination of exposing observable and react hooks.
 // preferably we should not make this a context anymore and instead just a vm?
 
-export type BackgroundEffectsState = {
-  supported: boolean | undefined;
-  processor: undefined | VideoProcessor;
-};
-
-/**
- * This allows to use a LiveKit processor for the matrixRTCSDK (VideoProcessor).
- * They have slightly different apis so we need to convert them with this
- * function before we can use them in the matrixRTCSDK.
- */
-export function fromLivekitProcessor(
-  wrapper: ProcessorWrapper<BackgroundOptions>,
-): VideoProcessor {
-  const processedTrack = (): MediaStreamTrack => {
-    if (wrapper.processedTrack === undefined)
-      throw new Error(`Processor ${wrapper.name} produced no track`);
-    return wrapper.processedTrack;
-  };
-  return {
-    name: wrapper.name,
-    init: async (track: MediaStreamTrack, element?: HTMLMediaElement) => {
-      await wrapper.init({ kind: Track.Kind.Video, track, element });
-      return processedTrack();
-    },
-    restart: async (track: MediaStreamTrack, element?: HTMLMediaElement) => {
-      await wrapper.restart({ kind: Track.Kind.Video, track, element });
-      return processedTrack();
-    },
-    destroy: async () => wrapper.destroy(),
-  };
-}
-
 const BackgroundEffectsContext = createContext<BackgroundEffects | undefined>(
   undefined,
 );
@@ -102,7 +67,7 @@ export function applyProcessor(
       return;
     }
     const track = videoTrack.mediaStreamTrack;
-    videoTrack.setProcessor(convertToLivekitProcessor(processor)).catch((e) => {
+    videoTrack.setProcessor(toLivekitProcessor(processor)).catch((e) => {
       if (track.readyState === "ended")
         logger.debug("Video processor not attached: the track ended first");
       else logger.warn("Failed to attach video processor", e);