diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index f47c59cba..fd0793662 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -178,6 +178,46 @@ test.describe("background effects section", () => { }); }); +test.describe("a pipeline that fails to build", () => { + test("offers no effects for the session, and keeps the choice", async ({ + page, + }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Failing user", "Pipeline fails"); + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + await page.route("**/*.tflite*", async (route) => route.abort()); + + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + await expect( + page.getByText("Background effects are not supported on this platform."), + ).toBeVisible({ timeout: 30_000 }); + await expect(tile).toHaveAttribute("aria-disabled", "true"); + await page.keyboard.press("Escape"); + // The camera and microphone carry on without it. + expect(distance(await cameraColour(preview), camera)).toBeLessThan(30); + await expect(page.getByTestId("incall_mute")).toHaveAttribute( + "aria-checked", + "true", + ); + + await page.unroute("**/*.tflite*"); + await page.reload(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + await expect(tile).toHaveAttribute("aria-checked", "true"); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); +}); + test.describe("the chosen effect", () => { test("chosen effect survives leaving and rejoining", async ({ page }) => { test.setTimeout(120_000); diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index b3740cb8f..4a6bd4d74 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -15,7 +15,7 @@ import { mockMuteStates, mockMediaDevices, } from "../utils/test"; -import { constant } from "../state/Behavior"; +import { type Behavior, constant } from "../state/Behavior"; import type { CallViewModel } from "../state/CallViewModel/CallViewModel"; import type { Alignment, Layout } from "../state/layout-types"; import type { SpotlightTileViewModel } from "../state/TileViewModel"; @@ -122,6 +122,11 @@ const noPipeline = constant({ processor: undefined, }); +/** The provider's verdict, which the SDK's check stands for here. */ +function pipelineAsTheSdkSays(): Behavior { + return constant({ supported: sdkSupportMock(), processor: undefined }); +} + const twoMicsAndOneCamMediaDevices = mockMediaDevices({ audioInput: { available$: constant( @@ -284,7 +289,7 @@ describe("createCallFooterViewModel", () => { testScope(), mockMuteStates(), twoMicsAndOneCamMediaDevices, - noPipeline, + pipelineAsTheSdkSays(), /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, @@ -388,6 +393,31 @@ describe("createCallFooterViewModel", () => { expect(vm.backgroundEffectSettling$.value).toBe(false); }); + it("offers nothing once the pipeline fails to build, keeping the choice", () => { + platformMock.mockReturnValue("desktop"); + backgroundEffectSetting.setValue("blur"); + const pipeline$ = new BehaviorSubject({ + supported: true, + processor: undefined, + }); + const vm = createLobbyFooterViewModel( + testScope(), + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + pipeline$, + /* openSettings */ undefined, + /* hangup */ undefined, + /* showLogo */ false, + ); + expect(vm.backgroundEffect$.value).toBe("blur"); + + pipeline$.next({ supported: false, processor: undefined }); + expect(vm.selectBackgroundEffect$.value).toBeUndefined(); + expect(vm.addBackgroundImage$.value).toBeUndefined(); + expect(vm.backgroundEffect$.value).toBe("none"); + expect(backgroundEffectSetting.getValue()).toBe("blur"); + }); + it("adding is unavailable at the limit", () => { sdkSupportMock.mockReturnValue(true); const added$ = addedBackgrounds.added$ as BehaviorSubject< @@ -480,7 +510,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - noPipeline, + pipelineAsTheSdkSays(), /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index a750f5401..7d66d4c72 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -17,10 +17,7 @@ import { import { supportsAudioOutputSelection } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; -import { - supportsBackgroundProcessors, - usesFallbackProcessing, -} from "../livekit/backgroundProcessing"; +import { usesFallbackProcessing } from "../livekit/backgroundProcessing"; import { parseEffect, serializeEffect, @@ -130,10 +127,14 @@ function buildDeviceBehaviors( ), ); - const supported = supportsBackgroundProcessors(); + // Also false once the pipeline has failed to build this session. + const supported$ = processorState$.pipe( + map(({ supported }) => supported === true), + distinctUntilChanged(), + ); const slow = usesFallbackProcessing(); - const offered$ = disableSwitcher$.pipe( - map((switcherDisabled) => !switcherDisabled && supported), + const offered$ = combineLatest([disableSwitcher$, supported$]).pipe( + map(([switcherDisabled, supported]) => !switcherDisabled && supported), ); // A new object for each refusal, so the same one twice is shown twice. const refusal$ = new BehaviorSubject( @@ -164,8 +165,10 @@ function buildDeviceBehaviors( ), selectVideoButtonOption$: constant(mediaDevices.videoInput.select), backgroundEffect$: scope.behavior( - backgroundEffectSetting.value$.pipe( - map((raw) => (supported ? serializeEffect(parseEffect(raw)) : "none")), + combineLatest([backgroundEffectSetting.value$, supported$]).pipe( + map(([raw, supported]) => + supported ? serializeEffect(parseEffect(raw)) : "none", + ), ), ), selectBackgroundEffect$: scope.behavior( diff --git a/src/livekit/BackgroundEffectTransformer.test.ts b/src/livekit/BackgroundEffectTransformer.test.ts index 400a50416..9d864f11c 100644 --- a/src/livekit/BackgroundEffectTransformer.test.ts +++ b/src/livekit/BackgroundEffectTransformer.test.ts @@ -13,7 +13,10 @@ import { } from "@livekit/track-processors"; import { ImageSegmenter } from "@mediapipe/tasks-vision"; -import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; +import { + BackgroundEffectTransformer, + canSegment, +} from "./BackgroundEffectTransformer"; const frame = {} as VideoFrame; const controller = {} as TransformStreamDefaultController; @@ -78,4 +81,34 @@ describe("BackgroundEffectTransformer", () => { expect(transformer.options.blurRadius).toBeUndefined(); expect(transformer.options.imagePath).toBeUndefined(); }); + + it("segments on the CPU where the GPU refuses", async () => { + vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue(); + const build = vi + .spyOn(ImageSegmenter, "createFromOptions") + .mockRejectedValueOnce(new Error("blocklisted")) + .mockResolvedValue({} as ImageSegmenter); + + await new BackgroundEffectTransformer({}).init( + {} as VideoTransformerInitOptions, + ); + expect(build.mock.calls.map(([, o]) => o.baseOptions?.delegate)).toEqual([ + "GPU", + "CPU", + ]); + }); + + it("says whether a segmenter can be built at all", async () => { + vi.stubGlobal("OffscreenCanvas", class {}); + const close = vi.fn(); + const build = vi + .spyOn(ImageSegmenter, "createFromOptions") + .mockResolvedValue({ close } as unknown as ImageSegmenter); + expect(await canSegment()).toBe(true); + expect(close).toHaveBeenCalled(); + + build.mockRejectedValue(new Error("no model")); + expect(await canSegment()).toBe(false); + vi.unstubAllGlobals(); + }); }); diff --git a/src/livekit/BackgroundEffectTransformer.ts b/src/livekit/BackgroundEffectTransformer.ts index ce0a1e5e6..a2231ba3e 100644 --- a/src/livekit/BackgroundEffectTransformer.ts +++ b/src/livekit/BackgroundEffectTransformer.ts @@ -11,6 +11,7 @@ import { VideoTransformer, type VideoTransformerInitOptions, type BackgroundOptions, + type SegmenterOptions, } from "@livekit/track-processors"; import { logger } from "matrix-js-sdk/lib/logger"; import { ImageSegmenter } from "@mediapipe/tasks-vision"; @@ -89,17 +90,10 @@ export class BackgroundEffectTransformer extends BackgroundTransformer { inputElement: inputVideo, }); - this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, { - baseOptions: { - modelAssetPath, - delegate: "GPU", - ...this.options.segmenterOptions, - }, - canvas: this.canvas, - runningMode: "VIDEO", - outputCategoryMask: true, - outputConfidenceMasks: false, - }); + this.imageSegmenter = await createSegmenter( + this.canvas, + this.options.segmenterOptions, + ); // BackgroundTransformer's own init applies these, and this one replaces it. if (this.options.imagePath) { @@ -113,3 +107,38 @@ export class BackgroundEffectTransformer extends BackgroundTransformer { this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false); } } + +/** + * Whether a segmenter can be built here, asked once before any camera depends + * on one: a pipeline that fails to build takes the camera down with it. + */ +export async function canSegment(): Promise { + try { + (await createSegmenter(new OffscreenCanvas(1, 1))).close(); + return true; + } catch (e) { + logger.warn("Background effects cannot run here", e); + return false; + } +} + +/** On the GPU, or on the CPU where the GPU refuses, as a blocklisted one does. */ +async function createSegmenter( + canvas: HTMLCanvasElement | OffscreenCanvas | undefined, + segmenterOptions?: SegmenterOptions, +): Promise { + const build = async (delegate: "GPU" | "CPU"): Promise => + ImageSegmenter.createFromOptions(wasmFileset, { + baseOptions: { modelAssetPath, delegate, ...segmenterOptions }, + canvas, + runningMode: "VIDEO", + outputCategoryMask: true, + outputConfidenceMasks: false, + }); + try { + return await build("GPU"); + } catch (e) { + logger.warn("Segmenting on the CPU, as the GPU refused", e); + return build("CPU"); + } +} diff --git a/src/livekit/TrackProcessorContext.test.ts b/src/livekit/TrackProcessorContext.test.ts index a74f6e1e0..79cf0ee48 100644 --- a/src/livekit/TrackProcessorContext.test.ts +++ b/src/livekit/TrackProcessorContext.test.ts @@ -63,7 +63,8 @@ vi.mock("./backgroundImages", async () => { }; }); vi.mock("./BackgroundEffectTransformer", () => ({ - BackgroundEffectTransformer: vi.fn(), + BackgroundEffectTransformer: vi.fn(function () {}), + canSegment: async (): Promise => Promise.resolve(true), })); const processor = {} as ProcessorWrapper; diff --git a/src/livekit/TrackProcessorContext.tsx b/src/livekit/TrackProcessorContext.tsx index 0f049666c..593390f6e 100644 --- a/src/livekit/TrackProcessorContext.tsx +++ b/src/livekit/TrackProcessorContext.tsx @@ -30,7 +30,10 @@ import { } from "rxjs"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; -import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; +import { + BackgroundEffectTransformer, + canSegment, +} from "./BackgroundEffectTransformer"; import { type SyncedCameraTrack } from "./cameraTrack"; import { OneStepPipeline } from "./OneStepPipeline"; import { addedBackgrounds } from "./backgroundImages"; @@ -51,6 +54,8 @@ export type ProcessorState = { settling?: boolean; /** The camera this provider's pipeline is synced to. */ cameraTrack?: SyncedCameraTrack; + /** While asking whether the pipeline can be built, before it is attached. */ + preparing?: boolean; }; const ProcessorContext = createContext( @@ -168,6 +173,7 @@ export const ProcessorProvider: FC = ({ children }) => { effect$: backgroundEffectSetting.value$, setEffect: backgroundEffectSetting.setValue, added$: addedBackgrounds.added$, + canSegment, pipeline: new OneStepPipeline(transformer, "background-effect"), transformer, }), diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index 5af080c1c..75de52981 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -9,6 +9,7 @@ import { type FC, useCallback, useMemo, + useRef, useState, type JSX, useEffect, @@ -140,10 +141,15 @@ export const LobbyView: FC = ({ }, ); - const { processor } = useTrackProcessor(); + const { processor, preparing } = useTrackProcessor(); const trackProcessorState$ = useTrackProcessorObservable$(); - const initialProcessor = useInitial(() => processor); + // Taken once it is known whether the pipeline builds, and then kept, as + // useInitial keeps a value: one attached later is synced below. + const initialProcessorRef = useRef<{ value: typeof processor }>(undefined); + if (!preparing) initialProcessorRef.current ??= { value: processor }; + const initialProcessor = initialProcessorRef.current?.value; + const processorTaken = initialProcessorRef.current !== undefined; const localTrackOptions = useMemo( () => ({ // The only reason we request audio here is to get the audio permission @@ -153,12 +159,20 @@ export const LobbyView: FC = ({ // We also pass in a clone because livekit mutates the object passed in, // which would cause the devices to be re-opened on the next render. audio: Object.assign({}, initialAudioOptions), - video: videoEnabled && { - deviceId: videoInputId, - processor: initialProcessor, - }, + // Not opened before then, so that the preview never shows the room. + video: videoEnabled && + processorTaken && { + deviceId: videoInputId, + processor: initialProcessor, + }, }), - [initialAudioOptions, videoEnabled, videoInputId, initialProcessor], + [ + initialAudioOptions, + videoEnabled, + processorTaken, + videoInputId, + initialProcessor, + ], ); const onError = useCallback( diff --git a/src/state/BackgroundEffects.test.ts b/src/state/BackgroundEffects.test.ts index 9e024088e..5df1d5886 100644 --- a/src/state/BackgroundEffects.test.ts +++ b/src/state/BackgroundEffects.test.ts @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { BehaviorSubject, distinctUntilChanged, map } from "rxjs"; +import { BehaviorSubject, distinctUntilChanged, map, of } from "rxjs"; import { type BackgroundProcessorWrapper } from "@livekit/track-processors"; import { @@ -46,8 +46,10 @@ function fakePipeline(): { }; } -/** One letter per state: idle, waiting for a frame, attached. */ +/** One letter per state: idle, preparing, waiting for a frame, attached. */ function letter(state: ProcessorState): string { + if (state.supported === false) return "x"; + if (state.preparing) return "p"; if (state.processor === undefined) return "i"; return state.settling ? "w" : "a"; } @@ -55,14 +57,17 @@ function letter(state: ProcessorState): string { describe("the pipeline's state", () => { function testState({ effect, + answer = "-y", firstFrame = "", expected, }: { effect: string; + /** When the trial build answers, from being asked. */ + answer?: string; firstFrame?: string; expected: string; }): void { - withTestScheduler(({ behavior, schedule, expectObservable }) => { + withTestScheduler(({ behavior, cold, schedule, expectObservable }) => { const transformer = { onFirstFrame: undefined as (() => void) | undefined, }; @@ -71,6 +76,7 @@ describe("the pipeline's state", () => { effect$: behavior(effect, { n: "none", b: "blur" }), setEffect: vi.fn(), added$: new BehaviorSubject([]), + canSegment: () => cold(answer, { y: true, n: false }), pipeline: fakePipeline().pipeline, transformer, }); @@ -83,15 +89,21 @@ describe("the pipeline's state", () => { it("defaults to no effect", () => testState({ effect: "n", expected: "i" })); + it("attaches only once it is known the pipeline builds", () => + testState({ effect: "nb", answer: "--y", expected: "ip-w" })); + + it("offers none where the pipeline fails to build", () => + testState({ effect: "nb", answer: "-n", expected: "ipx" })); + it("waits from the first effect until a frame carries it", () => testState({ - effect: " nb-n-b", - firstFrame: " --f", - expected: " iwa", + effect: " nb---n-b", + firstFrame: "---f", + expected: " ipwa", })); it("attaches on first use and stays attached", () => - testState({ effect: "nb-n", firstFrame: "--f", expected: "iwa" })); + testState({ effect: "nb--n", firstFrame: "---f", expected: "ipwa" })); }); describe("background effects", () => { @@ -108,6 +120,7 @@ describe("background effects", () => { effect$, setEffect, added$, + canSegment: () => of(true), pipeline: fake.pipeline, transformer: { onFirstFrame: undefined }, ...options, @@ -117,8 +130,6 @@ describe("background effects", () => { effect$.next(raw); await flushPromises(); }; - const blur = async (on: boolean): Promise => - choose(on ? "blur" : "none"); beforeEach(() => { effect$ = new BehaviorSubject("none"); @@ -127,6 +138,17 @@ describe("background effects", () => { fake = fakePipeline(); }); + it("offers none for the session where the pipeline fails to build", async () => { + const effects = build({ canSegment: () => of(false) }); + await choose("blur"); + + expect(effects.state$.value.supported).toBe(false); + expect(effects.state$.value.processor).toBeUndefined(); + expect(effects.state$.value.settling).toBe(false); + // A browser update may fix it, so the next session asks again. + expect(effect$.value).toBe("blur"); + }); + it("puts a shipped background on as that picture", async () => { build(); await choose(`image:${shippedBackgrounds[0].id}`); @@ -140,10 +162,10 @@ describe("background effects", () => { it("switches in place rather than reattaching", async () => { const effects = build(); - await blur(true); + await choose("blur"); const pipeline = effects.state$.value.processor; - await blur(false); - await blur(true); + await choose("none"); + await choose("blur"); expect(effects.state$.value.processor).toBe(pipeline); expect(fake.switches).toEqual([ @@ -156,10 +178,10 @@ describe("background effects", () => { it("switches one at a time, skipping those overtaken", async () => { const finished = fake.holdNext(); build(); - await blur(true); - await blur(false); - await blur(true); - await blur(false); + await choose("blur"); + await choose("none"); + await choose("blur"); + await choose("none"); expect(fake.switches).toHaveLength(1); finished(); diff --git a/src/state/BackgroundEffects.ts b/src/state/BackgroundEffects.ts index 929d852cb..36ec5fa5f 100644 --- a/src/state/BackgroundEffects.ts +++ b/src/state/BackgroundEffects.ts @@ -9,9 +9,15 @@ import { combineLatest, distinctUntilChanged, filter, + first, + from, map, Observable, + type ObservableInput, + of, scan, + startWith, + switchMap, } from "rxjs"; import { type BackgroundProcessorWrapper, @@ -41,6 +47,8 @@ export interface BackgroundEffectsOptions { setEffect: (id: EffectId) => void; /** The backgrounds the device keeps, undefined until it has said. */ added$: Behavior; + /** Builds a segmenter on trial, and answers whether it could. */ + canSegment: () => ObservableInput; /** * Shared by the pre-join preview and the call. Building or destroying it is * what primes it, and a primed pipeline lets its next frame through @@ -64,14 +72,17 @@ export class BackgroundEffects { effect$, setEffect, added$, + canSegment, pipeline, transformer, }: BackgroundEffectsOptions, ) { const choice$ = effect$.pipe(map(parseEffect)); - const wanted$ = choice$.pipe( - map((effect) => effect.kind !== "none"), - distinctUntilChanged(), + const wanted$ = scope.behavior( + choice$.pipe( + map((effect) => effect.kind !== "none"), + distinctUntilChanged(), + ), ); combineLatest([choice$, added$]) @@ -87,6 +98,18 @@ export class BackgroundEffects { setEffect("none"); }); + // Asked once an effect is first wanted. A browser update may fix what + // fails, so the choice is kept for the next session rather than cleared. + const buildable$ = scope.behavior( + supported + ? wanted$.pipe( + first(Boolean), + switchMap(() => from(canSegment())), + startWith(undefined), + ) + : of(undefined), + ); + const drewAFrame$ = scope.behavior( new Observable((subscriber) => { subscriber.next(false); @@ -97,19 +120,24 @@ export class BackgroundEffects { }), ); + // In one step, so that nothing sees the pipeline neither preparing nor + // attached. this.state$ = scope.behavior( - combineLatest([wanted$, drewAFrame$]).pipe( - scan<[boolean, boolean], ProcessorState>( - (previous, [wanted, drewAFrame]) => { + combineLatest([wanted$, buildable$, drewAFrame$]).pipe( + scan<[boolean, boolean | undefined, boolean], ProcessorState>( + (previous, [wanted, buildable, drewAFrame]) => { // Attached the first time an effect is wanted and never detached // after, so someone who never turns one on pays for none of it. const attached = - previous.processor !== undefined || (supported && wanted); + previous.processor !== undefined || + (wanted && buildable === true); + const preparing = supported && wanted && buildable === undefined; return { - supported, + supported: supported && buildable !== false, processor: attached ? pipeline : undefined, - settling: attached && !drewAFrame, + settling: (preparing || attached) && !drewAFrame, cameraTrack: this.cameraTrack, + preparing, }; }, { supported, processor: undefined }, diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts index 6884ce0e3..240846f33 100644 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -451,6 +451,18 @@ describe("turning the camera on with an effect chosen", () => { expect(madeWith).toEqual([processor]); }); + it("waits to make a camera track until it is known the pipeline builds", async () => { + state$.next({ supported: true, processor: undefined, preparing: true }); + await publisher.createAndSetupTracks(); + videoEnabled$.next(true); + await flushPromises(); + expect(madeWith).toEqual([]); + + state$.next({ supported: true, processor, preparing: false }); + await flushPromises(); + expect(madeWith).toEqual([processor]); + }); + it("replaces a camera track turned off before the effect was chosen", async () => { state$.next({ supported: true, processor }); const stopped = Object.assign(Object.create(LocalVideoTrack.prototype), { diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index 176e3c2e0..726e61f26 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -14,6 +14,8 @@ import { Track, } from "livekit-client"; import { + filter, + firstValueFrom, map, NEVER, type Observable, @@ -189,14 +191,19 @@ export class Publisher { // We are using the `ParticipantEvent.LocalTrackPublished` to be notified // when tracks are actually published, and at that point // we can pause upstream if needed (depending on if startPublishing has been called). - if (audio && video) { - // Enable both at once in order to have a single permission prompt! - void lkRoom.localParticipant.enableCameraAndMicrophone(); - } else if (audio) { - void lkRoom.localParticipant.setMicrophoneEnabled(true); - } else if (video) { - void lkRoom.localParticipant.setCameraEnabled(true); - } + const enableTracks = (): void => { + if (audio && video) { + // Enable both at once in order to have a single permission prompt! + void lkRoom.localParticipant.enableCameraAndMicrophone(); + } else if (audio) { + void lkRoom.localParticipant.setMicrophoneEnabled(true); + } else if (video) { + void lkRoom.localParticipant.setCameraEnabled(true); + } + }; + if (video && this.trackerProcessorState$.value.preparing) + void this.pipelineKnown().then(enableTracks); + else enableTracks(); return Promise.resolve(); } @@ -412,7 +419,9 @@ export class Publisher { this.muteStates.video.setHandler(async (enable) => { try { this.logger.debug(`handler: Setting LiveKit camera enabled: ${enable}`); - const { processor } = this.trackerProcessorState$.value; + const { processor } = enable + ? await this.pipelineKnown() + : this.trackerProcessorState$.value; if (enable && processor) await this.dropCameraWithoutEffect(lkRoom); await lkRoom.localParticipant.setCameraEnabled(enable); // Unmute will restart the track if it was paused upstream, @@ -441,6 +450,16 @@ export class Publisher { await lkRoom.localParticipant.unpublishTrack(track); } + /** + * The pipeline's state once it is known whether it builds: a camera track + * made before then would start without the effect. + */ + private async pipelineKnown(): Promise { + return firstValueFrom( + this.trackerProcessorState$.pipe(filter((s) => !s.preparing)), + ); + } + private observeTrackProcessors( scope: ObservableScope, room: LivekitRoom,