diff --git a/locales/en/app.json b/locales/en/app.json index c3c2dd990..8cf55a74c 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -3,6 +3,7 @@ "user_menu": "User menu" }, "action": { + "add_background_image": "Add image", "close": "Close", "copy_link": "Copy link", "edit": "Edit", diff --git a/package.json b/package.json index 87de5b211..95fa242fd 100644 --- a/package.json +++ b/package.json @@ -94,6 +94,7 @@ "copy-to-clipboard": "^3.3.3", "eslint-plugin-element-call": "link:eslint", "eslint-plugin-storybook": "^10.3.6", + "fake-indexeddb": "^6.2.5", "fetch-mock": "11.1.5", "i18next": "^26.0.0", "i18next-browser-languagedetector": "^8.0.0", diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index e71ebab7e..db60d7ab9 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -214,6 +214,172 @@ async function chooseAndWear( return { camera, picture }; } +test.describe("a background of one's own", () => { + test("adds an image from the device", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Adding user", "Own background"); + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + const red = await redImage(page); + + await addImage(page, red); + const mine = page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }); + await expect(mine).toBeVisible(); + await expect(mine).toHaveAttribute("aria-checked", "false"); + + await mine.click(); + await expect + .poll(async () => distance(await averageColour(preview), RED), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, RED) / 2); + }); + + test("adding an image makes no upload", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Private user", "Own background kept"); + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + const red = await redImage(page); + const sent: { request: string; body: Buffer | null }[] = []; + page.on("request", (request) => + sent.push({ + request: `${request.method()} ${request.url()}`, + body: request.postDataBuffer(), + }), + ); + + await addImage(page, red); + await page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }) + .click(); + // Kept, chosen and in force: anything that would send it has had its turn. + await expect + .poll(async () => distance(await averageColour(preview), RED), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, RED) / 2); + // The file as given, and the copy kept, which is encoded anew. + const samples = [red, await keptBytes(page)].map((bytes) => + bytes.subarray(bytes.length - 64), + ); + const carriesIt = sent.filter( + ({ request, body }) => + request.includes("/_matrix/media/") || + samples.some((sample) => body?.includes(sample)), + ); + expect(carriesIt.map(({ request }) => request)).toEqual([]); + }); + + test("added image survives rejoining and is gone after storage is cleared", async ({ + page, + }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Keeping user", "Own background again"); + const preview = page.locator("video").first(); + const camera = await cameraColour(preview); + const red = await redImage(page); + await addImage(page, red); + const mine = page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }); + await mine.click(); + await page.keyboard.press("Escape"); + + await page.reload(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + await expect(mine).toHaveAttribute("aria-checked", "true"); + await page.keyboard.press("Escape"); + await expect + .poll(async () => distance(await averageColour(preview), RED), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, RED) / 2); + + await page.evaluate( + async () => + new Promise((resolve, reject) => { + const request = indexedDB.deleteDatabase( + "element-call-background-images", + ); + request.onsuccess = (): void => resolve(); + request.onerror = (): void => reject(request.error); + request.onblocked = (): void => resolve(); + }), + ); + await page.reload(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + await page.getByRole("button", { name: "Camera", exact: true }).click(); + await expect( + page.getByRole("group", { name: "Background effects" }), + ).toBeVisible(); + await expect(mine).toHaveCount(0); + await page.keyboard.press("Escape"); + + await addImage(page, red); + await expect(mine).toBeVisible(); + }); +}); + +/** The bytes of the one image the device keeps. */ +async function keptBytes(page: Page): Promise { + const base64 = await page.evaluate( + async () => + new Promise((resolve, reject) => { + const open = indexedDB.open("element-call-background-images"); + open.onerror = (): void => reject(open.error); + open.onsuccess = (): void => { + const all = open.result + .transaction("backgrounds") + .objectStore("backgrounds") + .getAll(); + all.onerror = (): void => reject(all.error); + all.onsuccess = (): void => { + const reader = new FileReader(); + reader.onload = (): void => + resolve((reader.result as string).split(",")[1]); + reader.readAsDataURL((all.result[0] as { image: Blob }).image); + }; + }; + }), + ); + return Buffer.from(base64, "base64"); +} + +const RED: Colour = [255, 0, 0]; + +/** A solid red PNG, unlike the camera and the shipped pictures. */ +async function redImage(page: Page): Promise { + const dataUrl = await page.evaluate(() => { + const canvas = document.createElement("canvas"); + canvas.width = 320; + canvas.height = 180; + const context = canvas.getContext("2d")!; + context.fillStyle = "rgb(255, 0, 0)"; + context.fillRect(0, 0, canvas.width, canvas.height); + return canvas.toDataURL("image/png"); + }); + return Buffer.from(dataUrl.split(",")[1], "base64"); +} + +/** Adds an image from the camera menu's add tile, leaving the menu open. */ +async function addImage(page: Page, image: Buffer): Promise { + if (!(await page.getByRole("menu").isVisible())) + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const chooser = page.waitForEvent("filechooser"); + await page.getByRole("menuitem", { name: "Add image" }).click(); + await ( + await chooser + ).setFiles({ name: "mine.png", mimeType: "image/png", buffer: image }); +} + test.describe("joining with a background chosen", () => { test("publishes no frame of the room", async ({ browser }) => { // Two first builds of the pipeline, one of them held back. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b5c7c83d3..4e8419a06 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -163,6 +163,9 @@ importers: eslint-plugin-storybook: specifier: ^10.3.6 version: 10.6.0(eslint@8.57.1(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.3) + fake-indexeddb: + specifier: ^6.2.5 + version: 6.2.5 fetch-mock: specifier: 11.1.5 version: 11.1.5(node-fetch@2.7.0) @@ -3998,6 +4001,10 @@ packages: resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} + fake-indexeddb@6.2.5: + resolution: {integrity: sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==} + engines: {node: '>=18'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -9548,6 +9555,8 @@ snapshots: expect-type@1.4.0: {} + fake-indexeddb@6.2.5: {} + fast-deep-equal@3.1.3: {} fast-glob@3.3.3: diff --git a/src/components/BackgroundEffectGrid.tsx b/src/components/BackgroundEffectGrid.tsx index 02744fa41..66fcfb94d 100644 --- a/src/components/BackgroundEffectGrid.tsx +++ b/src/components/BackgroundEffectGrid.tsx @@ -12,6 +12,7 @@ import { BlockIcon, BlurIcon, CheckCircleSolidIcon, + PlusIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import styles from "./BackgroundEffectGrid.module.css"; @@ -38,6 +39,9 @@ interface Props { settling?: boolean; /** Id of the text saying what the user should know before choosing. */ describedBy?: string; + /** Offers to add a background of one's own. Omit to leave that tile out. */ + onAdd?: () => void; + addLabel?: string; } /** @@ -52,6 +56,8 @@ export const BackgroundEffectGrid: FC = ({ onSelect, settling = false, describedBy, + onAdd, + addLabel, }) => { const choose = (id: string): void => { if (id !== selected) onSelect?.(id); @@ -91,11 +97,31 @@ export const BackgroundEffectGrid: FC = ({ ); }); + // A command rather than a choice, so an item beside the radios. + const addTile = + onAdd === undefined ? null : ( + { + e.preventDefault(); + onAdd(); + }} + > + + + + {addLabel} + + ); + return (
{heading}
{tiles} + {addTile}
); diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index b2b355f24..647dd9d7a 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -137,6 +137,7 @@ export const Default: Story = { ], backgroundEffectNotice: undefined, backgroundEffectSettling: false, + addBackgroundImage: fn(), hangup: fn(), buttonSize: "lg", showFooter: true, diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index 0c2a7ce98..03a266728 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -68,6 +68,8 @@ export interface FooterActions { toggleVideo: (() => void) | undefined; /** Undefined where background effects can't be chosen. */ selectBackgroundEffect: ((id: string) => void) | undefined; + /** Undefined where no background can be added. */ + addBackgroundImage: ((file: File) => void) | undefined; toggleScreenSharing: (() => void) | undefined; /** Also controls if the settings button is visible */ openSettings: (() => void) | undefined; @@ -176,6 +178,7 @@ export const CallFooter: FC = ({ const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$); const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$); const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$); + const addBackgroundImage = useBehavior(vm.addBackgroundImage$); const backgroundEffects = useBackgroundEffectLabels( useBehavior(vm.backgroundEffects$), ); @@ -244,6 +247,7 @@ export const CallFooter: FC = ({ selectedBackgroundEffect={backgroundEffect} onSelectBackgroundEffect={selectBackgroundEffect} backgroundEffectSettling={backgroundEffectSettling} + onAddBackgroundImage={addBackgroundImage} backgroundEffectNotice={ backgroundEffectNotice === "unavailable" ? t("background_effects.unavailable") diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index e7d7ddd85..8330b792d 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -21,6 +21,10 @@ import { import { HeaderStyle } from "../UrlParams"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; import { shippedBackgrounds } from "../livekit/backgroundEffects"; +import { + type AddedBackground, + addedBackgrounds, +} from "../livekit/backgroundImages"; const platformMock = vi.hoisted(() => vi.fn(() => "desktop")); vi.mock("../Platform", () => ({ @@ -267,6 +271,7 @@ describe("createCallFooterViewModel", () => { it("offers nothing where the browser itself cannot run them", () => { sdkSupportMock.mockReturnValue(false); expect(lobbyOn("desktop").selectBackgroundEffect$.value).toBeUndefined(); + expect(lobbyOn("desktop").addBackgroundImage$.value).toBeUndefined(); }); it("puts no effect in force where the browser cannot run them", () => { @@ -299,6 +304,10 @@ describe("createCallFooterViewModel", () => { it("offers every effect in order", () => { sdkSupportMock.mockReturnValue(true); + const added$ = addedBackgrounds.added$ as BehaviorSubject< + AddedBackground[] | undefined + >; + added$.next([{ id: "mine", url: "blob:mine" }]); expect(lobbyOn("desktop").backgroundEffects$.value).toEqual([ { id: "none", kind: "none" }, { id: "blur", kind: "blur" }, @@ -307,7 +316,9 @@ describe("createCallFooterViewModel", () => { kind: "image", imageUrl: background.imagePath, })), + { id: "added:mine", kind: "image", imageUrl: "blob:mine" }, ]); + added$.next([]); }); it("says they are unavailable where they cannot be chosen", () => { diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index 372e2cb98..d99ee9b63 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -17,12 +17,17 @@ import { supportsBackgroundProcessors, supportsModernBackgroundProcessors, } from "@livekit/track-processors"; +import { logger } from "matrix-js-sdk/lib/logger"; import { parseEffect, serializeEffect, shippedBackgrounds, } from "../livekit/backgroundEffects"; +import { + type AddedBackground, + addedBackgrounds, +} from "../livekit/backgroundImages"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { type MenuOptions } from "./MediaMuteAndSwitchButton"; @@ -95,6 +100,7 @@ function buildDeviceBehaviors( | "backgroundEffects$" | "backgroundEffectNotice$" | "backgroundEffectSettling$" + | "addBackgroundImage$" > { const options$ = ( available$: Behavior>, @@ -157,7 +163,11 @@ function buildDeviceBehaviors( ), ), ), - backgroundEffects$: constant(backgroundEffectChoices()), + backgroundEffects$: scope.behavior( + addedBackgrounds.added$.pipe( + map((added) => backgroundEffectChoices(added ?? [])), + ), + ), backgroundEffectNotice$: scope.behavior( offered$.pipe( map((offered) => @@ -169,6 +179,22 @@ function buildDeviceBehaviors( backgroundEffectSettling$.pipe(distinctUntilChanged()), false, ), + // Kept and offered, not put on: that waits for the user to choose it. + addBackgroundImage$: scope.behavior( + offered$.pipe( + map((offered) => + offered + ? (file: File): void => { + addedBackgrounds + .add(file) + .catch((e) => + logger.warn("Could not keep that background", e), + ); + } + : undefined, + ), + ), + ), }; } @@ -336,7 +362,9 @@ export function createLobbyFooterViewModel( }; } -function backgroundEffectChoices(): BackgroundEffectChoice[] { +function backgroundEffectChoices( + added: AddedBackground[], +): BackgroundEffectChoice[] { return [ { id: serializeEffect({ kind: "none" }), kind: "none" }, { id: serializeEffect({ kind: "blur" }), kind: "blur" }, @@ -345,5 +373,10 @@ function backgroundEffectChoices(): BackgroundEffectChoice[] { kind: "image" as const, imageUrl: background.imagePath, })), + ...added.map((background) => ({ + id: serializeEffect({ kind: "added", id: background.id }), + kind: "image" as const, + imageUrl: background.url, + })), ]; } diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index 610cef1b3..fd3df86bc 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -195,6 +195,7 @@ export const VideoUnmute: Story = { backgroundEffects: effects, selectedBackgroundEffect: "none", onSelectBackgroundEffect: fn(), + onAddBackgroundImage: fn(), }, }; @@ -238,6 +239,11 @@ export const BackgroundEffects: Story = { true, ); + const add = within(section).getByRole("menuitem", { name: "Add image" }); + await expect( + tiles[3].compareDocumentPosition(add) & Node.DOCUMENT_POSITION_FOLLOWING, + ).toBeTruthy(); + await userEvent.click(menu.getByRole("menuitemradio", { name: "Blur" })); await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith("blur"); await expect(menu.getByRole("menu")).toBeVisible(); diff --git a/src/components/MediaMuteAndSwitchButton.test.tsx b/src/components/MediaMuteAndSwitchButton.test.tsx index 7b7c9b98b..24035d484 100644 --- a/src/components/MediaMuteAndSwitchButton.test.tsx +++ b/src/components/MediaMuteAndSwitchButton.test.tsx @@ -367,6 +367,77 @@ describe("MediaMuteAndSwitchButton", () => { expect(onSelectBackgroundEffect).not.toHaveBeenCalled(); }); + test("added images are chosen like shipped ones", async () => { + const user = userEvent.setup(); + const onSelectBackgroundEffect = vi.fn(); + const { getByRole } = renderComponent( + , + ); + + await user.click(getByRole("button", { name: "Camera" })); + const section = screen.getByRole("group", { name: "Background effects" }); + await user.click( + within(section).getByRole("menuitemradio", { name: "Background 3" }), + ); + expect(onSelectBackgroundEffect).toHaveBeenCalledWith("added:mine"); + }); + + test("adds the file chosen from the add tile, and stays open", async () => { + const user = userEvent.setup(); + const onAddBackgroundImage = vi.fn(); + const pick = vi + .spyOn(HTMLInputElement.prototype, "click") + .mockImplementation(() => {}); + try { + const { getByRole, container } = renderComponent( + , + ); + + await user.click(getByRole("button", { name: "Camera" })); + await user.click(screen.getByRole("menuitem", { name: "Add image" })); + expect(pick).toHaveBeenCalled(); + // The picker takes the focus, which would otherwise close the menu. + await user.keyboard("{Escape}"); + expect(screen.getByRole("menu")).toBeInTheDocument(); + + const file = new File(["x"], "mine.png", { type: "image/png" }); + // What the picker hands back; the open menu blocks pointer events. + fireEvent.change( + container.querySelector("input[type=file]")!, + { target: { files: [file] } }, + ); + expect(onAddBackgroundImage).toHaveBeenCalledWith(file); + screen.getByRole("menuitemradio", { name: "None", checked: true }); + } finally { + pick.mockRestore(); + } + }); + test("offers the background effects in a phone's drawer", async () => { const userAgent = vi .spyOn(navigator, "userAgent", "get") diff --git a/src/components/MediaMuteAndSwitchButton.tsx b/src/components/MediaMuteAndSwitchButton.tsx index 919202e55..681f878e0 100644 --- a/src/components/MediaMuteAndSwitchButton.tsx +++ b/src/components/MediaMuteAndSwitchButton.tsx @@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details. import { useCallback, useId, + useRef, useState, type CSSProperties, type FC, @@ -79,6 +80,8 @@ export interface MediaMuteAndSwitchButtonProps { backgroundEffectNotice?: string; /** Whether the first effect chosen is still being prepared. */ backgroundEffectSettling?: boolean; + /** Called with the file chosen from the add tile. Omit to leave it out. */ + onAddBackgroundImage?: (file: File) => void; /** * For any toggle and option this method will be called. * So toggles need to be implemented by listening here and setting the right toggle item to `enabled` @@ -113,6 +116,7 @@ export const MediaMuteAndSwitchButton: FC = ({ onSelectBackgroundEffect, backgroundEffectNotice, backgroundEffectSettling, + onAddBackgroundImage, onSelect, }) => { // Requested but not yet selected. Keyed by kind too, since Chrome uses @@ -122,11 +126,29 @@ export const MediaMuteAndSwitchButton: FC = ({ id: string; } | null>(null); const [menuOpen, setMenuOpen] = useState(false); + // The file picker takes the focus, which the menu reads as a click + // elsewhere; it is held open until the picker is done. + const choosingFile = useRef(false); + const fileInput = useRef(null); const onOpenChange = useCallback((open: boolean): void => { + if (!open && choosingFile.current) return; setMenuOpen(open); // Drop a request that never arrived. if (!open) setPlannedSelection(null); }, []); + const watchFileInput = useCallback( + (input: HTMLInputElement | null): (() => void) | undefined => { + fileInput.current = input; + if (input === null) return; + // Dismissing the picker fires cancel, which React doesn't type. + const done = (): void => { + choosingFile.current = false; + }; + input.addEventListener("cancel", done); + return (): void => input.removeEventListener("cancel", done); + }, + [], + ); const isBusy = busy ?? false; const { t } = useTranslation(); const devices = useMediaDevices(); @@ -328,6 +350,21 @@ export const MediaMuteAndSwitchButton: FC = ({ > {/* The mute button lives inside */} {button} + {onAddBackgroundImage !== undefined && ( + { + const file = e.target.files?.[0]; + // Cleared, so the same file can be chosen twice in a row. + e.target.value = ""; + choosingFile.current = false; + if (file) onAddBackgroundImage(file); + }} + /> + )} = ({ describedBy={ backgroundEffectNotice === undefined ? undefined : noticeId } + onAdd={ + onAddBackgroundImage === undefined + ? undefined + : (): void => { + choosingFile.current = true; + fileInput.current?.click(); + } + } + addLabel={t("action.add_background_image")} /> )} {/* In the list, so the menu grows no taller for it. */} diff --git a/src/livekit/BackgroundEffectsContext.test.ts b/src/livekit/BackgroundEffectsContext.test.ts index b6d48afaf..c14a66ba6 100644 --- a/src/livekit/BackgroundEffectsContext.test.ts +++ b/src/livekit/BackgroundEffectsContext.test.ts @@ -23,6 +23,7 @@ import { useSyncBackgroundEffects, } from "./BackgroundEffectsContext"; import { backgroundEffect } from "../settings/settings"; +import { type AddedBackground } from "./backgroundImages"; import { constant } from "../state/Behavior"; import { flushPromises, testScope } from "../utils/test"; @@ -51,6 +52,14 @@ vi.mock("@livekit/track-processors", () => ({ })); const platformMock = vi.hoisted(() => ({ platform: "desktop" })); vi.mock("../Platform", () => platformMock); +vi.mock("./backgroundImages", async () => { + const { BehaviorSubject } = await import("rxjs"); + return { + addedBackgrounds: { + added$: new BehaviorSubject(undefined), + }, + }; +}); vi.mock("./BackgroundEffectTransformer", () => ({ BackgroundEffectTransformer: vi.fn(), })); diff --git a/src/livekit/BackgroundEffectsContext.tsx b/src/livekit/BackgroundEffectsContext.tsx index cb84dd4d1..615d23904 100644 --- a/src/livekit/BackgroundEffectsContext.tsx +++ b/src/livekit/BackgroundEffectsContext.tsx @@ -26,6 +26,7 @@ import { combineLatest } from "rxjs"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; import { OneStepPipeline } from "./OneStepPipeline"; +import { addedBackgrounds } from "./backgroundImages"; import { type Behavior } from "../state/Behavior"; import { ObservableScope } from "../state/ObservableScope"; import { @@ -136,6 +137,7 @@ export const BackgroundEffectsProvider: FC = ({ children }) => { supported: supportsBackgroundProcessors(), effect$: backgroundEffectSetting.value$, setEffect: backgroundEffectSetting.setValue, + added$: addedBackgrounds.added$, pipeline: new OneStepPipeline(transformer, "background-effect"), transformer, }), diff --git a/src/livekit/backgroundEffects.test.ts b/src/livekit/backgroundEffects.test.ts index cac6901cc..c2f02adf9 100644 --- a/src/livekit/backgroundEffects.test.ts +++ b/src/livekit/backgroundEffects.test.ts @@ -22,6 +22,7 @@ describe("the chosen background effect", () => { { kind: "none" }, { kind: "blur" }, ...shippedBackgrounds.map(({ id }) => ({ kind: "shipped" as const, id })), + { kind: "added", id: "d3b07384" }, ]; for (const effect of effects) expect(parseEffect(serializeEffect(effect))).toEqual(effect); @@ -30,6 +31,7 @@ describe("the chosen background effect", () => { test("falls back to no effect when the stored form names nothing we ship", () => { expect(parseEffect("image:gone")).toEqual({ kind: "none" }); expect(parseEffect("")).toEqual({ kind: "none" }); + expect(parseEffect("added:")).toEqual({ kind: "none" }); }); test("reads a stored value that is not a string as no effect", () => { diff --git a/src/livekit/backgroundEffects.ts b/src/livekit/backgroundEffects.ts index 467c90d08..51de44b5f 100644 --- a/src/livekit/backgroundEffects.ts +++ b/src/livekit/backgroundEffects.ts @@ -24,16 +24,18 @@ export const shippedBackgrounds: ShippedBackground[] = [ export type BackgroundEffect = | { kind: "none" } | { kind: "blur" } - | { kind: "shipped"; id: string }; + | { kind: "shipped"; id: string } + | { kind: "added"; id: string }; /** An effect as the setting stores it, and as the menu names its option. */ -export type EffectId = "none" | "blur" | `image:${string}`; +export type EffectId = "none" | "blur" | `image:${string}` | `added:${string}`; export const noEffect: BackgroundEffect = { kind: "none" }; /** - * Reads a stored effect; one we no longer ship reads as no effect. Stored by - * hand or by another build, the value need not even be a string. + * Reads a stored effect; one we no longer ship reads as no effect. Whether an + * added one is still kept only the device knows. Stored by hand or by another + * build, the value need not even be a string. */ export function parseEffect(raw: unknown): BackgroundEffect { if (typeof raw !== "string") return noEffect; @@ -43,6 +45,10 @@ export function parseEffect(raw: unknown): BackgroundEffect { if (shippedBackgrounds.some((b) => b.id === id)) return { kind: "shipped", id }; } + if (raw.startsWith("added:")) { + const id = raw.slice("added:".length); + if (id) return { kind: "added", id }; + } return noEffect; } @@ -52,6 +58,8 @@ export function serializeEffect(effect: BackgroundEffect): EffectId { return "blur"; case "shipped": return `image:${effect.id}`; + case "added": + return `added:${effect.id}`; default: return "none"; } diff --git a/src/livekit/backgroundImages.test.ts b/src/livekit/backgroundImages.test.ts new file mode 100644 index 000000000..252a879b6 --- /dev/null +++ b/src/livekit/backgroundImages.test.ts @@ -0,0 +1,95 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { IDBFactory } from "fake-indexeddb"; + +import { + AddedBackgrounds, + type BackgroundImageStorage, + IndexedDBImageStorage, + type KeptImage, +} from "./backgroundImages"; +import { flushPromises } from "../utils/test"; + +describe("AddedBackgrounds", () => { + beforeEach(() => { + let n = 0; + vi.spyOn(URL, "createObjectURL").mockImplementation(() => `blob:${++n}`); + }); + + it("offers what the device keeps, oldest first", async () => { + const image = new Blob(["x"], { type: "image/png" }); + const store = new AddedBackgrounds( + memoryStorage([ + { id: "later", image, addedAt: 2 }, + { id: "earlier", image, addedAt: 1 }, + ]), + ); + expect(store.added$.value).toBeUndefined(); + await flushPromises(); + expect(store.added$.value?.map(({ id }) => id)).toEqual([ + "earlier", + "later", + ]); + }); + + it("keeps an added image and offers it after the others", async () => { + const storage = memoryStorage(); + const store = new AddedBackgrounds(storage); + await flushPromises(); + const id = await store.add(new Blob(["x"], { type: "image/png" })); + expect(store.added$.value?.map((a) => a.id)).toEqual([id]); + expect((await storage.list()).map((k) => k.id)).toEqual([id]); + }); + + it("offers nothing where the browser keeps nothing", async () => { + const store = new AddedBackgrounds(null); + await flushPromises(); + expect(store.added$.value).toEqual([]); + await expect(store.add(new Blob(["x"]))).rejects.toThrow(); + }); + + it("offers none, rather than waiting, where the device can't be read", async () => { + const unreadable: BackgroundImageStorage = { + ...memoryStorage(), + list: async () => Promise.reject(new Error("storage cleared")), + }; + const store = new AddedBackgrounds(unreadable); + await flushPromises(); + expect(store.added$.value).toEqual([]); + }); +}); + +describe("IndexedDBImageStorage", () => { + it("keeps images across sessions", async () => { + const indexedDB = new IDBFactory(); + const image = new Blob(["x"], { type: "image/webp" }); + const first = new IndexedDBImageStorage(indexedDB); + await first.put({ id: "a", image, addedAt: 1 }); + await first.put({ id: "b", image, addedAt: 2 }); + + // A session of its own, as after a reload. + const second = new IndexedDBImageStorage(indexedDB); + expect( + (await second.list()).map(({ id, addedAt }) => [id, addedAt]), + ).toEqual([ + ["a", 1], + ["b", 2], + ]); + }); +}); + +function memoryStorage(kept: KeptImage[] = []): BackgroundImageStorage { + return { + list: async (): Promise => Promise.resolve([...kept]), + put: async (image): Promise => { + kept.push(image); + return Promise.resolve(); + }, + }; +} diff --git a/src/livekit/backgroundImages.ts b/src/livekit/backgroundImages.ts new file mode 100644 index 000000000..3465c3a24 --- /dev/null +++ b/src/livekit/backgroundImages.ts @@ -0,0 +1,126 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { BehaviorSubject } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { type Behavior } from "../state/Behavior"; + +/** A background the user added, as the menu and the pipeline use it. */ +export interface AddedBackground { + id: string; + url: string; +} + +/** A background the user added, as it is kept on the device. */ +export interface KeptImage { + id: string; + image: Blob; + addedAt: number; +} + +/** Where added backgrounds are kept. */ +export interface BackgroundImageStorage { + list(): Promise; + put(image: KeptImage): Promise; +} + +/** The backgrounds this device keeps, which never leave it. */ +export class AddedBackgrounds { + private readonly subject$ = new BehaviorSubject< + AddedBackground[] | undefined + >(undefined); + /** Undefined until what the device keeps has been read. */ + public readonly added$: Behavior = + this.subject$; + + public constructor(private readonly storage: BackgroundImageStorage | null) { + this.read().catch((e) => { + logger.warn("Could not read added backgrounds", e); + this.subject$.next([]); + }); + } + + private async read(): Promise { + const kept = (await this.storage?.list()) ?? []; + kept.sort((a, b) => a.addedAt - b.addedAt); + this.subject$.next( + kept.map(({ id, image }) => ({ id, url: URL.createObjectURL(image) })), + ); + } + + /** Keeps a file as a background, and answers with its id. */ + public async add(file: Blob): Promise { + if (this.storage === null) + throw new Error("This browser keeps no backgrounds"); + const kept: KeptImage = { + id: crypto.randomUUID(), + image: file, + addedAt: Date.now(), + }; + await this.storage.put(kept); + this.subject$.next([ + ...(this.subject$.value ?? []), + { id: kept.id, url: URL.createObjectURL(file) }, + ]); + return kept.id; + } +} + +const DB_NAME = "element-call-background-images"; +const STORE = "backgrounds"; + +export class IndexedDBImageStorage implements BackgroundImageStorage { + private db?: Promise; + + public constructor(private readonly indexedDB: IDBFactory) {} + + private async open(): Promise { + return (this.db ??= new Promise((resolve, reject) => { + const request = this.indexedDB.open(DB_NAME, 1); + request.onupgradeneeded = (): void => { + request.result.createObjectStore(STORE, { keyPath: "id" }); + }; + request.onsuccess = (): void => resolve(request.result); + request.onerror = (): void => reject(request.error); + })); + } + + private async run( + mode: IDBTransactionMode, + body: (store: IDBObjectStore) => IDBRequest, + ): Promise { + const db = await this.open(); + return new Promise((resolve, reject) => { + const request = body(db.transaction(STORE, mode).objectStore(STORE)); + request.onsuccess = (): void => resolve(request.result); + request.onerror = (): void => reject(request.error); + }); + } + + public async list(): Promise { + return this.run("readonly", (s) => s.getAll()); + } + + public async put(image: KeptImage): Promise { + await this.run("readwrite", (s) => s.put(image)); + } +} + +function openStorage(): BackgroundImageStorage | null { + // Merely reading indexedDB throws where it is disabled. + try { + return globalThis.indexedDB + ? new IndexedDBImageStorage(globalThis.indexedDB) + : null; + } catch (e) { + logger.warn("Could not reach IndexedDB", e); + return null; + } +} + +export const addedBackgrounds = new AddedBackgrounds(openStorage()); diff --git a/src/state/BackgroundEffects.test.ts b/src/state/BackgroundEffects.test.ts index 9927b96f9..7a8377345 100644 --- a/src/state/BackgroundEffects.test.ts +++ b/src/state/BackgroundEffects.test.ts @@ -15,6 +15,7 @@ import { type BackgroundEffectsOptions, } from "./BackgroundEffects"; import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext"; +import { type AddedBackground } from "../livekit/backgroundImages"; import { shippedBackgrounds } from "../livekit/backgroundEffects"; import { flushPromises, testScope, withTestScheduler } from "../utils/test"; @@ -70,6 +71,7 @@ describe("the pipeline's state", () => { supported: true, effect$: behavior(effect, { n: "none", b: "blur" }), setEffect: vi.fn(), + added$: new BehaviorSubject([]), pipeline: fakePipeline().pipeline, transformer, }); @@ -95,6 +97,7 @@ describe("the pipeline's state", () => { describe("background effects", () => { let effect$: BehaviorSubject; + let added$: BehaviorSubject; let setEffect: (raw: string) => void; let fake: ReturnType; @@ -105,6 +108,7 @@ describe("background effects", () => { supported: true, effect$, setEffect, + added$, pipeline: fake.pipeline, transformer: { onFirstFrame: undefined }, ...options, @@ -119,6 +123,7 @@ describe("background effects", () => { beforeEach(() => { effect$ = new BehaviorSubject("none"); + added$ = new BehaviorSubject(undefined); setEffect = vi.fn((raw: string) => effect$.next(raw)); fake = fakePipeline(); }); @@ -172,4 +177,56 @@ describe("background effects", () => { expect(effect$.value).toBe("none"); expect(effects.state$.value.processor).toBeUndefined(); }); + + it("puts an added background on as its picture", async () => { + added$.next([{ id: "mine", url: "blob:mine" }]); + build(); + await choose("added:mine"); + expect(fake.switches).toEqual([ + { mode: "virtual-background", imagePath: "blob:mine" }, + ]); + }); + + it("switches between any two effects in place", async () => { + added$.next([{ id: "mine", url: "blob:mine" }]); + const shipped = shippedBackgrounds[0]; + const options: Record = { + none: { mode: "disabled" }, + blur: { mode: "background-blur", blurRadius: 15 }, + [`image:${shipped.id}`]: { + mode: "virtual-background", + imagePath: shipped.imagePath, + }, + "added:mine": { mode: "virtual-background", imagePath: "blob:mine" }, + }; + // Each kind to each other kind, and back. + const order = [ + "blur", + `image:${shipped.id}`, + "added:mine", + "blur", + "added:mine", + `image:${shipped.id}`, + "none", + "added:mine", + "none", + "blur", + ]; + const effects = build(); + await choose(order[0]); + const pipeline = effects.state$.value.processor; + for (const raw of order.slice(1)) await choose(raw); + + expect(effects.state$.value.processor).toBe(pipeline); + expect(fake.switches).toEqual(order.map((raw) => options[raw])); + }); + + it("forgets an added background the device no longer keeps", () => { + effect$.next("added:gone"); + build(); + // Not before the device has said what it keeps. + expect(effect$.value).toBe("added:gone"); + added$.next([]); + expect(effect$.value).toBe("none"); + }); }); diff --git a/src/state/BackgroundEffects.ts b/src/state/BackgroundEffects.ts index 5fd376b68..d9e7fb831 100644 --- a/src/state/BackgroundEffects.ts +++ b/src/state/BackgroundEffects.ts @@ -30,6 +30,7 @@ import { imagePathFor, parseEffect, } from "../livekit/backgroundEffects"; +import { type AddedBackground } from "../livekit/backgroundImages"; export interface BackgroundEffectsOptions { /** Whether this browser can run a pipeline at all. */ @@ -38,6 +39,8 @@ export interface BackgroundEffectsOptions { effect$: Behavior; /** Stores a choice, to forget one that can't be honoured. */ setEffect: (id: EffectId) => void; + /** The backgrounds the device keeps, undefined until it has said. */ + added$: Behavior; /** The background processor pipeline to be switched between effects. */ pipeline: BackgroundProcessorWrapper; /** Tells, once, that a frame carrying an effect has been drawn. */ @@ -56,6 +59,7 @@ export function createBackgroundEffects( supported, effect$, setEffect, + added$, pipeline, transformer, }: BackgroundEffectsOptions, @@ -66,10 +70,18 @@ export function createBackgroundEffects( distinctUntilChanged(), ); - choice$.pipe(scope.bind()).subscribe((effect) => { - if (effect.kind === "none") return; - if (!supported) setEffect("none"); - }); + combineLatest([choice$, added$]) + .pipe(scope.bind()) + .subscribe(([effect, added]) => { + if (effect.kind === "none") return; + if ( + !supported || + (effect.kind === "added" && + added !== undefined && + !added.some(({ id }) => id === effect.id)) + ) + setEffect("none"); + }); const drewAFrame$ = scope.behavior( new Observable((subscriber) => { @@ -101,10 +113,17 @@ export function createBackgroundEffects( ); const switchTo = oneSwitchAtATime(pipeline); - combineLatest([state$, choice$]) + combineLatest([state$, choice$, added$]) .pipe( filter(([{ processor }]) => processor !== undefined), - map(([, effect]) => switchOptionsFor(effect)), + map(([, effect, added]) => + switchOptionsFor( + effect, + effect.kind === "added" + ? added?.find(({ id }) => id === effect.id)?.url + : undefined, + ), + ), distinctUntilChanged(deepCompare), scope.bind(), ) @@ -137,18 +156,24 @@ function oneSwitchAtATime( }; } +/** `addedUrl` is the picture of the added background chosen, if still kept. */ function switchOptionsFor( effect: BackgroundEffect, + addedUrl: string | undefined, ): SwitchBackgroundProcessorOptions { + const withImage = ( + imagePath: string | undefined, + ): SwitchBackgroundProcessorOptions => + imagePath + ? { mode: "virtual-background", imagePath } + : { mode: "disabled" }; switch (effect.kind) { case "blur": return { mode: "background-blur", blurRadius }; - case "shipped": { - const imagePath = imagePathFor(effect.id); - return imagePath - ? { mode: "virtual-background", imagePath } - : { mode: "disabled" }; - } + case "shipped": + return withImage(imagePathFor(effect.id)); + case "added": + return withImage(addedUrl); default: return { mode: "disabled" }; }