diff --git a/locales/en/app.json b/locales/en/app.json index 8cf55a74c..deb9ed702 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -99,6 +99,10 @@ }, "disconnected_banner": "Connectivity to the server has been lost.", "error": { + "background_animated": "Animated images cannot be used as a background", + "background_not_an_image": "That file is not a supported image", + "background_not_kept": "That background could not be saved", + "background_undecodable": "That image could not be opened", "call_is_not_supported": "Call is not supported", "call_not_found": "Call not found", "call_not_found_description": "<0>That link doesn't appear to belong to any existing call. Check that you have the right link, or <2>create a new one.", diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index db60d7ab9..b7a384f28 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -328,6 +328,128 @@ test.describe("a background of one's own", () => { }); }); +test.describe("what can be added", () => { + test("refuses an animated image", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Refused user", "Animated refused"); + await addImage(page, ANIMATED_GIF, { + name: "moving.gif", + type: "image/gif", + }); + + await expect( + page.getByText("Animated images cannot be used as a background"), + ).toBeVisible(); + const section = page.getByRole("group", { name: "Background effects" }); + await expect( + section.getByRole("menuitemradio", { name: "Background 3" }), + ).toHaveCount(0); + await expect( + section.getByRole("menuitemradio", { name: "None" }), + ).toHaveAttribute("aria-checked", "true"); + expect(await keptImages(page)).toEqual([]); + }); + + test("reduces an oversized image", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Large user", "Oversized reduced"); + await addImage(page, await pngImage(page, { width: 4000, height: 3000 })); + + await expect( + page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }), + ).toBeVisible(); + await expect(page.getByRole("alert")).toHaveCount(0); + const [kept] = await keptImages(page); + expect(kept).toMatchObject({ width: 1920, height: 1440 }); + }); + + test("lays every image on an opaque ground", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Logo user", "Transparent made opaque"); + await addImage( + page, + await pngImage(page, { width: 320, height: 180, transparent: true }), + ); + + await expect( + page + .getByRole("group", { name: "Background effects" }) + .getByRole("menuitemradio", { name: "Background 3" }), + ).toBeVisible(); + const [kept] = await keptImages(page); + expect(kept.minAlpha).toBe(255); + }); +}); + +/** Two frames of one pixel, red then blue. */ +const ANIMATED_GIF = Buffer.from( + "R0lGODlhAQABAPEAAP8AAAAA/wAAAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQACgAAACwAAAAAAQABAAACAkQBACH5BAAKAAAALAAAAAABAAEAAAICTAEAOw==", + "base64", +); + +/** A red PNG of the given size; transparent leaves its right half empty. */ +async function pngImage( + page: Page, + size: { width: number; height: number; transparent?: boolean }, +): Promise { + const dataUrl = await page.evaluate(({ width, height, transparent }) => { + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext("2d")!; + context.fillStyle = "rgb(255, 0, 0)"; + context.fillRect(0, 0, transparent ? width / 2 : width, height); + return canvas.toDataURL("image/png"); + }, size); + return Buffer.from(dataUrl.split(",")[1], "base64"); +} + +/** The images the device keeps, as it keeps them. */ +async function keptImages( + page: Page, +): Promise<{ width: number; height: number; minAlpha: number }[]> { + return page.evaluate(async () => { + const db = await new Promise((resolve, reject) => { + const request = indexedDB.open("element-call-background-images"); + request.onsuccess = (): void => resolve(request.result); + request.onerror = (): void => reject(request.error); + }); + if (!db.objectStoreNames.contains("backgrounds")) return []; + const kept = await new Promise<{ image: Blob }[]>((resolve, reject) => { + const request = db + .transaction("backgrounds") + .objectStore("backgrounds") + .getAll(); + request.onsuccess = (): void => resolve(request.result); + request.onerror = (): void => reject(request.error); + }); + db.close(); + return Promise.all( + kept.map(async ({ image }) => { + const bitmap = await createImageBitmap(image); + const canvas = new OffscreenCanvas(bitmap.width, bitmap.height); + const context = canvas.getContext("2d")!; + context.drawImage(bitmap, 0, 0); + const { data } = context.getImageData( + 0, + 0, + bitmap.width, + bitmap.height, + ); + let minAlpha = 255; + for (let i = 3; i < data.length; i += 4) + minAlpha = Math.min(minAlpha, data[i]); + return { width: bitmap.width, height: bitmap.height, minAlpha }; + }), + ); + }); +} + /** The bytes of the one image the device keeps. */ async function keptBytes(page: Page): Promise { const base64 = await page.evaluate( @@ -370,14 +492,16 @@ async function redImage(page: Page): Promise { } /** Adds an image from the camera menu's add tile, leaving the menu open. */ -async function addImage(page: Page, image: Buffer): Promise { +async function addImage( + page: Page, + image: Buffer, + { name, type } = { name: "mine.png", type: "image/png" }, +): Promise { if (!(await page.getByRole("menu").isVisible())) await page.getByRole("button", { name: "Camera", exact: true }).click(); const chooser = page.waitForEvent("filechooser"); await page.getByRole("menuitem", { name: "Add image" }).click(); - await ( - await chooser - ).setFiles({ name: "mine.png", mimeType: "image/png", buffer: image }); + await (await chooser).setFiles({ name, mimeType: type, buffer: image }); } test.describe("joining with a background chosen", () => { diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index 647dd9d7a..6d3a72d1e 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -137,6 +137,7 @@ export const Default: Story = { ], backgroundEffectNotice: undefined, backgroundEffectSettling: false, + backgroundImageRefusal: undefined, addBackgroundImage: fn(), hangup: fn(), buttonSize: "lg", diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index 03a266728..0d8ba24fc 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -33,6 +33,11 @@ import { useBehavior } from "../useBehavior"; import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel"; import { LayoutSwitch } from "../room/LayoutSwitch"; import { type BackgroundEffectOption } from "./BackgroundEffectGrid"; +import { type UnusableReason } from "../livekit/backgroundImages"; + +export interface BackgroundImageRefusal { + reason: UnusableReason | "not-kept"; +} /** A background effect on offer, which the view names. */ export interface BackgroundEffectChoice { @@ -91,6 +96,8 @@ export interface FooterState { backgroundEffectNotice: "unavailable" | "slow" | undefined; /** Whether the first effect chosen is still being prepared. */ backgroundEffectSettling: boolean; + /** Why the last file offered couldn't be kept, if it couldn't. */ + backgroundImageRefusal: BackgroundImageRefusal | undefined; showFooter: boolean; /* This is needed for WindowMode = "flat" */ @@ -179,6 +186,21 @@ export const CallFooter: FC = ({ const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$); const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$); const addBackgroundImage = useBehavior(vm.addBackgroundImage$); + const refusal = useBehavior(vm.backgroundImageRefusal$); + const refusalMessage = useMemo(() => { + switch (refusal?.reason) { + case undefined: + return undefined; + case "not-an-image": + return { text: t("error.background_not_an_image") }; + case "animated": + return { text: t("error.background_animated") }; + case "undecodable": + return { text: t("error.background_undecodable") }; + case "not-kept": + return { text: t("error.background_not_kept") }; + } + }, [refusal, t]); const backgroundEffects = useBackgroundEffectLabels( useBehavior(vm.backgroundEffects$), ); @@ -248,6 +270,7 @@ export const CallFooter: FC = ({ onSelectBackgroundEffect={selectBackgroundEffect} backgroundEffectSettling={backgroundEffectSettling} onAddBackgroundImage={addBackgroundImage} + backgroundImageRefusal={refusalMessage} backgroundEffectNotice={ backgroundEffectNotice === "unavailable" ? t("background_effects.unavailable") diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index 0ea70c127..38ca2be10 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -8,7 +8,12 @@ Please see LICENSE in the repository root for full details. import { afterEach, describe, expect, it, vi } from "vitest"; import { BehaviorSubject } from "rxjs"; -import { testScope, mockMuteStates, mockMediaDevices } from "../utils/test"; +import { + flushPromises, + testScope, + mockMuteStates, + mockMediaDevices, +} from "../utils/test"; import { constant } from "../state/Behavior"; import type { CallViewModel } from "../state/CallViewModel/CallViewModel"; import type { Alignment, Layout } from "../state/layout-types"; @@ -21,9 +26,12 @@ import { import { HeaderStyle } from "../UrlParams"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; import { shippedBackgrounds } from "../livekit/backgroundEffects"; +import type * as BackgroundImages from "../livekit/backgroundImages"; import { type AddedBackground, addedBackgrounds, + maxAddedBackgrounds, + UnusableImage, } from "../livekit/backgroundImages"; const platformMock = vi.hoisted(() => vi.fn(() => "desktop")); @@ -41,6 +49,20 @@ vi.mock("@livekit/track-processors", () => ({ supportsModernBackgroundProcessors: (): boolean => modernRouteMock(), })); +const store = vi.hoisted(() => ({ + add: vi.fn<(file: Blob) => Promise>(), +})); +vi.mock("../livekit/backgroundImages", async (original) => { + const { BehaviorSubject } = await import("rxjs"); + return { + ...(await original()), + addedBackgrounds: { + added$: new BehaviorSubject([]), + add: store.add, + }, + }; +}); + const outputSelectionMock = vi.hoisted(() => vi.fn(() => true)); vi.mock("livekit-client", () => ({ supportsAudioOutputSelection: (): boolean => outputSelectionMock(), @@ -345,6 +367,51 @@ describe("createCallFooterViewModel", () => { expect(vm.backgroundEffectSettling$.value).toBe(false); }); + it("adding is unavailable at the limit", () => { + sdkSupportMock.mockReturnValue(true); + const added$ = addedBackgrounds.added$ as BehaviorSubject< + AddedBackground[] | undefined + >; + const vm = lobbyOn("desktop"); + expect(vm.addBackgroundImage$.value).toBeDefined(); + added$.next( + Array.from({ length: maxAddedBackgrounds }, (_, i) => ({ + id: `${i}`, + url: `blob:${i}`, + })), + ); + expect(vm.addBackgroundImage$.value).toBeUndefined(); + added$.next(added$.value!.slice(1)); + expect(vm.addBackgroundImage$.value).toBeDefined(); + added$.next([]); + }); + + it("says why each refused file was refused", async () => { + sdkSupportMock.mockReturnValue(true); + const vm = lobbyOn("desktop"); + store.add.mockRejectedValue(new UnusableImage("animated")); + vm.addBackgroundImage$.value!(new File(["x"], "a.gif")); + await flushPromises(); + const first = vm.backgroundImageRefusal$.value; + expect(first).toEqual({ reason: "animated" }); + + // The same again is a refusal of its own, so it is shown again. + vm.addBackgroundImage$.value!(new File(["x"], "a.gif")); + await flushPromises(); + expect(vm.backgroundImageRefusal$.value).toEqual({ reason: "animated" }); + expect(vm.backgroundImageRefusal$.value).not.toBe(first); + + store.add.mockRejectedValue(new Error("quota")); + vm.addBackgroundImage$.value!(new File(["x"], "b.png")); + await flushPromises(); + expect(vm.backgroundImageRefusal$.value).toEqual({ reason: "not-kept" }); + + store.add.mockResolvedValue("kept"); + vm.addBackgroundImage$.value!(new File(["x"], "c.png")); + await flushPromises(); + expect(vm.backgroundImageRefusal$.value).toBeUndefined(); + }); + it("availability is the same before and during a call", () => { for (const supported of [true, false]) { sdkSupportMock.mockReturnValue(supported); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index 360e092a3..4c4834595 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { + BehaviorSubject, combineLatest, distinctUntilChanged, map, @@ -27,6 +28,8 @@ import { import { type AddedBackground, addedBackgrounds, + maxAddedBackgrounds, + UnusableImage, } from "../livekit/backgroundImages"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; @@ -42,7 +45,11 @@ import { type MuteStates } from "../state/MuteStates"; import { createStaticViewModel, type ViewModel } from "../state/ViewModel"; import { HeaderStyle } from "../UrlParams"; import { platform } from "../Platform"; -import { type BackgroundEffectChoice, type FooterSnapshot } from "./CallFooter"; +import { + type BackgroundEffectChoice, + type BackgroundImageRefusal, + type FooterSnapshot, +} from "./CallFooter"; /** * Shared helper: maps MuteStates into the audio/video enabled + toggle behaviors @@ -101,6 +108,7 @@ function buildDeviceBehaviors( | "backgroundEffectNotice$" | "backgroundEffectSettling$" | "addBackgroundImage$" + | "backgroundImageRefusal$" > { const options$ = ( available$: Behavior>, @@ -122,6 +130,10 @@ function buildDeviceBehaviors( const offered$ = disableSwitcher$.pipe( map((switcherDisabled) => !switcherDisabled && supported), ); + // A new object for each refusal, so the same one twice is shown twice. + const refusal$ = new BehaviorSubject( + undefined, + ); return { audioOptions$: scope.behavior(options$(mediaDevices.audioInput.available$)), selectedAudio$: scope.behavior( @@ -181,20 +193,23 @@ function buildDeviceBehaviors( ), // Kept and offered, not put on: that waits for the user to choose it. addBackgroundImage$: scope.behavior( - offered$.pipe( - map((offered) => - offered + combineLatest([offered$, addedBackgrounds.added$]).pipe( + map(([offered, added]) => + offered && (added?.length ?? 0) < maxAddedBackgrounds ? (file: File): void => { - addedBackgrounds - .add(file) - .catch((e) => - logger.warn("Could not keep that background", e), - ); + refusal$.next(undefined); + addedBackgrounds.add(file).catch((e) => { + logger.warn("Could not keep that background", e); + refusal$.next({ + reason: e instanceof UnusableImage ? e.reason : "not-kept", + }); + }); } : undefined, ), ), ), + backgroundImageRefusal$: refusal$, }; } diff --git a/src/components/MediaMuteAndSwitchButton.module.css b/src/components/MediaMuteAndSwitchButton.module.css index e2a470ed4..6450440db 100644 --- a/src/components/MediaMuteAndSwitchButton.module.css +++ b/src/components/MediaMuteAndSwitchButton.module.css @@ -115,3 +115,8 @@ Please see LICENSE in the repository root for full details. flex-shrink: 0; color: var(--cpd-color-icon-secondary); } + +.refusal { + margin: var(--cpd-space-2x) + calc(var(--cpd-space-4x) - var(--cpd-border-width-1)) 0; +} diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index 769587489..4f6dc6e95 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -339,6 +339,42 @@ export const BackgroundEffectsSettling: Story = { }, }; +/** A file that can't be used: said where it was chosen, and nothing changes. */ +export const BackgroundImageRefused: Story = { + args: { + ...VideoUnmute.args, + selectedBackgroundEffect: "blur", + backgroundEffectNotice: + "Background effects run slowly on this platform, which may cause your video to stutter.", + backgroundImageRefusal: { text: "That file is not a supported image" }, + }, + decorators: [ + (Story): JSX.Element => ( + + + + ), + ], + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Camera" })); + const menu = within(document.body); + const message = await menu.findByText("That file is not a supported image"); + const list = document.body.querySelector( + `.${styles.deviceList}`, + )!; + await expect(list.scrollHeight).toBeGreaterThan(list.clientHeight); + const scrollport = list.getBoundingClientRect(); + const shown = message.getBoundingClientRect(); + await expect(shown.bottom).toBeLessThanOrEqual(scrollport.bottom + 1); + await expect(shown.top).toBeGreaterThanOrEqual(scrollport.top - 1); + await expect(menu.queryByText(/run slowly/)).toBeNull(); + await expect( + menu.getByRole("menuitemradio", { name: "Blur" }), + ).toHaveAttribute("aria-checked", "true"); + }, +}; + /** In a short call the effects scroll into view with the list. */ export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = { args: { diff --git a/src/components/MediaMuteAndSwitchButton.test.tsx b/src/components/MediaMuteAndSwitchButton.test.tsx index 21ccaedd0..437dab0a0 100644 --- a/src/components/MediaMuteAndSwitchButton.test.tsx +++ b/src/components/MediaMuteAndSwitchButton.test.tsx @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { afterEach, describe, expect, test, vi } from "vitest"; +import { afterEach, describe, expect, onTestFinished, test, vi } from "vitest"; import { axe } from "vitest-axe"; import { act, @@ -468,6 +468,50 @@ describe("MediaMuteAndSwitchButton", () => { } }); + test("a refusal can be dismissed and does not outlive the menu", async () => { + const user = userEvent.setup(); + // jsdom lays nothing out, so has nothing to scroll. + const scroll = Element.prototype.scrollIntoView; + Element.prototype.scrollIntoView = vi.fn(); + onTestFinished(() => { + Element.prototype.scrollIntoView = scroll; + }); + const menu = (refusal?: { text: string }): JSX.Element => + withProviders( + , + ); + const { rerender, getByRole } = render(menu()); + await user.click(getByRole("button", { name: "Camera" })); + + rerender(menu({ text: "That file is not a supported image" })); + expect( + screen.getByText("That file is not a supported image"), + ).toBeVisible(); + screen.getByRole("menuitemradio", { name: "Blur", checked: true }); + await user.click(screen.getByRole("button", { name: "Close" })); + expect(screen.queryByText("That file is not a supported image")).toBeNull(); + + // The next refusal is shown, even if it says the same. + rerender(menu({ text: "That file is not a supported image" })); + expect( + screen.getByText("That file is not a supported image"), + ).toBeVisible(); + + await user.keyboard("{Escape}"); + await user.click(getByRole("button", { name: "Camera" })); + expect(screen.queryByText("That file is not a supported image")).toBeNull(); + }); + test("offers the background effects in a phone's drawer", async () => { platformMock.mockReturnValue("android"); const userAgent = vi diff --git a/src/components/MediaMuteAndSwitchButton.tsx b/src/components/MediaMuteAndSwitchButton.tsx index daa568023..365584819 100644 --- a/src/components/MediaMuteAndSwitchButton.tsx +++ b/src/components/MediaMuteAndSwitchButton.tsx @@ -16,6 +16,7 @@ import { type ReactElement, } from "react"; import { + Alert, Button, Menu, MenuItem, @@ -82,6 +83,8 @@ export interface MediaMuteAndSwitchButtonProps { backgroundEffectSettling?: boolean; /** Called with the file chosen from the add tile. Omit to leave it out. */ onAddBackgroundImage?: (file: File) => void; + /** Why the last file offered couldn't be used; a new object each time. */ + backgroundImageRefusal?: { text: string }; /** * For any toggle and option this method will be called. * So toggles need to be implemented by listening here and setting the right toggle item to `enabled` @@ -120,6 +123,7 @@ export const MediaMuteAndSwitchButton: FC = ({ backgroundEffectNotice, backgroundEffectSettling, onAddBackgroundImage, + backgroundImageRefusal, onSelect, }) => { // Requested but not yet selected. Keyed by kind too, since Chrome uses @@ -133,11 +137,24 @@ export const MediaMuteAndSwitchButton: FC = ({ // elsewhere; it is held open until the picker is done. const choosingFile = useRef(false); const fileInput = useRef(null); - const onOpenChange = useCallback((open: boolean): void => { - if (!open && choosingFile.current) return; - setMenuOpen(open); - // Drop a request that never arrived. - if (!open) setPlannedSelection(null); + // A refusal is about the attempt that caused it: shown until dismissed or + // until the menu closes, and again for the next one. + const [seenRefusal, setSeenRefusal] = useState(); + const refusal = + backgroundImageRefusal !== seenRefusal ? backgroundImageRefusal : undefined; + const onOpenChange = useCallback( + (open: boolean): void => { + if (!open && choosingFile.current) return; + setMenuOpen(open); + // Drop a request that never arrived. + if (!open) setPlannedSelection(null); + if (!open) setSeenRefusal(backgroundImageRefusal); + }, + [backgroundImageRefusal], + ); + // Scrolled to as it appears, as the list may be scrolled away from it. + const scrollIntoView = useCallback((element: HTMLElement | null): void => { + element?.scrollIntoView({ block: "nearest" }); }, []); const watchFileInput = useCallback( (input: HTMLInputElement | null): (() => void) | undefined => { @@ -489,12 +506,23 @@ export const MediaMuteAndSwitchButton: FC = ({ /> )} {/* In the list, so the menu grows no taller for it. */} - {showEffects && backgroundEffectNotice !== undefined && ( -
- - {backgroundEffectNotice} + {showEffects && refusal !== undefined && ( +
+ setSeenRefusal(refusal)} + />
)} + {showEffects && + refusal === undefined && + backgroundEffectNotice !== undefined && ( +
+ + {backgroundEffectNotice} +
+ )}
diff --git a/src/livekit/backgroundEffects.test.ts b/src/livekit/backgroundEffects.test.ts index c2f02adf9..e4219811b 100644 --- a/src/livekit/backgroundEffects.test.ts +++ b/src/livekit/backgroundEffects.test.ts @@ -15,6 +15,7 @@ import { serializeEffect, shippedBackgrounds, } from "./backgroundEffects"; +import { maxAddedBackgrounds } from "./backgroundImages"; describe("the chosen background effect", () => { test("round-trips through its stored form", () => { @@ -39,6 +40,14 @@ describe("the chosen background effect", () => { expect(parseEffect(raw)).toEqual({ kind: "none" }); }); + test("fills the grid at the limit and one short of it", () => { + // No effect, blur and the shipped ones, then the added ones. + const fixed = 2 + shippedBackgrounds.length; + expect((fixed + maxAddedBackgrounds) % 3).toBe(0); + // One short, the add tile takes the last place. + expect((fixed + maxAddedBackgrounds - 1 + 1) % 3).toBe(0); + }); + test("gives every shipped background an image to draw", () => { for (const { id } of shippedBackgrounds) expect(imagePathFor(id)).toBeTruthy(); diff --git a/src/livekit/backgroundImages.test.ts b/src/livekit/backgroundImages.test.ts index 252a879b6..6e52a7105 100644 --- a/src/livekit/backgroundImages.test.ts +++ b/src/livekit/backgroundImages.test.ts @@ -13,6 +13,9 @@ import { type BackgroundImageStorage, IndexedDBImageStorage, type KeptImage, + maxAddedBackgrounds, + prepareImage, + UnusableImage, } from "./backgroundImages"; import { flushPromises } from "../utils/test"; @@ -29,6 +32,7 @@ describe("AddedBackgrounds", () => { { id: "later", image, addedAt: 2 }, { id: "earlier", image, addedAt: 1 }, ]), + asGiven, ); expect(store.added$.value).toBeUndefined(); await flushPromises(); @@ -40,13 +44,43 @@ describe("AddedBackgrounds", () => { it("keeps an added image and offers it after the others", async () => { const storage = memoryStorage(); - const store = new AddedBackgrounds(storage); + const store = new AddedBackgrounds(storage, asGiven); await flushPromises(); const id = await store.add(new Blob(["x"], { type: "image/png" })); expect(store.added$.value?.map((a) => a.id)).toEqual([id]); expect((await storage.list()).map((k) => k.id)).toEqual([id]); }); + it("keeps the prepared image, not the file", async () => { + const storage = memoryStorage(); + const prepared = new Blob(["y"], { type: "image/webp" }); + const store = new AddedBackgrounds(storage, async () => + Promise.resolve(prepared), + ); + await store.add(new Blob(["x"], { type: "image/png" })); + expect((await storage.list())[0].image).toBe(prepared); + }); + + it("keeps nothing of a file it refuses", async () => { + const storage = memoryStorage(); + const store = new AddedBackgrounds(storage, async () => + Promise.reject(new UnusableImage("animated")), + ); + await flushPromises(); + await expect(store.add(new Blob(["x"]))).rejects.toThrow(UnusableImage); + expect(await storage.list()).toEqual([]); + expect(store.added$.value).toEqual([]); + }); + + it("keeps no more than the grid holds", async () => { + const storage = memoryStorage(); + const store = new AddedBackgrounds(storage, asGiven); + for (let i = 0; i < maxAddedBackgrounds; i++) + await store.add(new Blob([`${i}`])); + await expect(store.add(new Blob(["more"]))).rejects.toThrow(RangeError); + expect(await storage.list()).toHaveLength(maxAddedBackgrounds); + }); + it("offers nothing where the browser keeps nothing", async () => { const store = new AddedBackgrounds(null); await flushPromises(); @@ -65,6 +99,22 @@ describe("AddedBackgrounds", () => { }); }); +describe("prepareImage", () => { + // Refused before anything is decoded, which only a real browser could do. + it("refuses a file that isn't an image", async () => { + await expect( + prepareImage(new Blob(["x"], { type: "text/plain" })), + ).rejects.toEqual(new UnusableImage("not-an-image")); + }); + + it("refuses an animated type where the browser can't count frames", async () => { + expect(globalThis.ImageDecoder).toBeUndefined(); + await expect( + prepareImage(new Blob(["GIF89a"], { type: "image/gif" })), + ).rejects.toEqual(new UnusableImage("animated")); + }); +}); + describe("IndexedDBImageStorage", () => { it("keeps images across sessions", async () => { const indexedDB = new IDBFactory(); @@ -84,6 +134,9 @@ describe("IndexedDBImageStorage", () => { }); }); +// Decoding needs a real browser; the end-to-end checks cover it. +const asGiven = async (file: Blob): Promise => Promise.resolve(file); + function memoryStorage(kept: KeptImage[] = []): BackgroundImageStorage { return { list: async (): Promise => Promise.resolve([...kept]), diff --git a/src/livekit/backgroundImages.ts b/src/livekit/backgroundImages.ts index 3465c3a24..8f48abdf3 100644 --- a/src/livekit/backgroundImages.ts +++ b/src/livekit/backgroundImages.ts @@ -10,6 +10,21 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../state/Behavior"; +export const maxAddedBackgrounds = 5; + +/** The longest edge kept; the pipeline scales every image to the camera. */ +export const maxStoredEdge = 1920; + +/** Why a file can't be used as a background. */ +export type UnusableReason = "not-an-image" | "animated" | "undecodable"; + +export class UnusableImage extends Error { + public constructor(public readonly reason: UnusableReason) { + super(reason); + this.name = "UnusableImage"; + } +} + /** A background the user added, as the menu and the pipeline use it. */ export interface AddedBackground { id: string; @@ -38,7 +53,10 @@ export class AddedBackgrounds { public readonly added$: Behavior = this.subject$; - public constructor(private readonly storage: BackgroundImageStorage | null) { + public constructor( + private readonly storage: BackgroundImageStorage | null, + private readonly prepare: (file: Blob) => Promise = prepareImage, + ) { this.read().catch((e) => { logger.warn("Could not read added backgrounds", e); this.subject$.next([]); @@ -53,24 +71,87 @@ export class AddedBackgrounds { ); } - /** Keeps a file as a background, and answers with its id. */ + /** + * Keeps a file as a background, and answers with its id. Throws + * {@link UnusableImage} for a file that can't be used, and a RangeError + * once maxAddedBackgrounds are kept. + */ public async add(file: Blob): Promise { if (this.storage === null) throw new Error("This browser keeps no backgrounds"); + const image = await this.prepare(file); + if ((await this.storage.list()).length >= maxAddedBackgrounds) + throw new RangeError( + `${maxAddedBackgrounds} backgrounds are kept already`, + ); const kept: KeptImage = { id: crypto.randomUUID(), - image: file, + image, addedAt: Date.now(), }; await this.storage.put(kept); this.subject$.next([ ...(this.subject$.value ?? []), - { id: kept.id, url: URL.createObjectURL(file) }, + { id: kept.id, url: URL.createObjectURL(image) }, ]); return kept.id; } } +/** + * Refuses what can't be used, reduces what is larger than is kept, and lays + * every image on an opaque ground: a background has to cover what is behind + * it, so a transparent pixel would be a hole in it. + */ +export async function prepareImage(file: Blob): Promise { + if (!file.type.startsWith("image/")) throw new UnusableImage("not-an-image"); + if (await isAnimated(file)) throw new UnusableImage("animated"); + + let bitmap: ImageBitmap; + try { + bitmap = await createImageBitmap(file); + } catch (e) { + logger.debug("Could not decode the chosen file", e); + throw new UnusableImage("undecodable"); + } + try { + const scale = Math.min( + 1, + maxStoredEdge / Math.max(bitmap.width, bitmap.height), + ); + const canvas = new OffscreenCanvas( + Math.round(bitmap.width * scale), + Math.round(bitmap.height * scale), + ); + const context = canvas.getContext("2d"); + if (!context) throw new UnusableImage("undecodable"); + context.fillStyle = "black"; + context.fillRect(0, 0, canvas.width, canvas.height); + context.drawImage(bitmap, 0, 0, canvas.width, canvas.height); + return await canvas.convertToBlob({ type: "image/webp", quality: 0.9 }); + } finally { + bitmap.close(); + } +} + +// Read from the file, not its type: a still and an animated WebP share one. +async function isAnimated(file: Blob): Promise { + if (typeof ImageDecoder === "undefined") return /gif|apng/.test(file.type); + try { + const decoder = new ImageDecoder({ + data: await file.arrayBuffer(), + type: file.type, + }); + // Some browsers have no selected track until the tracks are ready. + await decoder.tracks.ready; + await decoder.completed; + return (decoder.tracks.selectedTrack?.frameCount ?? 1) > 1; + } catch (e) { + logger.debug("Could not read the frame count, judging by type", e); + return /gif|apng/.test(file.type); + } +} + const DB_NAME = "element-call-background-images"; const STORE = "backgrounds";