diff --git a/apps/multiplatform/CODE.md b/apps/multiplatform/CODE.md index 26a36e75bb..67fa676414 100644 --- a/apps/multiplatform/CODE.md +++ b/apps/multiplatform/CODE.md @@ -289,6 +289,7 @@ desktop/src/jvmMain/kotlin/chat/simplex/desktop/ -- Desktop app (1 file) | common/.../common/StoreWindowState.kt (desktopMain) | spec/architecture.md | product/views/settings.md | | common/.../common/model/NtfManager.desktop.kt (desktopMain) | spec/services/notifications.md | product/flows/messaging.md | | common/.../common/views/helpers/AppUpdater.kt (desktopMain) | spec/architecture.md | product/views/settings.md | +| common/.../common/platform/AnimatedImage.desktop.kt (desktopMain) | spec/client/chat-view.md | product/views/chat.md | ### Haskell Core Sources (at `../../src/Simplex/Chat/` relative to `apps/multiplatform/`) diff --git a/apps/multiplatform/common/build.gradle.kts b/apps/multiplatform/common/build.gradle.kts index 43bc114f21..ec4235d344 100644 --- a/apps/multiplatform/common/build.gradle.kts +++ b/apps/multiplatform/common/build.gradle.kts @@ -72,7 +72,6 @@ kotlin { api("org.jetbrains.compose.ui:ui-text:${rootProject.extra["compose.version"] as String}") implementation("org.jetbrains.compose.material:material-icons-core:1.7.3") implementation("org.jetbrains.compose.material:material-icons-extended:1.7.3") - implementation("org.jetbrains.compose.components:components-animatedimage:${rootProject.extra["compose.version"] as String}") //Barcode api("org.boofcv:boofcv-core:1.1.3") implementation("com.godaddy.android.colorpicker:compose-color-picker-jvm:0.7.0") diff --git a/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/chat/item/CIImageView.android.kt b/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/chat/item/CIImageView.android.kt index ae5b8043ed..5538655a92 100644 --- a/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/chat/item/CIImageView.android.kt +++ b/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/chat/item/CIImageView.android.kt @@ -3,6 +3,7 @@ package chat.simplex.common.views.chat.item import android.os.Build.VERSION.SDK_INT import androidx.compose.runtime.Composable import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.State import androidx.compose.ui.graphics.ImageBitmap import androidx.compose.ui.graphics.painter.BitmapPainter import androidx.compose.ui.graphics.painter.Painter @@ -24,6 +25,7 @@ actual fun SimpleAndAnimatedImageView( file: CIFile?, imageProvider: () -> ImageGalleryProvider, smallView: Boolean, + blurred: State, // coil drives the animation itself here, so there is nothing to pause ImageView: @Composable (painter: Painter, onClick: () -> Unit) -> Unit ) { val context = LocalContext.current diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/item/CIImageView.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/item/CIImageView.kt index 7ce44475b5..67fc0a038c 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/item/CIImageView.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/item/CIImageView.kt @@ -210,7 +210,7 @@ fun CIImageView( val loaded = res.value if (loaded != null && file != null) { val (imageBitmap, data, _) = loaded - SimpleAndAnimatedImageView(data, imageBitmap, file, imageProvider, smallView, @Composable { painter, onClick -> ImageView(painter, image, file.fileSource, onClick) }) + SimpleAndAnimatedImageView(data, imageBitmap, file, imageProvider, smallView, blurred, @Composable { painter, onClick -> ImageView(painter, image, file.fileSource, onClick) }) } else { imageView(previewBitmap, onClick = { if (file != null) { @@ -281,5 +281,6 @@ expect fun SimpleAndAnimatedImageView( file: CIFile?, imageProvider: () -> ImageGalleryProvider, smallView: Boolean, + blurred: State, ImageView: @Composable (painter: Painter, onClick: () -> Unit) -> Unit ) diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/item/ImageFullScreenView.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/item/ImageFullScreenView.kt index 8d96102daa..b1604adc84 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/item/ImageFullScreenView.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/item/ImageFullScreenView.kt @@ -148,8 +148,6 @@ fun ImageFullScreenView(imageProvider: () -> ImageGalleryProvider, close: () -> ) } .fillMaxSize() - // LALAL - // https://github.com/JetBrains/compose-multiplatform/pull/2015/files#diff-841b3825c504584012e1d1c834d731bae794cce6acad425d81847c8bbbf239e0R24 if (media is ProviderMedia.Image) { val (data: ByteArray, imageBitmap: ImageBitmap) = media FullScreenImageView(modifier, data, imageBitmap) diff --git a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/platform/AnimatedImage.desktop.kt b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/platform/AnimatedImage.desktop.kt new file mode 100644 index 0000000000..e12bae4280 --- /dev/null +++ b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/platform/AnimatedImage.desktop.kt @@ -0,0 +1,199 @@ +package chat.simplex.common.platform + +import androidx.compose.runtime.* +import androidx.compose.ui.graphics.ImageBitmap +import androidx.compose.ui.graphics.asComposeImageBitmap +import chat.simplex.common.simplexWindowState +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.first +import org.jetbrains.skia.Bitmap +import org.jetbrains.skia.Codec +import org.jetbrains.skia.ColorAlphaType +import org.jetbrains.skia.Data + +// Animated images are decoded from data received from other users, which is what the bounds below are for + +// In bytes as the file chooses the color type, 1920x1920 at 4 bytes a pixel is ~15MB +private const val MAX_ANIMATED_RASTER_BYTES: Long = 1920L * 1920 * 4 +// 65535x32 is only 2.1MP, so each side is bounded as well +private const val MAX_ANIMATED_SIDE = 4096 +// Skia copies the encoded bytes into native memory and scans them to count frames +private const val MAX_ANIMATED_FILE_SIZE = 32 * 1024 * 1024 +// Counting frames builds a table the codec holds while it plays, several times the file's size for minimal ones +private const val MAX_ANIMATED_FRAMES = 10_000 +// 10ms or less is how "as fast as possible" is written, and browsers substitute 100ms for it +private const val MAX_UNSPECIFIED_FRAME_DURATION_MS = 10 +private const val DEFAULT_FRAME_DURATION_MS = 100L +private const val MIN_FRAME_DURATION_MS = 20L +// A frame costing more than this holds most of a core to show under 10 frames a second +private const val MAX_FRAME_DECODE_MS = 100L +// Far above what a frame within the bounds above can cost, so only a stall reaches it +private const val MAX_WAITED_FRAME_COST_MS = 10 * MAX_FRAME_DECODE_MS +private const val SLOW_FRAME_COST = 2 +internal const val MAX_SLOW_FRAME_DEBT = 4 +private const val NO_PRIOR_FRAME = -1 +// A frame given no prior frame is rebuilt by recursing down its chain, so a long enough one overflows the +// native stack, which no catch can stop. Real animations rebuild nothing. +private const val MAX_REBUILT_FRAMES = 64 + +// Read once, as asking the codec about a frame allocates and the loop may repeat forever +private class Animation(val codec: Codec, val priorFrames: IntArray, val frameDelays: LongArray) + +/** + * The current frame of [data], or [still] when it is not an animation, falls outside the bounds above, or + * fails before showing a frame; after that it stops on the frame it reached. Decoding runs off the UI thread. + */ +@Composable +fun rememberAnimatedImage(data: ByteArray, still: ImageBitmap, hidden: () -> Boolean = { false }): ImageBitmap { + // Keyed as the decoding is, so frames are not written into a replaced state, and hidden is not a key so it + // pauses instead of restarting. Every frame is a new wrapper, and only its identity says the image changed. + val frame = remember(data, still) { mutableStateOf(still, neverEqualPolicy()) } + LaunchedEffect(data, still) { + withContext(animationDecoder) { + val animation = openAnimation(data) ?: return@withContext + try { + playFrames(animation, hidden) { frame.value = it } + } finally { + animation.codec.close() + } + } + } + return frame.value +} + +// Decoding several large animations must not starve the long running calls that share this pool +@OptIn(ExperimentalCoroutinesApi::class) +private val animationDecoder = Dispatchers.Default.limitedParallelism(2) + +private fun openAnimation(data: ByteArray): Animation? { + if (!looksAnimatable(data) || !fileSizeWithinBounds(data.size)) return null + var codec: Codec? = null + var animation: Animation? = null + try { + // Skia retains the encoded bytes, so this native buffer is freed as soon as the codec has taken it + val encoded = Data.makeFromBytes(data) + codec = try { + Codec.makeFromData(encoded) + } finally { + encoded.close() + } + animation = boundedAnimation(codec) + } catch (e: Throwable) { + Log.e(TAG, "Unable to read animated image: $e") + } + // The codec is only left open for an animation that took it, so no bound can return past closing it + if (animation == null) codec?.close() + return animation +} + +private fun boundedAnimation(codec: Codec): Animation? { + val info = codec.imageInfo + if (!rasterWithinBounds(info.width, info.height, info.bytesPerPixel)) return null + // Counting frames scans the file, while dimensions are only read from the header + val frameCount = codec.frameCount + if (!frameCountWithinBounds(frameCount)) return null + val requiredFrames = IntArray(frameCount) + val frameDelays = LongArray(frameCount) + for (i in 0 until frameCount) { + val frameInfo = codec.getFrameInfo(i) + requiredFrames[i] = frameInfo.requiredFrame + frameDelays[i] = frameDuration(frameInfo.duration) + } + if (!rebuiltFramesWithinBounds(requiredFrames)) return null + return Animation(codec, IntArray(frameCount) { priorFrame(it, requiredFrames[it]) }, frameDelays) +} + +internal fun looksAnimatable(data: ByteArray): Boolean = + data.startsWith("GIF8") || (data.startsWith("RIFF") && data.startsWith("WEBP", offset = 8)) + +private fun ByteArray.startsWith(ascii: String, offset: Int = 0): Boolean { + if (size < offset + ascii.length) return false + return ascii.indices.all { this[offset + it] == ascii[it].code.toByte() } +} + +internal fun rasterWithinBounds(width: Int, height: Int, bytesPerPixel: Int): Boolean { + if (width !in 1..MAX_ANIMATED_SIDE || height !in 1..MAX_ANIMATED_SIDE) return false + // 0 bytes per pixel would let any raster pass the bound below + if (bytesPerPixel < 1) return false + // The sides are bounded before they are multiplied, so the product cannot overflow + return width.toLong() * height * bytesPerPixel <= MAX_ANIMATED_RASTER_BYTES +} + +private suspend fun playFrames(animation: Animation, hidden: () -> Boolean, showFrame: (ImageBitmap) -> Unit) { + try { + val codec = animation.codec + val bitmap = Bitmap() + // The codec reports only the first frame's alpha type, and a frame with alpha cannot be read into an + // opaque bitmap. allocPixels returns false rather than throwing. + if (!bitmap.allocPixels(codec.imageInfo.withColorAlphaType(ColorAlphaType.PREMUL))) return + var loopsLeft = codec.repetitionCount // negative repeats forever + var debt = 0 + while (true) { + for (i in animation.priorFrames.indices) { + awaitFramesAreSeen(hidden) + val startedDecoding = System.nanoTime() + codec.readPixels(bitmap, i, animation.priorFrames[i]) + // Wall time, so a frame can overrun by being descheduled rather than by being expensive + val decodedIn = System.nanoTime() - startedDecoding + debt = slowFrameDebt(debt, decodedIn > MAX_FRAME_DECODE_MS * 1_000_000) + // The bitmap is never closed, as the wrapper points at its pixels and a frame may still be drawn + showFrame(bitmap.asComposeImageBitmap()) + if (debt >= MAX_SLOW_FRAME_DEBT) { + Log.d(TAG, "Animation too expensive to decode, stopping on this frame") + return + } + delay(frameWait(animation.frameDelays[i], decodedIn / 1_000_000)) + } + if (loopsLeft == 0) return + if (loopsLeft > 0) loopsLeft-- + } + } catch (e: CancellationException) { + throw e // the view is gone, not a decoding failure + } catch (e: Throwable) { + Log.e(TAG, "Unable to play animated image: $e") + } +} + +// Composition survives the window being minimized or hidden, and the caller knows when its image cannot be seen +private suspend fun awaitFramesAreSeen(hidden: () -> Boolean) { + if (framesAreSeen(hidden)) return + snapshotFlow { framesAreSeen(hidden) }.first { it } +} + +private fun framesAreSeen(hidden: () -> Boolean): Boolean = + simplexWindowState.windowVisible.value && !simplexWindowState.windowState.isMinimized && !hidden() + +// Waiting out the cost as well as the delay leaves an animation about half a decoder thread. The cost is +// wall time, so a stall is only waited out so far. +internal fun frameWait(delayMs: Long, costMs: Long): Long = + maxOf(delayMs, costMs.coerceAtMost(MAX_WAITED_FRAME_COST_MS)) + +internal fun fileSizeWithinBounds(size: Int): Boolean = size <= MAX_ANIMATED_FILE_SIZE + +// A file of no frames would spin the playback loop uncancellably, as it only suspends inside the range +internal fun frameCountWithinBounds(frameCount: Int): Boolean = frameCount in 2..MAX_ANIMATED_FRAMES + +// The frame the codec may decode this one from, which is the one before it when the bitmap still holds it. +// Rebuilding the chain instead costs 9.10ms a frame against 0.05ms, and Skia refuses a frame it did not ask for. +internal fun priorFrame(index: Int, requiredFrame: Int): Int = + if (requiredFrame == index - 1) index - 1 else NO_PRIOR_FRAME + +// requiredFrames is what each frame continues; one that continues nothing starts a chain of its own +internal fun rebuiltFramesWithinBounds(requiredFrames: IntArray): Boolean { + val chain = IntArray(requiredFrames.size) + requiredFrames.forEachIndexed { index, required -> + val continues = required in 0 until index + chain[index] = if (continues) chain[required] + 1 else 1 + if (continues && priorFrame(index, required) == NO_PRIOR_FRAME && chain[required] > MAX_REBUILT_FRAMES) return false + } + return true +} + +// Two expensive frames in a row reach the debt, and so do frames that alternate with cheap ones, which a +// count that reset would miss +internal fun slowFrameDebt(debt: Int, tooSlow: Boolean): Int = + (debt + if (tooSlow) SLOW_FRAME_COST else -1).coerceAtLeast(0) + +internal fun frameDuration(declaredMs: Int): Long = + if (declaredMs <= MAX_UNSPECIFIED_FRAME_DURATION_MS) DEFAULT_FRAME_DURATION_MS + else declaredMs.toLong().coerceAtLeast(MIN_FRAME_DURATION_MS) diff --git a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/chat/item/CIImageView.desktop.kt b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/chat/item/CIImageView.desktop.kt index b4a24e3572..98ffa7c8a4 100644 --- a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/chat/item/CIImageView.desktop.kt +++ b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/chat/item/CIImageView.desktop.kt @@ -1,6 +1,7 @@ package chat.simplex.common.views.chat.item import androidx.compose.runtime.Composable +import androidx.compose.runtime.State import androidx.compose.ui.graphics.* import androidx.compose.ui.graphics.painter.BitmapPainter import androidx.compose.ui.graphics.painter.Painter @@ -15,10 +16,15 @@ actual fun SimpleAndAnimatedImageView( file: CIFile?, imageProvider: () -> ImageGalleryProvider, smallView: Boolean, + blurred: State, ImageView: @Composable (painter: Painter, onClick: () -> Unit) -> Unit ) { - // LALAL make it animated too - ImageView(BitmapPainter(imageBitmap)) { + // The small view is the chat list preview, which the layout keeps on screen without pause, so it stays a + // still image. A full screen modal is shown beside the chat rather than in place of it, so this item keeps + // composing under one and would otherwise decode where nobody can see it. + val frame = if (smallView) imageBitmap + else rememberAnimatedImage(data, imageBitmap) { blurred.value || ModalManager.fullscreen.hasModalsOpen() } + ImageView(BitmapPainter(frame)) { if (getLoadedFilePath(file) != null) { ModalManager.fullscreen.showCustomModal(animated = false) { close -> ImageFullScreenView(imageProvider, close) diff --git a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/chat/item/ImageFullScreenView.desktop.kt b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/chat/item/ImageFullScreenView.desktop.kt index bd395c2c97..583aa8f52f 100644 --- a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/chat/item/ImageFullScreenView.desktop.kt +++ b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/chat/item/ImageFullScreenView.desktop.kt @@ -19,8 +19,10 @@ import kotlin.math.max @Composable actual fun FullScreenImageView(modifier: Modifier, data: ByteArray, imageBitmap: ImageBitmap) { + // Decoded once, as an animation recomposes this on every frame + val still = remember(data) { getBitmapFromByteArray(data, false) ?: MR.images.decentralized.image.toComposeImageBitmap() } Image( - getBitmapFromByteArray(data, false) ?: MR.images.decentralized.image.toComposeImageBitmap(), + rememberAnimatedImage(data, still), contentDescription = stringResource(MR.strings.image_descr), contentScale = ContentScale.Fit, modifier = modifier, diff --git a/apps/multiplatform/common/src/desktopTest/kotlin/chat/simplex/app/AnimatedImageBoundsTest.kt b/apps/multiplatform/common/src/desktopTest/kotlin/chat/simplex/app/AnimatedImageBoundsTest.kt new file mode 100644 index 0000000000..b79dd69779 --- /dev/null +++ b/apps/multiplatform/common/src/desktopTest/kotlin/chat/simplex/app/AnimatedImageBoundsTest.kt @@ -0,0 +1,252 @@ +package chat.simplex.app + +import chat.simplex.common.platform.MAX_SLOW_FRAME_DEBT +import chat.simplex.common.platform.frameWait +import chat.simplex.common.platform.fileSizeWithinBounds +import chat.simplex.common.platform.frameCountWithinBounds +import chat.simplex.common.platform.frameDuration +import chat.simplex.common.platform.looksAnimatable +import chat.simplex.common.platform.priorFrame +import chat.simplex.common.platform.rasterWithinBounds +import chat.simplex.common.platform.rebuiltFramesWithinBounds +import chat.simplex.common.platform.slowFrameDebt +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +// The bounds an animated image must satisfy, checked as arithmetic: skiko's native library is not on the +// test runtime classpath, and these numbers are the part that has to be right about someone else's file. +class AnimatedImageBoundsTest { + private val BYTES_PER_PIXEL = 4 // what a GIF or WebP decodes to + + @Test + fun testOrdinaryAnimationIsWithinBounds() { + assertTrue(rasterWithinBounds(64, 64, BYTES_PER_PIXEL)) + assertTrue(rasterWithinBounds(1244, 554, BYTES_PER_PIXEL)) + } + + @Test + fun testHugeDeclaredDimensionsAreRejected() { + // A 17GB raster, declared by a GIF of 35 bytes + assertFalse(rasterWithinBounds(65535, 65535, BYTES_PER_PIXEL)) + } + + @Test + fun testDimensionsOverRasterBudgetAreRejected() { + // Plausible-looking, but one raster of this size is ~64MB and a chat shows several at once + assertFalse(rasterWithinBounds(4000, 4000, BYTES_PER_PIXEL)) + } + + @Test + fun testAspectRatioIsBoundedOnEachSideSeparately() { + // Only 2.1MP, so the raster bound alone would animate this with a 65535-pixel scanline + assertFalse(rasterWithinBounds(65535, 32, BYTES_PER_PIXEL)) + assertFalse(rasterWithinBounds(32, 65535, BYTES_PER_PIXEL)) + assertTrue(rasterWithinBounds(3000, 500, BYTES_PER_PIXEL)) + assertTrue(rasterWithinBounds(4096, 900, BYTES_PER_PIXEL)) + } + + @Test + fun testBudgetBoundariesAreExact() { + assertTrue(rasterWithinBounds(1920, 1920, BYTES_PER_PIXEL)) + assertFalse(rasterWithinBounds(1921, 1920, BYTES_PER_PIXEL)) + assertFalse(rasterWithinBounds(4097, 100, BYTES_PER_PIXEL)) + } + + @Test + fun testEmptyDimensionsAreRejected() { + assertFalse(rasterWithinBounds(0, 64, BYTES_PER_PIXEL)) + assertFalse(rasterWithinBounds(64, 0, BYTES_PER_PIXEL)) + assertFalse(rasterWithinBounds(-1, 64, BYTES_PER_PIXEL)) + } + + @Test + fun testWiderColorTypesCountAgainstTheSameBudget() { + // The file chooses its color type, so the 1920x1920 that fits at four bytes is twice the raster at eight + assertFalse(rasterWithinBounds(1920, 1920, 8)) + assertTrue(rasterWithinBounds(1357, 1357, 8)) + // A color type claiming no bytes per pixel would otherwise make any raster look free + assertFalse(rasterWithinBounds(4096, 4096, 0)) + } + + @Test + fun testAnimatableContainersAreRecognized() { + assertTrue(looksAnimatable("GIF89a...".toByteArray())) + assertTrue(looksAnimatable("GIF87a...".toByteArray())) + assertTrue(looksAnimatable("RIFF????WEBPVP8X".toByteArray())) + } + + @Test + fun testPhotosNeverReachTheAnimationDecoder() { + assertFalse(looksAnimatable(bytes(0x89, 'P'.code, 'N'.code, 'G'.code, 0x0D, 0x0A, 0x1A, 0x0A))) + assertFalse(looksAnimatable(bytes(0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46))) + // A RIFF container that is not WebP, a wave file say + assertFalse(looksAnimatable("RIFF????WAVEfmt ".toByteArray())) + } + + @Test + fun testShortDataIsRejectedWithoutReadingPastTheEnd() { + assertFalse(looksAnimatable(ByteArray(0))) + assertFalse(looksAnimatable("GIF".toByteArray())) + // Long enough for the RIFF tag, too short for the format that follows it + assertFalse(looksAnimatable("RIFF".toByteArray())) + assertFalse(looksAnimatable("RIFF1234WEB".toByteArray())) + } + + @Test + fun testPriorFrameIsReusedOnlyWhenTheBitmapHoldsIt() { + assertEquals(4, priorFrame(5, 4)) + // An older required frame is no longer in the bitmap, which is also how a predecessor disposed to what + // came before it is skipped, as Skia never requires one + assertEquals(-1, priorFrame(5, 2)) + assertEquals(-1, priorFrame(5, -1)) + // For the first frame, -1 is both its required frame and no prior frame + assertEquals(-1, priorFrame(0, -1)) + } + + @Test + fun testOnlyFilesSmallEnoughToScanAreWithinBounds() { + assertTrue(fileSizeWithinBounds(0)) + // The largest animation in this repository + assertTrue(fileSizeWithinBounds(6_013_354)) + assertTrue(fileSizeWithinBounds(32 * 1024 * 1024)) + assertFalse(fileSizeWithinBounds(32 * 1024 * 1024 + 1)) + } + + @Test + fun testOnlyAnimationsWorthHoldingFramesForAreWithinBounds() { + assertFalse(frameCountWithinBounds(0)) + assertFalse(frameCountWithinBounds(1)) + assertFalse(frameCountWithinBounds(-1)) + assertTrue(frameCountWithinBounds(2)) + // The longest animation in this repository, and the bound itself + assertTrue(frameCountWithinBounds(1041)) + assertTrue(frameCountWithinBounds(10_000)) + assertFalse(frameCountWithinBounds(10_001)) + } + + @Test + fun testAnimationsThatRebuildNothingAreWithinBounds() { + // What a real animation looks like: each frame continues the one before it, so nothing is rebuilt + assertTrue(rebuiltFramesWithinBounds(IntArray(5000) { it - 1 })) + assertTrue(rebuiltFramesWithinBounds(intArrayOf(-1, -1, -1))) + } + + @Test + fun testShortRebuiltChainsAreWithinBounds() { + // A GIF disposing to what came before it: frame 2 continues frame 0, rebuilding two frames + assertTrue(rebuiltFramesWithinBounds(intArrayOf(-1, 0, 0, 2, 2, 4))) + } + + @Test + fun testLongRebuiltChainsAreRejected() { + // Alternating disposal makes every other frame rebuild the chain before it, which Skia recurses through: + // 8000 frames of that overflows the native stack and kills the app + val alternating = IntArray(8000) { if (it % 2 == 0) it - 2 else it - 1 } + assertFalse(rebuiltFramesWithinBounds(alternating)) + // The bound is on what a rebuild costs, not on how long the animation is + assertTrue(rebuiltFramesWithinBounds(IntArray(8000) { it - 1 })) + } + + @Test + fun testRebuiltChainBoundIsExact() { + fun chainOf(length: Int) = IntArray(length + 2) { if (it == length + 1) it - 2 else it - 1 } + assertTrue(rebuiltFramesWithinBounds(chainOf(64))) + assertFalse(rebuiltFramesWithinBounds(chainOf(65))) + } + + @Test + fun testFramesContinuingSomethingImpossibleStartTheirOwnChain() { + // A file is not trusted to say a frame continues itself, a later frame, or one that is not there + assertTrue(rebuiltFramesWithinBounds(IntArray(5000) { it })) + assertTrue(rebuiltFramesWithinBounds(IntArray(5000) { it + 1 })) + assertTrue(rebuiltFramesWithinBounds(IntArray(5000) { 9999 })) + } + + @Test + fun testAFrameCheaperThanItsDelayWaitsAsItAlwaysDid() { + assertEquals(70, frameWait(70, 0)) + assertEquals(70, frameWait(70, 2)) + assertEquals(70, frameWait(70, 70)) + } + + @Test + fun testAFrameDearerThanItsDelayIsWaitedOut() { + assertEquals(85, frameWait(20, 85)) + assertEquals(500, frameWait(20, 500)) + assertEquals(1000, frameWait(20, 1000)) + } + + @Test + fun testAStallIsNotWaitedOut() { + // Wall time counts a machine that suspended mid-decode, which the frame never spent + assertEquals(1000, frameWait(20, 30_000)) + assertEquals(1000, frameWait(20, 8L * 60 * 60 * 1000)) + assertEquals(5000, frameWait(5000, 30_000)) + } + + @Test + fun testTwoExpensiveFramesInARowStopTheAnimation() { + var debt = slowFrameDebt(0, tooSlow = true) + assertTrue(debt < MAX_SLOW_FRAME_DEBT) + debt = slowFrameDebt(debt, tooSlow = true) + assertTrue(debt >= MAX_SLOW_FRAME_DEBT) + } + + @Test + fun testAFrameThatOnlyOverranIsPaidOff() { + // One expensive frame among cheap ones is a busy machine, not an expensive animation + var debt = slowFrameDebt(0, tooSlow = true) + repeat(4) { debt = slowFrameDebt(debt, tooSlow = false) } + assertEquals(0, debt) + } + + @Test + fun testAlternatingExpensiveFramesStillStopTheAnimation() { + // Frames that alternate are never expensive twice in a row, which is what a count that resets would miss + var debt = 0 + var frames = 0 + while (debt < MAX_SLOW_FRAME_DEBT && frames < 100) { + debt = slowFrameDebt(debt, tooSlow = frames % 2 == 0) + frames++ + } + assertEquals(5, frames) + } + + @Test + fun testCheapFramesEarnNoCreditAgainstLaterExpensiveOnes() { + var debt = 0 + repeat(1000) { debt = slowFrameDebt(debt, tooSlow = false) } + assertEquals(0, debt) + debt = slowFrameDebt(debt, tooSlow = true) + debt = slowFrameDebt(debt, tooSlow = true) + assertTrue(debt >= MAX_SLOW_FRAME_DEBT) + } + + @Test + fun testFrameDurationSubstitutesTheDefaultForFramesInAHurry() { + // Skia reports a GIF delay in milliseconds, so "no delay" and "one centisecond" arrive as 0 and 10 + assertEquals(100, frameDuration(0)) + assertEquals(100, frameDuration(10)) + // Not expected from Skia, but read from the file + assertEquals(100, frameDuration(-1)) + } + + @Test + fun testFrameDurationKeepsAuthoredDelays() { + assertEquals(70, frameDuration(70)) + assertEquals(600, frameDuration(600)) + assertEquals(Int.MAX_VALUE.toLong(), frameDuration(Int.MAX_VALUE)) + } + + @Test + fun testFrameDurationRaisesDelaysBelowTheFloor() { + assertEquals(20, frameDuration(11)) + assertEquals(20, frameDuration(19)) + assertEquals(20, frameDuration(20)) + assertEquals(21, frameDuration(21)) + } + + private fun bytes(vararg values: Int): ByteArray = values.map { it.toByte() }.toByteArray() +} diff --git a/apps/multiplatform/product/gaps.md b/apps/multiplatform/product/gaps.md index 25535d8003..aae24bdca8 100644 --- a/apps/multiplatform/product/gaps.md +++ b/apps/multiplatform/product/gaps.md @@ -222,9 +222,10 @@ Desktop users cannot send voice messages. The record button either does nothing Several other Desktop features are also marked with `LALAL` placeholders: - **QR Code Scanner** (`QRCodeScanner.desktop.kt:12`) -- scanning QR codes is not implemented on Desktop -- **Animated Drawables** (`Utils.desktop.kt:179`) -- animated image support (e.g., GIF in-line rendering) is not implemented -- **Animated Chat Images** (`CIImageView.desktop.kt:19`) -- animated image rendering in chat items -- **isImage detection** (`Images.desktop.kt:168`) -- image type detection (implemented but marked as incomplete) +- **Animated Drawables** (`Utils.desktop.kt:236`) -- `getDrawableFromUri` returns null, so `isAnimImage` falls back to the file extension +- **isImage detection** (`Images.desktop.kt:189`) -- image type detection (implemented but marked as incomplete) + +Desktop cannot decode WebP in chat: `decodeBoundedBufferedImage` (`Utils.desktop.kt:191`) reads through ImageIO, which has no WebP reader, so a received `.webp` renders only as the sender's preview and never opens full screen, and a picked one is skipped. Wallpapers and link previews decode WebP, as they read through Skia instead (`Images.desktop.kt:204`). Received GIFs do animate in chat items and full screen; the animated image decoder accepts WebP but is never reached for it. --- diff --git a/apps/multiplatform/product/views/chat.md b/apps/multiplatform/product/views/chat.md index 64abda7ee6..7862574f04 100644 --- a/apps/multiplatform/product/views/chat.md +++ b/apps/multiplatform/product/views/chat.md @@ -55,7 +55,7 @@ Each type has a dedicated composable in `views/chat/item/`: | Type | Composable | Description | |---|---|---| | Text | `FramedItemView` | Rendered with markdown (bold, italic, code, links, `@mentions`) via `CIMarkdownText` | -| Image | `CIImageView` | Thumbnail with tap-to-fullscreen via `ImageFullScreenView` | +| Image | `CIImageView` | Thumbnail with tap-to-fullscreen via `ImageFullScreenView`; animated GIFs play inline and full screen | | Video | `CIVideoView` | Video thumbnail with play button; inline playback via `VideoPlayerHolder` | | Voice | `CIVoiceView` | Waveform visualization with playback controls and duration | | File | `CIFileView` | File icon, name, size; download/open actions with progress indicator | diff --git a/apps/multiplatform/spec/client/chat-view.md b/apps/multiplatform/spec/client/chat-view.md index 728ace4936..a6691c2878 100644 --- a/apps/multiplatform/spec/client/chat-view.md +++ b/apps/multiplatform/spec/client/chat-view.md @@ -202,6 +202,18 @@ Long-press or right-click opens a dropdown menu with context-sensitive actions ( | `InvalidJSON` | -- | `CIInvalidJSONView` | `CIInvalidJSONView.kt` | | `CIMemberCreatedContact` | -- | `CIMemberCreatedContactView` | `CIMemberCreatedContactView.kt` | +### Animated Images + +`SimpleAndAnimatedImageView` is `expect`/`actual`. Android delegates to coil, which drives the animation +itself. Desktop decodes frames with Skia's `Codec` in `platform/AnimatedImage.desktop.kt`, where +`rememberAnimatedImage(data, still, hidden)` returns the frame to draw and falls back to the still image when +the data is not an animation, exceeds the decode bounds, or fails before showing a frame. Decoding runs off the UI thread +on two threads of the shared pool, and pauses while the window is minimized or hidden, while the image is behind the +privacy blur, and while a full screen modal covers the chat. An animation whose frames cost too much to +decode stops on the frame it reached rather than falling back to the still. The chat list preview (`smallView`) stays a +still image. Only GIF reaches this path: desktop decodes stills with ImageIO, which has no WebP reader, so a +received `.webp` renders only as the sender's preview and never opens full screen. + --- ## 6. Context Menu Actions diff --git a/apps/multiplatform/spec/impact.md b/apps/multiplatform/spec/impact.md index f808cf31ba..3a96638310 100644 --- a/apps/multiplatform/spec/impact.md +++ b/apps/multiplatform/spec/impact.md @@ -424,6 +424,7 @@ Path prefix: `common/src/desktopMain/kotlin/chat/simplex/common/` | `platform/Videos.desktop.kt` | PC10 | Low | Desktop video utilities | | `platform/Notifications.desktop.kt` | PC18 | Low | Desktop notification setup | | `platform/Images.desktop.kt` | PC10 | Low | Desktop image processing | +| `platform/AnimatedImage.desktop.kt` | PC10 | Low | Desktop animated image frame decoding (bounded) | | `platform/PlatformTextField.desktop.kt` | PC4 | Low | Desktop text field actual implementation | | `platform/Share.desktop.kt` | PC10 | Low | Desktop clipboard/share | | `platform/Back.desktop.kt` | PC1 | Low | Desktop back navigation | diff --git a/plans/2026-08-11-desktop-animated-images.md b/plans/2026-08-11-desktop-animated-images.md new file mode 100644 index 0000000000..82a0257b28 --- /dev/null +++ b/plans/2026-08-11-desktop-animated-images.md @@ -0,0 +1,113 @@ +# Animated images on desktop + +## The problem + +`SimpleAndAnimatedImageView` on desktop drew a single `BitmapPainter` and carried the marker +`// LALAL make it animated too`. Android decodes animations with coil, iOS with SwiftyGif, and desktop showed +the first frame and stopped. `ImageFullScreenView` carried a matching marker over the image branch. + +## Why this shape + +**Skia's `Codec`, which skiko already puts on the desktop classpath.** No new dependency. It decodes both GIF +and animated WebP, reports per-frame durations and repeat counts, and supports random access into frames. + +**Not `components-animatedimage`** (declared and unused until this change removed it). Its `animate()` +ignores the result of `allocPixels` and decodes inside composition. A 35-byte GIF declaring 65535x65535 asks +for a 17GB raster; `allocPixels` returns false, and the following `readPixels` throws +`IllegalArgumentException` from inside the composition — a remote crash from anyone who can send a file. It +also decodes on the UI thread, measured at ~11ms per frame for a 1244x554 animation. + +## Bounds + +Everything below is decoded from bytes somebody else composed, so each bound answers a specific crafted +input, and anything outside them keeps showing the still image the chat already renders. Animation degrades +to a picture, never to an error, and failures are never alerted — an alert per malformed file would itself +let a sender disrupt the app. + +| Bound | What it answers | +| --- | --- | +| raster measured in bytes, sides multiplied as `Long` | `65535 * 65535` overflows `Int` to a negative number and would pass a naive budget check | +| per-side cap, independent of the raster bound | 65535x32 is only 2.1MP and would otherwise animate with a 65535-pixel scanline | +| bytes per pixel read from the codec | the file chooses its color type; the budget must not assume four bytes | +| file size checked before the bytes are copied natively | Skia copies the encoded bytes and scans them to count frames | +| magic-byte prefilter (`GIF8`, `RIFF....WEBP`) | photos are most of what a chat holds and none are animations; they never reach a second decoder | +| `allocPixels` result honoured | it reports failure by returning false, and reading into an unallocated bitmap throws | +| frame count bound | counting the frames also builds a table of them, which a file of minimal frames makes several times its own size, and the codec holds it for as long as the animation plays | +| rebuilt frame chain bound | a frame the codec is given no prior frame for is rebuilt from its whole chain, and Skia recurses to do it: frames alternating their disposal make that chain as long as the file likes, and 8000 frames of it overflows the native stack and kills the app, which no catch can prevent. Real animations rebuild nothing at all | +| destination allocated with a premultiplied alpha type | the codec reports the alpha type of the first frame, and a frame that has alpha cannot be read into an opaque bitmap | +| frame duration floor, and 100ms substituted for delays of 10ms and less | the frames a file is allowed can all declare no delay at all, and Skia reports the usual "as fast as possible" delay of one centisecond as 10ms | +| a frame is waited out for what it cost as well as what it asks for | one very expensive frame among cheap ones owes nothing once the cheap ones have paid the debt off, and held 96.7% of a decoder thread indefinitely; waiting out the cost leaves any animation about half of one | +| an animation that owes too much for its frames stops on the one it reached | frames that alternate expensive with cheap are never slow twice in a row, so a count that resets never stops them | +| every native call that reads the file is inside an exception boundary | the frame count, the frame table and the repeat count are read from it too | + +Long frame delays are honoured rather than clamped - they are the author's, and they cost only the codec, the +raster and the frame table staying alive while nothing decodes. + +## Cost, and the optimisations that were rejected + +A frame continues the one before it, and the codec has to be told that the bitmap already holds it. Without +that it decodes the whole chain back to the last independent frame, so a frame costs as much as its index and +a loop costs the square of the frame count. Measured over one loop of the GIFs in `images/`, decode only: + +| | frames | chain re-decoded | prior frame reused | +| --- | --- | --- | --- | +| files.gif | 196 | 5.93 ms/frame | 0.06 ms | +| connection.gif | 240 | 9.22 ms/frame | 0.09 ms | +| groups.gif | 309 | 9.10 ms/frame | 0.05 ms | +| user-addresses.gif | 1041 | 25.92 ms/frame, worst 77 ms | 0.04 ms | + +Pixels are identical either way. The cost of a frame is then its own, and an animation stops on the frame it +reached once it owes too much: a frame over 100ms counts double what a frame under it forgives. This is wall +time, so a single frame can overrun by being descheduled, and a busy machine should not turn a cheap +animation into a still - but a file whose frames alternate expensive and cheap is never slow twice in a row, +and a run of them is what a count that resets would miss. Measured on a 1920x1920 GIF of 400 such frames, which holds +67% of a core indefinitely against a count that resets. Frames tuned to stay just under the threshold owe +nothing at all, and one expensive frame among cheap enough ones owes nothing for long, which is why a frame +is also waited out for what it cost: a frame of 3s among four cheap ones drops from 96.7% of a decoder thread +to 49.3%, every frame at 99ms from 83.0% to 49.9%, and the GIFs in `images/` stay exactly where they were - +none of their frames decodes in as long as it asks to be shown, by three orders of magnitude. + +Two optimisations were measured and **rejected**. Both were measured before the prior frame was reused, so +their per-frame figures are against a decode that was two orders of magnitude more expensive; the conclusions +are kept because they are about ratios, but the numbers are worth taking again: + +- **Decoding at display size.** Scaled decode is supported at arbitrary sizes, but it costs CPU rather than + saving it: 2000x891 goes from 177.8ms to 300.3ms per frame (+68%) to save 59% of the raster — and it only + engages on the files that are already the most expensive. +- **Half-depth pixels.** Skia refuses `RGB_565` and `ARGB_4444` for GIF outright. It works only for opaque + WebP, at +11% decode for -50% raster, which does not justify a format-specific path. + +What was kept: decoding is confined to two threads of the shared pool, so untrusted decode work cannot starve +the long running calls that share it; and frames are only decoded while they can be seen — not while the app +is minimized or sits in the tray, not while the image is behind the privacy blur, where each frame would otherwise be +decoded, uploaded and then blurred away again for nobody, and not while a full screen modal covers the +chat, which is shown beside it rather than in place of it: the viewer would otherwise leave the same +animation decoding twice, and the rest of the chat decoding where nobody can see it. The chat list preview stays a still image for +the same reason: it is a 36sp box that the desktop layout keeps on screen the whole time, so animating it +would hold a raster and spend a frame of work per listed chat, without pause. + +## Verification + +- 20 000 fuzzed mutations (bit flips, truncations, header corruption) over a real corpus plus crafted hostile + files: no exception escapes the structure, no hangs. +- Frames advance, per-frame delays are read correctly, and the loop wraps back to frame 0 after a full cycle + with byte-identical pixels. +- An oversized animation is refused by the bounds and still renders through the existing still-image path. +- A GIF of 8000 frames alternating their disposal, which passes every other bound at an 8x8 raster, crashed + the process with SIGSEGV before the chain bound and is refused by it now, while the GIFs in `images/`, a + 1920x1920 animation and a GIF disposing to what came before it all still play. +- Every frame of the GIFs in `images/` decodes with the prior frame reused, with pixels identical to decoding + the chain, and a GIF whose first frame is opaque and disposed to the background decodes past its first frame + only into a premultiplied destination. +- Unit tests cover every bound as arithmetic - the raster, the frame count, the rebuilt chains, the frame + durations and the debt an expensive frame owes; skiko's native library is not on the test + runtime classpath, so decoding is measured with the library added to a standalone classpath. + +## Deliberately not in this change + +- **WebP still images do not decode on desktop at all.** Desktop decodes images with ImageIO, which has no + WebP reader, so a received `.webp` never loads and picking one to send is dropped. Both the chat item and + the full screen viewer reach this code only after that decode has succeeded, so until that separate fix + lands it is GIFs that animate in the app, and the WebP path here is exercised by measurement only. +- **The decode raster is left to the collector.** Releasing it explicitly needs to know which thread Compose + Desktop draws on, and skiko uses a different redrawer per platform; guessing risks a use-after-free.