From 30088f38c07683a581bbcb8ad6fd8b7ba89767fc Mon Sep 17 00:00:00 2001 From: catfromplan9 <104175360+catfromplan9@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:37:09 +0000 Subject: [PATCH] Implement support for generating animated thumbnails in the media thumbnailer (#18831) Signed-off-by: cat --- changelog.d/18831.feature | 1 + .../federation/transport/server/federation.py | 16 +- synapse/media/media_repository.py | 209 +++++++++++------- synapse/media/thumbnailer.py | 120 ++++++++-- synapse/rest/client/media.py | 57 +++-- synapse/rest/media/thumbnail_resource.py | 60 +++-- tests/media/test_media_storage.py | 70 +++++- tests/rest/client/test_media.py | 151 +++++++++++++ 8 files changed, 550 insertions(+), 134 deletions(-) create mode 100644 changelog.d/18831.feature diff --git a/changelog.d/18831.feature b/changelog.d/18831.feature new file mode 100644 index 0000000000..022105c0b5 --- /dev/null +++ b/changelog.d/18831.feature @@ -0,0 +1 @@ +Add animation support to the media thumbnailer, gated behind the `animated` query parameter on the thumbnail endpoint (defaults to off). diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index d783e6da51..8a8914bf4f 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -37,6 +37,7 @@ from synapse.federation.transport.server._base import ( BaseFederationServlet, ) from synapse.http.servlet import ( + parse_boolean, parse_boolean_from_args, parse_integer, parse_integer_from_args, @@ -46,7 +47,7 @@ from synapse.http.servlet import ( ) from synapse.http.site import SynapseRequest from synapse.media._base import DEFAULT_MAX_TIMEOUT_MS, MAXIMUM_ALLOWED_MAX_TIMEOUT_MS -from synapse.media.thumbnailer import ThumbnailProvider +from synapse.media.thumbnailer import ANIMATED_THUMBNAIL_TYPE, ThumbnailProvider from synapse.types import JsonDict from synapse.util import SYNAPSE_VERSION from synapse.util.ratelimitutils import FederationRateLimiter @@ -875,8 +876,9 @@ class FederationMediaThumbnailServlet(BaseFederationServerServlet): width = parse_integer(request, "width", required=True) height = parse_integer(request, "height", required=True) method = parse_string(request, "method", "scale") + animated = parse_boolean(request, "animated", default=False) # TODO Parse the Accept header to get an prioritised list of thumbnail types. - m_type = "image/png" + m_type = ANIMATED_THUMBNAIL_TYPE if animated else "image/png" max_timeout_ms = parse_integer( request, "timeout_ms", default=DEFAULT_MAX_TIMEOUT_MS ) @@ -884,7 +886,15 @@ class FederationMediaThumbnailServlet(BaseFederationServerServlet): if self.dynamic_thumbnails: await self.thumbnail_provider.select_or_generate_local_thumbnail( - request, media_id, width, height, method, m_type, max_timeout_ms, True + request, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + True, + animated=animated, ) else: await self.thumbnail_provider.respond_local_thumbnail( diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 3344e4c7be..bbd31976cb 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -68,7 +68,11 @@ from synapse.media.storage_provider import ( FileStorageProviderBackend, StorageProviderWrapper, ) -from synapse.media.thumbnailer import Thumbnailer, ThumbnailError +from synapse.media.thumbnailer import ( + ANIMATED_THUMBNAIL_TYPE, + Thumbnailer, + ThumbnailError, +) from synapse.media.url_previewer import UrlPreviewer from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia from synapse.types import UserID @@ -1127,6 +1131,7 @@ class MediaRepository: t_height: int, t_method: str, t_type: str, + animated: bool = False, ) -> BytesIO | None: m_width = thumbnailer.width m_height = thumbnailer.height @@ -1144,12 +1149,12 @@ class MediaRepository: m_width, m_height = thumbnailer.transpose() if t_method == "crop": - return thumbnailer.crop(t_width, t_height, t_type) + return thumbnailer.crop(t_width, t_height, t_type, animated=animated) elif t_method == "scale": t_width, t_height = thumbnailer.aspect(t_width, t_height) t_width = min(m_width, t_width) t_height = min(m_height, t_height) - return thumbnailer.scale(t_width, t_height, t_type) + return thumbnailer.scale(t_width, t_height, t_type, animated=animated) return None @@ -1161,6 +1166,7 @@ class MediaRepository: t_method: str, t_type: str, url_cache: bool, + animated: bool = False, ) -> tuple[str, FileInfo] | None: async with self.media_storage.ensure_media_is_in_local_cache( FileInfo(None, media_id, url_cache=url_cache) @@ -1186,6 +1192,7 @@ class MediaRepository: t_height, t_method, t_type, + animated, ) if t_byte_source: @@ -1199,7 +1206,7 @@ class MediaRepository: height=t_height, method=t_method, type=t_type, - length=t_byte_source.tell(), + length=t_byte_source.getbuffer().nbytes, ), ) @@ -1236,6 +1243,7 @@ class MediaRepository: t_height: int, t_method: str, t_type: str, + animated: bool = False, ) -> str | None: async with self.media_storage.ensure_media_is_in_local_cache( FileInfo(server_name, file_id) @@ -1262,6 +1270,7 @@ class MediaRepository: t_height, t_method, t_type, + animated, ) if t_byte_source: @@ -1274,7 +1283,7 @@ class MediaRepository: height=t_height, method=t_method, type=t_type, - length=t_byte_source.tell(), + length=t_byte_source.getbuffer().nbytes, ), ) @@ -1389,6 +1398,12 @@ class MediaRepository: ) # Now we generate the thumbnails for each dimension, store it + # + # For animated source images we also generate and cache an + # animated WebP thumbnail per (size, method), served only when a + # client requests `?animated=true`. These are deduplicated since + # the animated thumbnail is always WebP regardless of `t_type`. + animated_done: set[tuple[int, int, str]] = set() for (t_width, t_height, t_type), t_method in thumbnails.items(): # Generate the thumbnail if t_method == "crop": @@ -1411,81 +1426,127 @@ class MediaRepository: logger.error("Unrecognized method: %r", t_method) continue - if not t_byte_source: - continue + if t_byte_source: + await self._store_thumbnail( + server_name, + media_id, + file_id, + url_cache, + t_width, + t_height, + t_method, + t_type, + t_byte_source, + ) - file_info = FileInfo( - server_name=server_name, - file_id=file_id, - url_cache=url_cache, - thumbnail=ThumbnailInfo( - width=t_width, - height=t_height, - method=t_method, - type=t_type, - length=t_byte_source.tell(), - ), - ) - - async with self.media_storage.store_into_file(file_info) as ( - f, - fname, + if ( + thumbnailer.is_animated + and (t_width, t_height, t_method) not in animated_done ): - try: - await self.media_storage.write_to_file(t_byte_source, f) - finally: - t_byte_source.close() - - # We flush and close the file to ensure that the bytes have - # been written before getting the size. - f.flush() - f.close() - - t_len = os.path.getsize(fname) - - # Write to database - if server_name: - # Multiple remote media download requests can race (when - # using multiple media repos), so this may throw a violation - # constraint exception. If it does we'll delete the newly - # generated thumbnail from disk (as we're in the ctx - # manager). - # - # However: we've already called `finish()` so we may have - # also written to the storage providers. This is preferable - # to the alternative where we call `finish()` *after* this, - # where we could end up having an entry in the DB but fail - # to write the files to the storage providers. - try: - await self.store.store_remote_media_thumbnail( - server_name, - media_id, - file_id, - t_width, - t_height, - t_type, - t_method, - t_len, - ) - except Exception as e: - thumbnail_exists = ( - await self.store.get_remote_media_thumbnail( - server_name, - media_id, - t_width, - t_height, - t_type, - ) - ) - if not thumbnail_exists: - raise e - else: - await self.store.store_local_thumbnail( - media_id, t_width, t_height, t_type, t_method, t_len + animated_done.add((t_width, t_height, t_method)) + a_byte_source = await defer_to_thread( + self.hs.get_reactor(), + thumbnailer.crop + if t_method == "crop" + else thumbnailer.scale, + t_width, + t_height, + ANIMATED_THUMBNAIL_TYPE, + True, + ) + if a_byte_source: + await self._store_thumbnail( + server_name, + media_id, + file_id, + url_cache, + t_width, + t_height, + t_method, + ANIMATED_THUMBNAIL_TYPE, + a_byte_source, ) return {"width": m_width, "height": m_height} + async def _store_thumbnail( + self, + server_name: str | None, + media_id: str, + file_id: str, + url_cache: bool, + t_width: int, + t_height: int, + t_method: str, + t_type: str, + t_byte_source: BytesIO, + ) -> None: + """Store a generated thumbnail to the configured storage and database.""" + file_info = FileInfo( + server_name=server_name, + file_id=file_id, + url_cache=url_cache, + thumbnail=ThumbnailInfo( + width=t_width, + height=t_height, + method=t_method, + type=t_type, + length=t_byte_source.getbuffer().nbytes, + ), + ) + + async with self.media_storage.store_into_file(file_info) as (f, fname): + try: + await self.media_storage.write_to_file(t_byte_source, f) + finally: + t_byte_source.close() + + # We flush and close the file to ensure that the bytes have + # been written before getting the size. + f.flush() + f.close() + + t_len = os.path.getsize(fname) + + # Write to database + if server_name: + # Multiple remote media download requests can race (when + # using multiple media repos), so this may throw a violation + # constraint exception. If it does we'll delete the newly + # generated thumbnail from disk (as we're in the ctx + # manager). + # + # However: we've already called `finish()` so we may have + # also written to the storage providers. This is preferable + # to the alternative where we call `finish()` *after* this, + # where we could end up having an entry in the DB but fail + # to write the files to the storage providers. + try: + await self.store.store_remote_media_thumbnail( + server_name, + media_id, + file_id, + t_width, + t_height, + t_type, + t_method, + t_len, + ) + except Exception as e: + thumbnail_exists = await self.store.get_remote_media_thumbnail( + server_name, + media_id, + t_width, + t_height, + t_type, + ) + if not thumbnail_exists: + raise e + else: + await self.store.store_local_thumbnail( + media_id, t_width, t_height, t_type, t_method, t_len + ) + async def _apply_media_retention_rules(self) -> None: """ Purge old local and remote media according to the media retention rules diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index cb86a0ff66..d60ca16f26 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -20,11 +20,12 @@ # # import logging +from collections.abc import Callable from io import BytesIO from types import TracebackType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any -from PIL import Image +from PIL import Image, ImageSequence from synapse.api.errors import Codes, NotFoundError, SynapseError, cs_error from synapse.config.repository import THUMBNAIL_SUPPORTED_MEDIA_FORMAT_MAP @@ -49,6 +50,11 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +# Content type used when generating animated thumbnails. The spec recommends servers +# prefer WebP when supporting animation. See +# https://spec.matrix.org/v1.19/client-server-api/#get_matrixclientv1mediathumbnailservernamemediaid +ANIMATED_THUMBNAIL_TYPE = "image/webp" + EXIF_ORIENTATION_TAG = 0x0112 EXIF_TRANSPOSE_MAPPINGS = { 2: Image.FLIP_LEFT_RIGHT, @@ -66,7 +72,7 @@ class ThumbnailError(Exception): class Thumbnailer: - FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG"} + FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG", "image/webp": "WEBP"} # Which image formats we allow Pillow to open. # This should intentionally be kept restrictive, because the decoder of any @@ -143,33 +149,72 @@ class Thumbnailer: else: return max((max_height * self.width) // self.height, 1), max_height - def _resize(self, width: int, height: int) -> Image.Image: + def _resize_image(self, image: Image.Image, width: int, height: int) -> Image.Image: # 1-bit or 8-bit color palette images need converting to RGB # otherwise they will be scaled using nearest neighbour which # looks awful. # # If the image has transparency, use RGBA instead. - if self.image.mode in ["1", "L", "P"]: - if self.image.info.get("transparency", None) is not None: - with self.image: - self.image = self.image.convert("RGBA") + if image.mode in ["1", "L", "P"]: + if image.info.get("transparency", None) is not None: + converted = image.convert("RGBA") else: - with self.image: - self.image = self.image.convert("RGB") - return self.image.resize((width, height), Image.LANCZOS) + converted = image.convert("RGB") + else: + converted = image + return converted.resize((width, height), Image.LANCZOS) + + @property + def is_animated(self) -> bool: + return getattr(self.image, "is_animated", False) + + def _encode_animated_from( + self, transform: Callable[[Image.Image], Image.Image] + ) -> BytesIO: + """Apply `transform` to every frame of the source image and encode the + result as an animated thumbnail.""" + frames = [] + durations = [] + loop = self.image.info.get("loop", 0) + for frame in ImageSequence.Iterator(self.image): + # Copy the frame to avoid referencing the original image memory. + f = frame.copy() + if f.mode != "RGBA": + f = f.convert("RGBA") + frames.append(transform(f)) + # A duration of 0 is valid (interpretation is implementation-defined, + # see RFC 9649 section 2.7.1.1), so only fall back when unset. The + # 100ms default matches libwebp's animation tools. + duration = frame.info.get("duration") + if duration is None: + duration = self.image.info.get("duration", 100) + durations.append(duration) + return self._encode_animated(frames, durations, loop) @trace - def scale(self, width: int, height: int, output_type: str) -> BytesIO: + def scale( + self, width: int, height: int, output_type: str, animated: bool = False + ) -> BytesIO: """Rescales the image to the given dimensions. + If `animated` is set and the source image is animated, the animation is + preserved. Otherwise a static thumbnail of the first frame is produced. + Returns: The bytes of the encoded image ready to be written to disk """ - with self._resize(width, height) as scaled: + if animated and self.is_animated: + return self._encode_animated_from( + lambda f: self._resize_image(f, width, height) + ) + + with self._resize_image(self.image, width, height) as scaled: return self._encode_image(scaled, output_type) @trace - def crop(self, width: int, height: int, output_type: str) -> BytesIO: + def crop( + self, width: int, height: int, output_type: str, animated: bool = False + ) -> BytesIO: """Rescales and crops the image to the given dimensions preserving aspect:: (w_in / h_in) = (w_scaled / h_scaled) @@ -196,7 +241,14 @@ class Thumbnailer: crop_right = width + crop_left crop = (crop_left, 0, crop_right, height) - with self._resize(scaled_width, scaled_height) as scaled_image: + if animated and self.is_animated: + return self._encode_animated_from( + lambda f: self._resize_image(f, scaled_width, scaled_height).crop(crop) + ) + + with self._resize_image( + self.image, scaled_width, scaled_height + ) as scaled_image: with scaled_image.crop(crop) as cropped: return self._encode_image(cropped, output_type) @@ -206,6 +258,35 @@ class Thumbnailer: if fmt == "JPEG" or fmt == "PNG" and output_image.mode == "CMYK": output_image = output_image.convert("RGB") output_image.save(output_bytes_io, fmt, quality=80) + output_bytes_io.seek(0) + return output_bytes_io + + def _encode_animated( + self, + frames: list[Image.Image], + durations: list[int], + loop: int, + ) -> BytesIO: + """ + Encode a list of RGBA frames into an animated WebP, preserving per-frame + durations and the loop count. + """ + output_bytes_io = BytesIO() + if not frames: + raise ThumbnailError("No frames to encode for animated thumbnail") + + save_kwargs: dict[str, Any] = { + "format": "WEBP", + "save_all": True, + "append_images": frames[1:], + "loop": loop, + "duration": durations, + "minimize_size": True, + "quality": 80, + } + + frames[0].save(output_bytes_io, **save_kwargs) + output_bytes_io.seek(0) return output_bytes_io def close(self) -> None: @@ -318,6 +399,7 @@ class ThumbnailProvider: max_timeout_ms: int, for_federation: bool, allow_authenticated: bool = True, + animated: bool = False, ) -> None: media_info = await self.media_repo.get_local_media_info( request, media_id, max_timeout_ms @@ -379,6 +461,7 @@ class ThumbnailProvider: desired_method, desired_type, url_cache=bool(media_info.url_cache), + animated=animated, ) if thumbnail_result: @@ -413,6 +496,7 @@ class ThumbnailProvider: ip_address: str, use_federation: bool, allow_authenticated: bool = True, + animated: bool = False, ) -> None: media_info = await self.media_repo.get_remote_media_info( server_name, @@ -474,6 +558,7 @@ class ThumbnailProvider: desired_height, desired_method, desired_type, + animated=animated, ) if file_path: @@ -633,6 +718,9 @@ class ThumbnailProvider: # width/height/method so we can just call the "generate exact" # methods. + # A stored animated thumbnail is always of the animated type, so + # regenerate it as animated. + regen_animated = file_info.thumbnail.type == ANIMATED_THUMBNAIL_TYPE if server_name: await self.media_repo.generate_remote_exact_thumbnail( server_name, @@ -642,6 +730,7 @@ class ThumbnailProvider: t_height=file_info.thumbnail.height, t_method=file_info.thumbnail.method, t_type=file_info.thumbnail.type, + animated=regen_animated, ) else: await self.media_repo.generate_local_exact_thumbnail( @@ -651,6 +740,7 @@ class ThumbnailProvider: t_method=file_info.thumbnail.method, t_type=file_info.thumbnail.type, url_cache=url_cache, + animated=regen_animated, ) responder = await self.media_storage.fetch_media(file_info) diff --git a/synapse/rest/client/media.py b/synapse/rest/client/media.py index c740659cdd..c2e30b687b 100644 --- a/synapse/rest/client/media.py +++ b/synapse/rest/client/media.py @@ -36,7 +36,12 @@ from synapse.http.server import ( set_corp_headers, set_cors_headers, ) -from synapse.http.servlet import RestServlet, parse_integer, parse_string +from synapse.http.servlet import ( + RestServlet, + parse_boolean, + parse_integer, + parse_string, +) from synapse.http.site import SynapseRequest from synapse.media._base import ( DEFAULT_MAX_TIMEOUT_MS, @@ -45,7 +50,7 @@ from synapse.media._base import ( ) from synapse.media.media_repository import MediaRepository from synapse.media.media_storage import MediaStorage -from synapse.media.thumbnailer import ThumbnailProvider +from synapse.media.thumbnailer import ANIMATED_THUMBNAIL_TYPE, ThumbnailProvider from synapse.server import HomeServer from synapse.util.stringutils import parse_and_validate_server_name @@ -163,8 +168,9 @@ class ThumbnailResource(RestServlet): width = parse_integer(request, "width", required=True) height = parse_integer(request, "height", required=True) method = parse_string(request, "method", "scale") + animated = parse_boolean(request, "animated", default=False) # TODO Parse the Accept header to get an prioritised list of thumbnail types. - m_type = "image/png" + m_type = ANIMATED_THUMBNAIL_TYPE if animated else "image/png" max_timeout_ms = parse_integer( request, "timeout_ms", default=DEFAULT_MAX_TIMEOUT_MS ) @@ -181,6 +187,7 @@ class ThumbnailResource(RestServlet): m_type, max_timeout_ms, False, + animated=animated, ) else: await self.thumbnailer.respond_local_thumbnail( @@ -204,23 +211,33 @@ class ThumbnailResource(RestServlet): return ip_address = request.getClientAddress().host - remote_resp_function = ( - self.thumbnailer.select_or_generate_remote_thumbnail - if self.dynamic_thumbnails - else self.thumbnailer.respond_remote_thumbnail - ) - await remote_resp_function( - request, - server_name, - media_id, - width, - height, - method, - m_type, - max_timeout_ms, - ip_address, - True, - ) + if self.dynamic_thumbnails: + await self.thumbnailer.select_or_generate_remote_thumbnail( + request, + server_name, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + ip_address, + True, + animated=animated, + ) + else: + await self.thumbnailer.respond_remote_thumbnail( + request, + server_name, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + ip_address, + True, + ) self.media_repo.mark_recently_accessed(server_name, media_id) diff --git a/synapse/rest/media/thumbnail_resource.py b/synapse/rest/media/thumbnail_resource.py index 536fea4c32..35afb7ab54 100644 --- a/synapse/rest/media/thumbnail_resource.py +++ b/synapse/rest/media/thumbnail_resource.py @@ -25,7 +25,12 @@ import re from typing import TYPE_CHECKING from synapse.http.server import set_corp_headers, set_cors_headers -from synapse.http.servlet import RestServlet, parse_integer, parse_string +from synapse.http.servlet import ( + RestServlet, + parse_boolean, + parse_integer, + parse_string, +) from synapse.http.site import SynapseRequest from synapse.media._base import ( DEFAULT_MAX_TIMEOUT_MS, @@ -33,7 +38,7 @@ from synapse.media._base import ( respond_404, ) from synapse.media.media_storage import MediaStorage -from synapse.media.thumbnailer import ThumbnailProvider +from synapse.media.thumbnailer import ANIMATED_THUMBNAIL_TYPE, ThumbnailProvider from synapse.util.stringutils import parse_and_validate_server_name if TYPE_CHECKING: @@ -78,8 +83,9 @@ class ThumbnailResource(RestServlet): width = parse_integer(request, "width", required=True) height = parse_integer(request, "height", required=True) method = parse_string(request, "method", "scale") + animated = parse_boolean(request, "animated", default=False) # TODO Parse the Accept header to get an prioritised list of thumbnail types. - m_type = "image/png" + m_type = ANIMATED_THUMBNAIL_TYPE if animated else "image/png" max_timeout_ms = parse_integer( request, "timeout_ms", default=DEFAULT_MAX_TIMEOUT_MS ) @@ -97,6 +103,7 @@ class ThumbnailResource(RestServlet): max_timeout_ms, False, allow_authenticated=False, + animated=animated, ) else: await self.thumbnail_provider.respond_local_thumbnail( @@ -121,22 +128,33 @@ class ThumbnailResource(RestServlet): return ip_address = request.getClientAddress().host - remote_resp_function = ( - self.thumbnail_provider.select_or_generate_remote_thumbnail - if self.dynamic_thumbnails - else self.thumbnail_provider.respond_remote_thumbnail - ) - await remote_resp_function( - request, - server_name, - media_id, - width, - height, - method, - m_type, - max_timeout_ms, - ip_address, - use_federation=False, - allow_authenticated=False, - ) + if self.dynamic_thumbnails: + await self.thumbnail_provider.select_or_generate_remote_thumbnail( + request, + server_name, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + ip_address, + use_federation=False, + allow_authenticated=False, + animated=animated, + ) + else: + await self.thumbnail_provider.respond_remote_thumbnail( + request, + server_name, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + ip_address, + use_federation=False, + allow_authenticated=False, + ) self.media_repo.mark_recently_accessed(server_name, media_id) diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index 855a623ec0..54d1262054 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -52,7 +52,11 @@ from synapse.media.storage_provider import ( FileStorageProviderBackend, StorageProviderWrapper, ) -from synapse.media.thumbnailer import ThumbnailProvider +from synapse.media.thumbnailer import ( + ANIMATED_THUMBNAIL_TYPE, + Thumbnailer, + ThumbnailProvider, +) from synapse.module_api import ModuleApi from synapse.module_api.callbacks.spamchecker_callbacks import load_legacy_spam_checkers from synapse.rest import admin @@ -1407,3 +1411,67 @@ class MediaRepoSizeModuleCallbackTestCase(unittest.HomeserverTestCase): self.helper.upload_media(SMALL_PNG, tok=self.tok, expect_code=413) assert self.last_user_id == self.user assert self.last_size == len(SMALL_PNG) + + +def _make_animated_gif() -> bytes: + """Build a small two-frame animated GIF.""" + frames = [Image.new("RGB", (64, 64), color) for color in ((255, 0, 0), (0, 0, 255))] + out = BytesIO() + frames[0].save( + out, + format="GIF", + save_all=True, + append_images=frames[1:], + duration=100, + loop=0, + ) + return out.getvalue() + + +class ThumbnailerAnimatedTestCase(unittest.TestCase): + """Tests that the thumbnailer only animates when explicitly asked to.""" + + def setUp(self) -> None: + super().setUp() + self.tempdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tempdir, ignore_errors=True) + + self.gif_path = os.path.join(self.tempdir, "animated.gif") + with open(self.gif_path, "wb") as f: + f.write(_make_animated_gif()) + + self.png_path = os.path.join(self.tempdir, "static.png") + with open(self.png_path, "wb") as f: + f.write(SMALL_PNG) + + def test_scale_static_by_default(self) -> None: + """An animated source produces a static thumbnail unless animated=True.""" + with Thumbnailer(self.gif_path) as thumbnailer: + out = thumbnailer.scale(32, 32, "image/png") + result = Image.open(out) + self.assertFalse(getattr(result, "is_animated", False)) + + def test_scale_animated_when_requested(self) -> None: + """An animated source produces an animated thumbnail when animated=True.""" + with Thumbnailer(self.gif_path) as thumbnailer: + out = thumbnailer.scale(32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True) + result = Image.open(out) + self.assertEqual(result.format, "WEBP") + self.assertTrue(getattr(result, "is_animated", False)) + self.assertEqual(result.n_frames, 2) + + def test_crop_animated_when_requested(self) -> None: + with Thumbnailer(self.gif_path) as thumbnailer: + out = thumbnailer.crop(32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True) + result = Image.open(out) + self.assertEqual(result.format, "WEBP") + self.assertTrue(getattr(result, "is_animated", False)) + self.assertEqual(result.size, (32, 32)) + + def test_static_source_never_animates(self) -> None: + """A non-animated source stays static even when animated=True.""" + with Thumbnailer(self.png_path) as thumbnailer: + self.assertFalse(thumbnailer.is_animated) + out = thumbnailer.scale(1, 1, ANIMATED_THUMBNAIL_TYPE, animated=True) + result = Image.open(out) + self.assertFalse(getattr(result, "is_animated", False)) diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index 3409581c5e..8757e27d48 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -29,6 +29,7 @@ from unittest.mock import MagicMock, Mock, patch from urllib import parse from urllib.parse import quote, urlencode +from matrix_common.types.mxc_uri import MXCUri from parameterized import parameterized, parameterized_class from PIL import Image as Image @@ -3208,3 +3209,153 @@ class MediaUploadLimitsModuleOverrides(unittest.HomeserverTestCase): ) self.assertEqual(self.last_media_upload_limit_exceeded["sent_bytes"], 500) self.assertEqual(self.last_media_upload_limit_exceeded["attempted_bytes"], 800) + + +class AnimatedThumbnailTestCase(unittest.HomeserverTestCase): + """End-to-end tests for the `animated` query parameter on the local + thumbnail endpoint.""" + + servlets = [ + media.register_servlets, + login.register_servlets, + admin.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self.storage_path = self.mktemp() + self.media_store_path = self.mktemp() + os.mkdir(self.storage_path) + os.mkdir(self.media_store_path) + + config = self.default_config() + config["media_store_path"] = self.media_store_path + config["media_storage_providers"] = [ + { + "module": "synapse.media.storage_provider.FileStorageProviderBackend", + "store_local": True, + "store_synchronous": False, + "store_remote": True, + "config": {"directory": self.storage_path}, + } + ] + + return self.setup_test_homeserver(config=config) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.repo = hs.get_media_repository() + self.user = self.register_user("user", "pass") + self.tok = self.login("user", "pass") + + self.content_uri = self._upload(self._make_animated_gif(), "image/gif") + + def _make_animated_gif(self) -> bytes: + frames = [ + Image.new("RGB", (64, 64), color) for color in ((255, 0, 0), (0, 0, 255)) + ] + out = io.BytesIO() + frames[0].save( + out, + format="GIF", + save_all=True, + append_images=frames[1:], + duration=100, + loop=0, + ) + return out.getvalue() + + def _upload(self, data: bytes, content_type: str) -> MXCUri: + return self.get_success( + self.repo.create_or_update_content( + content_type, + "test", + io.BytesIO(data), + len(data), + UserID.from_string(self.user), + ) + ) + + def _thumbnail(self, content_uri: MXCUri, animated: str | None) -> FakeChannel: + params = "?width=32&height=32&method=scale" + if animated is not None: + params += f"&animated={animated}" + channel = self.make_request( + "GET", + f"/_matrix/client/v1/media/thumbnail/{content_uri.server_name}" + f"/{content_uri.media_id}{params}", + shorthand=False, + access_token=self.tok, + ) + self.pump() + self.assertEqual(channel.code, 200, channel.result) + return channel + + def test_default_is_static(self) -> None: + """Without the `animated` parameter a static thumbnail is returned.""" + channel = self._thumbnail(self.content_uri, animated=None) + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertFalse(getattr(result, "is_animated", False)) + + def test_animated_false_is_static(self) -> None: + channel = self._thumbnail(self.content_uri, animated="false") + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertFalse(getattr(result, "is_animated", False)) + + def test_animated_true_is_animated(self) -> None: + """With `animated=true` an animated WebP thumbnail is returned.""" + channel = self._thumbnail(self.content_uri, animated="true") + self.assertEqual( + channel.headers.getRawHeaders(b"Content-Type"), [b"image/webp"] + ) + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertEqual(result.format, "WEBP") + self.assertTrue(getattr(result, "is_animated", False)) + self.assertEqual(result.n_frames, 2) + + def test_animated_thumbnail_is_cached(self) -> None: + """Animated thumbnails are generated at upload and served from cache, + not regenerated on each request.""" + # The animated (WebP) thumbnail is stored alongside the static ones at + # upload time. + thumbnails = self.get_success( + self.store.get_local_media_thumbnails(self.content_uri.media_id) + ) + self.assertTrue( + any(info.type == "image/webp" for info in thumbnails), + thumbnails, + ) + + # Two consecutive requests return identical cached bytes. + first = self._thumbnail(self.content_uri, animated="true") + second = self._thumbnail(self.content_uri, animated="true") + self.assertEqual(first.result["body"], second.result["body"]) + + def test_non_animatable_source_falls_back_to_static(self) -> None: + """`animated=true` on a non-animatable source behaves as `animated=false`.""" + png_uri = self._upload(SMALL_PNG, "image/png") + channel = self._thumbnail(png_uri, animated="true") + self.assertEqual(channel.headers.getRawHeaders(b"Content-Type"), [b"image/png"]) + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertFalse(getattr(result, "is_animated", False)) + + @override_config({"dynamic_thumbnails": True}) + def test_dynamic_thumbnails_generates_and_caches_animated(self) -> None: + """With dynamic thumbnails, animated thumbnails are generated on demand + and then cached.""" + content_uri = self._upload(self._make_animated_gif(), "image/gif") + + channel = self._thumbnail(content_uri, animated="true") + self.assertEqual( + channel.headers.getRawHeaders(b"Content-Type"), [b"image/webp"] + ) + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertTrue(getattr(result, "is_animated", False)) + + # The on-demand generated thumbnail is now cached. + thumbnails = self.get_success( + self.store.get_local_media_thumbnails(content_uri.media_id) + ) + self.assertTrue( + any(info.type == "image/webp" for info in thumbnails), + thumbnails, + )