ui: send dropped .webm as video only when it has a video track (#7354)

* ui: send dropped .webm as video only when it has a video track .webm is as commonly an audio-only container as a video one, so the extension alone cannot tell whether there is a frame to embed. Read the container to decide: files with a video track are sent as video, the rest as files. Only done for files attached without the user saying how to send them (drag & drop, paste). An explicitly picked video is still sent as one, so .webm is now listed in the video file picker too. * plan: send dropped .webm as video only when it has a video track
2026-08-14 11:30:18 +00:00 · 2026-08-12 15:29:43 +01:00
parent 0a61b0ddea
commit abd5954675
7 changed files with 100 additions and 3 deletions
@@ -341,6 +341,19 @@ actual suspend fun getBitmapFromVideo(uri: URI, timestamp: Long?, random: Boolea
    VideoPlayerInterface.PreviewAndDuration(null, 0, 0)
  }

+actual suspend fun hasVideoTrack(uri: URI): Boolean {
+  val mmr = MediaMetadataRetriever()
+  return try {
+    mmr.setDataSource(androidAppContext, uri.toUri())
+    mmr.extractMetadata(MediaMetadataRetriever.METADATA_KEY_HAS_VIDEO) == "yes"
+  } catch (e: Exception) {
+    Log.e(TAG, "Utils.android hasVideoTrack error: ${e.message}")
+    false
+  } finally {
+    mmr.release()
+  }
+}
+
 actual fun ByteArray.toBase64StringForPassphrase(): String = Base64.encodeToString(this, Base64.DEFAULT)

 actual fun String.toByteArrayFromBase64ForPassphrase(): ByteArray = Base64.decode(this, Base64.DEFAULT)
@@ -285,7 +285,22 @@ expect fun AttachmentSelection(
 )

 fun MutableState<ComposeState>.onFilesAttached(uris: List<URI>) {
-  val groups = uris.groupBy { isImage(it) || isVideoUri(it) }
+  // The extension is enough to classify every format except .webm, which is just as commonly an
+  // audio-only container as a video one. An audio-only file has no frame to embed and is sent as a file,
+  // but that can only be told from the content, so reading it is deferred to a background thread.
+  // Only done here, where files arrive without the user saying how to send them (drag & drop, paste) -
+  // an explicitly picked video is still sent as one.
+  if (uris.none { isWebmUri(it) }) {
+    attachFiles(uris, emptySet())
+  } else {
+    CoroutineScope(Dispatchers.IO).launch {
+      attachFiles(uris, uris.filter { isWebmUri(it) && hasVideoTrack(it) }.toSet())
+    }
+  }
+}
+
+private fun MutableState<ComposeState>.attachFiles(uris: List<URI>, webmVideos: Set<URI>) {
+  val groups = uris.groupBy { isImage(it) || (isVideoUri(it) && (!isWebmUri(it) || it in webmVideos)) }
  val media = groups[true] ?: emptyList()
  val files = groups[false] ?: emptyList()
  if (media.isNotEmpty()) {
@@ -298,9 +313,12 @@ fun MutableState<ComposeState>.onFilesAttached(uris: List<URI>) {
 private fun isVideoUri(uri: URI): Boolean {
  val name = getFileName(uri)?.lowercase() ?: return false
  return name.endsWith(".mov") || name.endsWith(".avi") || name.endsWith(".mp4") ||
-      name.endsWith(".mpg") || name.endsWith(".mpeg") || name.endsWith(".mkv")
+      name.endsWith(".mpg") || name.endsWith(".mpeg") || name.endsWith(".mkv") ||
+      name.endsWith(".webm")
 }

+private fun isWebmUri(uri: URI): Boolean = getFileName(uri)?.lowercase()?.endsWith(".webm") == true
+
 fun MutableState<ComposeState>.processPickedFile(uri: URI?, text: String?) {
  if (uri != null) {
    val maxFileSize = value.maxFileSize
@@ -495,6 +495,9 @@ fun ciSenderProfile(ci: ChatItem, chatInfo: ChatInfo): LocalProfile? = when (val

 expect suspend fun getBitmapFromVideo(uri: URI, timestamp: Long? = null, random: Boolean = true, withAlertOnException: Boolean = true): VideoPlayerInterface.PreviewAndDuration

+// Whether the file really contains a video track. Reads container metadata only, without decoding a frame.
+expect suspend fun hasVideoTrack(uri: URI): Boolean
+
 fun showWrongUriAlert() {
  AlertManager.shared.showAlertMsg(
    title = generalGetString(MR.strings.non_content_uri_alert_title),
@@ -7,6 +7,10 @@ import chat.simplex.common.views.helpers.*
 import chat.simplex.res.MR
 import kotlinx.coroutines.*
 import org.jetbrains.compose.videoplayer.SkiaBitmapVideoSurface
+import uk.co.caprica.vlcj.media.Media
+import uk.co.caprica.vlcj.media.MediaEventAdapter
+import uk.co.caprica.vlcj.media.MediaParsedStatus
+import uk.co.caprica.vlcj.media.ParseFlag
 import uk.co.caprica.vlcj.media.VideoOrientation
 import uk.co.caprica.vlcj.player.base.*
 import uk.co.caprica.vlcj.player.component.CallbackMediaPlayerComponent
@@ -255,6 +259,43 @@ actual class VideoPlayer actual constructor(
      return@withContext VideoPlayerInterface.PreviewAndDuration(preview = preview, timestamp = 0L, duration = duration)
    }

+    // Parsing a local container header takes a few dozen ms, this is only a guard against a stuck parse
+    private const val PARSE_TIMEOUT_MS = 3000L
+
+    // Reads container metadata to tell whether there is a video track at all, without decoding a frame.
+    // libvlc signals the end of parsing with an event, so no polling or frame-decoding budget is needed.
+    suspend fun hasVideoTrack(uri: URI): Boolean = withContext(previewThread.asCoroutineDispatcher()) {
+      if (!uri.toFile().exists()) return@withContext false
+      val media = try {
+        vlcPreviewFactory.media().newMedia(uri.toFile().absolutePath)
+      } catch (e: Exception) {
+        Log.e(TAG, "hasVideoTrack unable to create media: ${e.stackTraceToString()}")
+        null
+      } ?: return@withContext false
+      try {
+        val parsed = CompletableDeferred<MediaParsedStatus?>()
+        media.events().addMediaEventListener(object: MediaEventAdapter() {
+          // vlcj maps an unknown status int to null, and a null here would throw on its event thread
+          override fun mediaParsedChanged(parsedMedia: Media?, newStatus: MediaParsedStatus?) {
+            parsed.complete(newStatus)
+          }
+        })
+        if (!media.parsing().parse(PARSE_TIMEOUT_MS.toInt(), ParseFlag.PARSE_LOCAL)) {
+          return@withContext false
+        }
+        if (withTimeoutOrNull(PARSE_TIMEOUT_MS) { parsed.await() } != MediaParsedStatus.DONE) {
+          media.parsing().stop()
+          return@withContext false
+        }
+        media.info().videoTracks().isNotEmpty()
+      } catch (e: Exception) {
+        Log.e(TAG, "hasVideoTrack error: ${e.stackTraceToString()}")
+        false
+      } finally {
+        media.release()
+      }
+    }
+
    val playerThread = Executors.newSingleThreadExecutor()
    private val previewThread = Executors.newSingleThreadExecutor()
    private val playersPool: ArrayList<Component> = ArrayList()
@@ -9,5 +9,6 @@ fun isVideo(uri: URI): Boolean {
      path.endsWith(".mp4") ||
      path.endsWith(".mpg") ||
      path.endsWith(".mpeg") ||
-      path.endsWith(".mkv")
+      path.endsWith(".mkv") ||
+      path.endsWith(".webm")
 }
@@ -255,6 +255,8 @@ actual suspend fun getBitmapFromVideo(uri: URI, timestamp: Long?, random: Boolea
  return VideoPlayer.getBitmapFromVideo(null, uri, withAlertOnException)
 }

+actual suspend fun hasVideoTrack(uri: URI): Boolean = VideoPlayer.hasVideoTrack(uri)
+
@OptIn(ExperimentalEncodingApi::class)
 actual fun ByteArray.toBase64StringForPassphrase(): String = Base64.encode(this)

@@ -0,0 +1,19 @@
+# Send dropped `.webm` as video only when it has a video track
+
+## Problem
+
+Dragging a `.webm` file onto the desktop compose area attaches it as a plain file instead of embedding it as a video with a preview frame and duration. Every other video container the app recognises (`.mov`, `.avi`, `.mp4`, `.mpg`, `.mpeg`, `.mkv`) embeds. The same omission hides `.webm` from the "Attach → video" file picker, so the only way to send one is "Choose file", which sends it as a document.
+
+## Cause
+
+`isVideoUri` (`apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/ComposeView.kt:298`) classifies attachments by file extension and does not list `.webm`; the desktop picker filter `isVideo` (`apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/platform/Videos.desktop.kt:5`) repeats the same list with the same omission. `onFilesAttached` groups the dropped URIs by `isImage(it) || isVideoUri(it)`, so a `.webm` fails both predicates, falls into the files group and reaches `processPickedFile`, which builds a `ComposePreview.FilePreview`.
+
+Adding the extension to both lists is not sufficient on its own. Unlike the other containers, `.webm` is used about as often for audio alone as for video — it is `MediaRecorder`'s default audio container, and Opus/Vorbis in WebM is widespread on the web. An audio-only file classified as video reaches the video branch of `processPickedMedia`, where `getBitmapFromVideo` finds no video track, returns a null preview and raises the "video decoding" alert; the item is then skipped and nothing is attached at all (`ComposeView.kt:366-376`). That is strictly worse than the file attachment the same drop produced before.
+
+## Fix
+
+Add `.webm` to both extension lists, and for `.webm` alone decide from the file's content rather than its name. A new `expect suspend fun hasVideoTrack(uri)` (`views/helpers/Utils.kt`) reports whether the container declares a video track, reading metadata only and never decoding a frame. On desktop it is implemented with libvlc's media parse (`platform/VideoPlayer.desktop.kt`), which signals completion with an event rather than a poll, so no frame-decoding budget is needed; measured at 12-346 ms across VP8, VP9, AV1, alpha and a 42 MB file, with a 3 s timeout as a guard against a stuck parse. On Android it uses `MediaMetadataRetriever.METADATA_KEY_HAS_VIDEO`. Either implementation answering "no", or failing, attaches the file as a file, which is always safe.
+
+`onFilesAttached` consults it only when a `.webm` is actually among the dropped URIs; every other attachment keeps the original synchronous code path on the caller thread, so the change adds no latency and no threading difference to images, documents or the other video containers. Files with a video track are sent as video, the rest as files.
+
+The content check is applied only where the user has not said how the file should be sent — drag & drop and paste. An explicitly picked video is still trusted: selecting an audio-only `.webm` through "Attach → video" raises the existing decoding error, which matches how the other containers already behave.