ui: send dropped .webm as video only when it has a video track (#7354)

* ui: send dropped .webm as video only when it has a video track .webm is as commonly an audio-only container as a video one, so the extension alone cannot tell whether there is a frame to embed. Read the container to decide: files with a video track are sent as video, the rest as files. Only done for files attached without the user saying how to send them (drag & drop, paste). An explicitly picked video is still sent as one, so .webm is now listed in the video file picker too. * plan: send dropped .webm as video only when it has a video track
2026-09-16 23:24:58 +00:00 · 2026-08-12 15:29:43 +01:00
parent 0a61b0ddea
commit abd5954675
7 changed files with 100 additions and 3 deletions
@@ -341,6 +341,19 @@ actual suspend fun getBitmapFromVideo(uri: URI, timestamp: Long?, random: Boolea
    VideoPlayerInterface.PreviewAndDuration(null, 0, 0)
  }

+actual suspend fun hasVideoTrack(uri: URI): Boolean {
+  val mmr = MediaMetadataRetriever()
+  return try {
+    mmr.setDataSource(androidAppContext, uri.toUri())
+    mmr.extractMetadata(MediaMetadataRetriever.METADATA_KEY_HAS_VIDEO) == "yes"
+  } catch (e: Exception) {
+    Log.e(TAG, "Utils.android hasVideoTrack error: ${e.message}")
+    false
+  } finally {
+    mmr.release()
+  }
+}
+
 actual fun ByteArray.toBase64StringForPassphrase(): String = Base64.encodeToString(this, Base64.DEFAULT)

 actual fun String.toByteArrayFromBase64ForPassphrase(): ByteArray = Base64.decode(this, Base64.DEFAULT)
@@ -285,7 +285,22 @@ expect fun AttachmentSelection(
 )

 fun MutableState<ComposeState>.onFilesAttached(uris: List<URI>) {
-  val groups = uris.groupBy { isImage(it) || isVideoUri(it) }
+  // The extension is enough to classify every format except .webm, which is just as commonly an
+  // audio-only container as a video one. An audio-only file has no frame to embed and is sent as a file,
+  // but that can only be told from the content, so reading it is deferred to a background thread.
+  // Only done here, where files arrive without the user saying how to send them (drag & drop, paste) -
+  // an explicitly picked video is still sent as one.
+  if (uris.none { isWebmUri(it) }) {
+    attachFiles(uris, emptySet())
+  } else {
+    CoroutineScope(Dispatchers.IO).launch {
+      attachFiles(uris, uris.filter { isWebmUri(it) && hasVideoTrack(it) }.toSet())
+    }
+  }
+}
+
+private fun MutableState<ComposeState>.attachFiles(uris: List<URI>, webmVideos: Set<URI>) {
+  val groups = uris.groupBy { isImage(it) || (isVideoUri(it) && (!isWebmUri(it) || it in webmVideos)) }
  val media = groups[true] ?: emptyList()
  val files = groups[false] ?: emptyList()
  if (media.isNotEmpty()) {
@@ -298,9 +313,12 @@ fun MutableState<ComposeState>.onFilesAttached(uris: List<URI>) {
 private fun isVideoUri(uri: URI): Boolean {
  val name = getFileName(uri)?.lowercase() ?: return false
  return name.endsWith(".mov") || name.endsWith(".avi") || name.endsWith(".mp4") ||
-      name.endsWith(".mpg") || name.endsWith(".mpeg") || name.endsWith(".mkv")
+      name.endsWith(".mpg") || name.endsWith(".mpeg") || name.endsWith(".mkv") ||
+      name.endsWith(".webm")
 }

+private fun isWebmUri(uri: URI): Boolean = getFileName(uri)?.lowercase()?.endsWith(".webm") == true
+
 fun MutableState<ComposeState>.processPickedFile(uri: URI?, text: String?) {
  if (uri != null) {
    val maxFileSize = value.maxFileSize
@@ -495,6 +495,9 @@ fun ciSenderProfile(ci: ChatItem, chatInfo: ChatInfo): LocalProfile? = when (val

 expect suspend fun getBitmapFromVideo(uri: URI, timestamp: Long? = null, random: Boolean = true, withAlertOnException: Boolean = true): VideoPlayerInterface.PreviewAndDuration

+// Whether the file really contains a video track. Reads container metadata only, without decoding a frame.
+expect suspend fun hasVideoTrack(uri: URI): Boolean
+
 fun showWrongUriAlert() {
  AlertManager.shared.showAlertMsg(
    title = generalGetString(MR.strings.non_content_uri_alert_title),
@@ -7,6 +7,10 @@ import chat.simplex.common.views.helpers.*
 import chat.simplex.res.MR
 import kotlinx.coroutines.*
 import org.jetbrains.compose.videoplayer.SkiaBitmapVideoSurface
+import uk.co.caprica.vlcj.media.Media
+import uk.co.caprica.vlcj.media.MediaEventAdapter
+import uk.co.caprica.vlcj.media.MediaParsedStatus
+import uk.co.caprica.vlcj.media.ParseFlag
 import uk.co.caprica.vlcj.media.VideoOrientation
 import uk.co.caprica.vlcj.player.base.*
 import uk.co.caprica.vlcj.player.component.CallbackMediaPlayerComponent
@@ -255,6 +259,43 @@ actual class VideoPlayer actual constructor(
      return@withContext VideoPlayerInterface.PreviewAndDuration(preview = preview, timestamp = 0L, duration = duration)
    }

+    // Parsing a local container header takes a few dozen ms, this is only a guard against a stuck parse
+    private const val PARSE_TIMEOUT_MS = 3000L
+
+    // Reads container metadata to tell whether there is a video track at all, without decoding a frame.
+    // libvlc signals the end of parsing with an event, so no polling or frame-decoding budget is needed.
+    suspend fun hasVideoTrack(uri: URI): Boolean = withContext(previewThread.asCoroutineDispatcher()) {
+      if (!uri.toFile().exists()) return@withContext false
+      val media = try {
+        vlcPreviewFactory.media().newMedia(uri.toFile().absolutePath)
+      } catch (e: Exception) {
+        Log.e(TAG, "hasVideoTrack unable to create media: ${e.stackTraceToString()}")
+        null
+      } ?: return@withContext false
+      try {
+        val parsed = CompletableDeferred<MediaParsedStatus?>()
+        media.events().addMediaEventListener(object: MediaEventAdapter() {
+          // vlcj maps an unknown status int to null, and a null here would throw on its event thread
+          override fun mediaParsedChanged(parsedMedia: Media?, newStatus: MediaParsedStatus?) {
+            parsed.complete(newStatus)
+          }
+        })
+        if (!media.parsing().parse(PARSE_TIMEOUT_MS.toInt(), ParseFlag.PARSE_LOCAL)) {
+          return@withContext false
+        }
+        if (withTimeoutOrNull(PARSE_TIMEOUT_MS) { parsed.await() } != MediaParsedStatus.DONE) {
+          media.parsing().stop()
+          return@withContext false
+        }
+        media.info().videoTracks().isNotEmpty()
+      } catch (e: Exception) {
+        Log.e(TAG, "hasVideoTrack error: ${e.stackTraceToString()}")
+        false
+      } finally {
+        media.release()
+      }
+    }
+
    val playerThread = Executors.newSingleThreadExecutor()
    private val previewThread = Executors.newSingleThreadExecutor()
    private val playersPool: ArrayList<Component> = ArrayList()
@@ -9,5 +9,6 @@ fun isVideo(uri: URI): Boolean {
      path.endsWith(".mp4") ||
      path.endsWith(".mpg") ||
      path.endsWith(".mpeg") ||
-      path.endsWith(".mkv")
+      path.endsWith(".mkv") ||
+      path.endsWith(".webm")
 }
@@ -255,6 +255,8 @@ actual suspend fun getBitmapFromVideo(uri: URI, timestamp: Long?, random: Boolea
  return VideoPlayer.getBitmapFromVideo(null, uri, withAlertOnException)
 }

+actual suspend fun hasVideoTrack(uri: URI): Boolean = VideoPlayer.hasVideoTrack(uri)
+
@OptIn(ExperimentalEncodingApi::class)
 actual fun ByteArray.toBase64StringForPassphrase(): String = Base64.encode(this)