desktop: video and audio players (#3052)

* desktop: video and audio players * making player working without preinstalled VLC * mac support * don't use vlc lib when not needed * updated jna version * changes in script * video player lazy loading * mac script changes * updated build gradle for preserving atrributes of file while copying * apply the same file stats on libs to make VLC checker happy * updated script * changes --------- Co-authored-by: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com>
2026-05-12 15:15:20 +00:00 · 2023-09-22 01:03:47 +08:00
parent 6de0ed4766
commit 2d7655281f
25 changed files with 787 additions and 187 deletions
@@ -29,6 +29,10 @@ fun initApp() {
  //testCrypto()
 }

+fun discoverVlcLibs(path: String) {
+  uk.co.caprica.vlcj.binding.LibC.INSTANCE.setenv("VLC_PLUGIN_PATH", path, 1)
+}
+
 private fun applyAppLocale() {
  val lang = ChatController.appPrefs.appLanguage.get()
  if (lang == null || lang == Locale.getDefault().language) return
@@ -21,6 +21,8 @@ actual val agentDatabaseFileName: String = "simplex_v1_agent.db"

 actual val databaseExportDir: File = tmpDir

+val vlcDir: File = File(System.getProperty("java.io.tmpdir") + File.separator + "simplex-vlc").also { it.deleteOnExit() }
+
 actual fun desktopOpenDatabaseDir() {
  if (Desktop.isDesktopSupported()) {
    try {
@@ -1,9 +1,17 @@
 package chat.simplex.common.platform

 import androidx.compose.runtime.MutableState
+import androidx.compose.runtime.mutableStateOf
 import chat.simplex.common.model.*
-import chat.simplex.common.views.usersettings.showInDevelopingAlert
-import kotlinx.coroutines.CoroutineScope
+import chat.simplex.common.views.helpers.AlertManager
+import chat.simplex.common.views.helpers.generalGetString
+import chat.simplex.res.MR
+import kotlinx.coroutines.*
+import uk.co.caprica.vlcj.player.base.MediaPlayer
+import uk.co.caprica.vlcj.player.base.State
+import uk.co.caprica.vlcj.player.component.AudioPlayerComponent
+import java.io.File
+import kotlin.math.max

 actual class RecorderNative: RecorderInterface {
  override fun start(onProgressUpdate: (position: Int?, finished: Boolean) -> Unit): String {
@@ -18,36 +26,187 @@ actual class RecorderNative: RecorderInterface {
 }

 actual object AudioPlayer: AudioPlayerInterface {
-  override fun play(fileSource: CryptoFile, audioPlaying: MutableState<Boolean>, progress: MutableState<Int>, duration: MutableState<Int>, resetOnEnd: Boolean) {
-    showInDevelopingAlert()
+  val player by lazy { AudioPlayerComponent().mediaPlayer() }
+
+  // Filepath: String, onProgressUpdate
+  private val currentlyPlaying: MutableState<Pair<CryptoFile, (position: Int?, state: TrackState) -> Unit>?> = mutableStateOf(null)
+  private var progressJob: Job? = null
+
+  enum class TrackState {
+    PLAYING, PAUSED, REPLACED
+  }
+
+  // Returns real duration of the track
+  private fun start(fileSource: CryptoFile, seek: Int? = null, onProgressUpdate: (position: Int?, state: TrackState) -> Unit): Int? {
+    val absoluteFilePath = getAppFilePath(fileSource.filePath)
+    if (!File(absoluteFilePath).exists()) {
+      Log.e(TAG, "No such file: ${fileSource.filePath}")
+      return null
+    }
+
+    VideoPlayerHolder.stopAll()
+    RecorderInterface.stopRecording?.invoke()
+    val current = currentlyPlaying.value
+    if (current == null || current.first != fileSource) {
+      stopListener()
+      player.stop()
+      runCatching {
+        if (fileSource.cryptoArgs != null) {
+          val tmpFile = fileSource.createTmpFileIfNeeded()
+          decryptCryptoFile(absoluteFilePath, fileSource.cryptoArgs, tmpFile.absolutePath)
+          player.media().prepare("file://${tmpFile.absolutePath}")
+        } else {
+          player.media().prepare("file://$absoluteFilePath")
+        }
+      }.onFailure {
+        Log.e(TAG, it.stackTraceToString())
+        AlertManager.shared.showAlertMsg(generalGetString(MR.strings.unknown_error), it.message)
+        return null
+      }
+    }
+    if (seek != null) player.seekTo(seek)
+    player.start()
+    currentlyPlaying.value = fileSource to onProgressUpdate
+    progressJob = CoroutineScope(Dispatchers.Default).launch {
+      onProgressUpdate(player.currentPosition, TrackState.PLAYING)
+      while(isActive && (player.isPlaying || player.status().state() == State.OPENING)) {
+        // Even when current position is equal to duration, the player has isPlaying == true for some time,
+        // so help to make the playback stopped in UI immediately
+        if (player.currentPosition == player.duration) {
+          onProgressUpdate(player.currentPosition, TrackState.PLAYING)
+          break
+        }
+        delay(50)
+        onProgressUpdate(player.currentPosition, TrackState.PLAYING)
+      }
+      onProgressUpdate(null, TrackState.PAUSED)
+      currentlyPlaying.value?.first?.deleteTmpFile()
+    }
+    return player.duration
+  }
+
+  private fun pause(): Int {
+    progressJob?.cancel()
+    progressJob = null
+    val position = player.currentPosition
+    player.pause()
+    return position
  }

  override fun stop() {
-    /*LALAL*/
+    if (currentlyPlaying.value == null) return
+    player.stop()
+    stopListener()
  }

-  override fun stop(item: ChatItem) {
-    /*LALAL*/
-  }
+  override fun stop(item: ChatItem) = stop(item.file?.fileName)

+  // FileName or filePath are ok
  override fun stop(fileName: String?) {
-    TODO("Not yet implemented")
+    if (fileName != null && currentlyPlaying.value?.first?.filePath?.endsWith(fileName) == true) {
+      stop()
+    }
+  }
+
+  private fun stopListener() {
+    val afterCoroutineCancel: CompletionHandler = {
+      // Notify prev audio listener about stop
+      currentlyPlaying.value?.second?.invoke(null, TrackState.REPLACED)
+      currentlyPlaying.value?.first?.deleteTmpFile()
+      currentlyPlaying.value = null
+    }
+    /** Preventing race by calling a code AFTER coroutine ends, so [TrackState] will be:
+     * [TrackState.PLAYING] -> [TrackState.PAUSED] -> [TrackState.REPLACED] (in this order)
+     * */
+    if (progressJob != null) {
+      progressJob?.invokeOnCompletion(afterCoroutineCancel)
+    } else {
+      afterCoroutineCancel(null)
+    }
+    progressJob?.cancel()
+    progressJob = null
+  }
+
+  override fun play(
+    fileSource: CryptoFile,
+    audioPlaying: MutableState<Boolean>,
+    progress: MutableState<Int>,
+    duration: MutableState<Int>,
+    resetOnEnd: Boolean,
+  ) {
+    if (progress.value == duration.value) {
+      progress.value = 0
+    }
+    val realDuration = start(fileSource, progress.value) { pro, state ->
+      if (pro != null) {
+        progress.value = pro
+      }
+      if (pro == null || pro == duration.value) {
+        audioPlaying.value = false
+        if (pro == duration.value) {
+          progress.value = if (resetOnEnd) 0 else duration.value
+        } else if (state == TrackState.REPLACED) {
+          progress.value = 0
+        }
+      }
+    }
+    audioPlaying.value = realDuration != null
+    // Update to real duration instead of what was received in ChatInfo
+    realDuration?.let { duration.value = it }
  }

  override fun pause(audioPlaying: MutableState<Boolean>, pro: MutableState<Int>) {
-    TODO("Not yet implemented")
+    pro.value = pause()
+    audioPlaying.value = false
  }

  override fun seekTo(ms: Int, pro: MutableState<Int>, filePath: String?) {
-    /*LALAL*/
+    pro.value = ms
+    if (currentlyPlaying.value?.first?.filePath == filePath) {
+      player.seekTo(ms)
+    }
  }

  override fun duration(unencryptedFilePath: String): Int? {
-    /*LALAL*/
-    return null
+    var res: Int? = null
+    try {
+      val helperPlayer = AudioPlayerComponent().mediaPlayer()
+      helperPlayer.media().startPaused("file://$unencryptedFilePath")
+      res = helperPlayer.duration
+      helperPlayer.stop()
+      helperPlayer.release()
+    } catch (e: Exception) {
+      Log.e(TAG, e.stackTraceToString())
+    }
+    return res
  }
 }

+val MediaPlayer.isPlaying: Boolean
+  get() = status().isPlaying
+
+fun MediaPlayer.seekTo(time: Int) {
+  controls().setTime(time.toLong())
+}
+
+fun MediaPlayer.start() {
+  controls().start()
+}
+
+fun MediaPlayer.pause() {
+  controls().pause()
+}
+
+fun MediaPlayer.stop() {
+  controls().stop()
+}
+
+private val MediaPlayer.currentPosition: Int
+  get() = max(0, status().time().toInt())
+
+val MediaPlayer.duration: Int
+  get() = media().info().duration().toInt()
+
 actual object SoundPlayer: SoundPlayerInterface {
  override fun start(scope: CoroutineScope, sound: Boolean) { /*LALAL*/ }
  override fun stop() { /*LALAL*/ }
@@ -3,51 +3,213 @@ package chat.simplex.common.platform
 import androidx.compose.runtime.MutableState
 import androidx.compose.runtime.mutableStateOf
 import androidx.compose.ui.graphics.ImageBitmap
-import chat.simplex.common.views.usersettings.showInDevelopingAlert
+import androidx.compose.ui.graphics.toComposeImageBitmap
+import chat.simplex.common.views.helpers.*
+import chat.simplex.res.MR
+import kotlinx.coroutines.*
+import uk.co.caprica.vlcj.player.base.MediaPlayer
+import uk.co.caprica.vlcj.player.component.CallbackMediaPlayerComponent
+import uk.co.caprica.vlcj.player.component.EmbeddedMediaPlayerComponent
+import java.awt.Component
+import java.io.File
 import java.net.URI
+import kotlin.math.max

-actual class VideoPlayer: VideoPlayerInterface {
-  actual companion object {
-    actual fun getOrCreate(
-      uri: URI,
-      gallery: Boolean,
-      defaultPreview: ImageBitmap,
-      defaultDuration: Long,
-      soundEnabled: Boolean
-    ): VideoPlayer = VideoPlayer().also {
-      it.preview.value = defaultPreview
-      it.duration.value = defaultDuration
-      it.soundEnabled.value = soundEnabled
-    }
-    actual fun enableSound(enable: Boolean, fileName: String?, gallery: Boolean): Boolean { /*TODO*/ return false }
-    actual fun release(uri: URI, gallery: Boolean, remove: Boolean) { /*TODO*/ }
-    actual fun stopAll() { /*LALAL*/ }
-    actual fun releaseAll() { /*LALAL*/ }
-  }
-
+actual class VideoPlayer actual constructor(
+  override val uri: URI,
+  override val gallery: Boolean,
+  private val defaultPreview: ImageBitmap,
+  defaultDuration: Long,
+  soundEnabled: Boolean
+): VideoPlayerInterface {
  override val soundEnabled: MutableState<Boolean> = mutableStateOf(false)
  override val brokenVideo: MutableState<Boolean> = mutableStateOf(false)
  override val videoPlaying: MutableState<Boolean> = mutableStateOf(false)
  override val progress: MutableState<Long> = mutableStateOf(0L)
  override val duration: MutableState<Long> = mutableStateOf(0L)
-  override val preview: MutableState<ImageBitmap> = mutableStateOf(ImageBitmap(0, 0))
+  override val preview: MutableState<ImageBitmap> = mutableStateOf(defaultPreview)
+
+  val mediaPlayerComponent = initializeMediaPlayerComponent()
+  val player by lazy { mediaPlayerComponent.mediaPlayer() }
+
+  init {
+    withBGApi {
+      setPreviewAndDuration()
+    }
+  }
+
+  private val currentVolume: Int by lazy { player.audio().volume() }
+  private var isReleased: Boolean = false
+
+  private val listener: MutableState<((position: Long?, state: TrackState) -> Unit)?> = mutableStateOf(null)
+  private var progressJob: Job? = null
+
+  enum class TrackState {
+    PLAYING, PAUSED, STOPPED
+  }
+
+  private fun start(seek: Long? = null, onProgressUpdate: (position: Long?, state: TrackState) -> Unit): Boolean {
+    val filepath = getAppFilePath(uri)
+    if (filepath == null || !File(filepath).exists()) {
+      Log.e(TAG, "No such file: $uri")
+      brokenVideo.value = true
+      return false
+    }
+
+    if (soundEnabled.value)  {
+      RecorderInterface.stopRecording?.invoke()
+    }
+    AudioPlayer.stop()
+    VideoPlayerHolder.stopAll()
+    val playerFilePath = uri.toString().replaceFirst("file:", "file://")
+    if (listener.value == null) {
+      runCatching {
+        player.media().prepare(playerFilePath)
+        if (seek != null) {
+          player.seekTo(seek.toInt())
+        }
+      }.onFailure {
+        Log.e(TAG, it.stackTraceToString())
+        AlertManager.shared.showAlertMsg(generalGetString(MR.strings.unknown_error), it.message)
+        brokenVideo.value = true
+        return false
+      }
+    }
+    player.start()
+    if (seek != null) player.seekTo(seek.toInt())
+    if (!player.isPlaying) {
+      // Can happen when video file is broken
+      AlertManager.shared.showAlertMsg(generalGetString(MR.strings.unknown_error))
+      brokenVideo.value = true
+      return false
+    }
+    listener.value = onProgressUpdate
+    // Player can only be accessed in one specific thread
+    progressJob = CoroutineScope(Dispatchers.Main).launch {
+      onProgressUpdate(player.currentPosition.toLong(), TrackState.PLAYING)
+      while (isActive && !isReleased && player.isPlaying) {
+        // Even when current position is equal to duration, the player has isPlaying == true for some time,
+        // so help to make the playback stopped in UI immediately
+        if (player.currentPosition == player.duration) {
+          onProgressUpdate(player.currentPosition.toLong(), TrackState.PLAYING)
+          break
+        }
+        delay(50)
+        onProgressUpdate(player.currentPosition.toLong(), TrackState.PLAYING)
+      }
+      if (isActive && !isReleased) {
+        onProgressUpdate(player.currentPosition.toLong(), TrackState.PAUSED)
+      }
+      onProgressUpdate(null, TrackState.PAUSED)
+    }
+
+    return true
+  }

  override fun stop() {
-    /*TODO*/
+    if (isReleased || !videoPlaying.value) return
+    player.controls().stop()
+    stopListener()
+  }
+
+  private fun stopListener() {
+    val afterCoroutineCancel: CompletionHandler = {
+      // Notify prev video listener about stop
+      listener.value?.invoke(null, TrackState.STOPPED)
+    }
+    /** Preventing race by calling a code AFTER coroutine ends, so [TrackState] will be:
+     * [TrackState.PLAYING] -> [TrackState.PAUSED] -> [TrackState.STOPPED] (in this order)
+     * */
+    if (progressJob != null) {
+      progressJob?.invokeOnCompletion(afterCoroutineCancel)
+    } else {
+      afterCoroutineCancel(null)
+    }
+    progressJob?.cancel()
+    progressJob = null
  }

  override fun play(resetOnEnd: Boolean) {
-    if (appPlatform.isDesktop) {
-      showInDevelopingAlert()
+    if (progress.value == duration.value) {
+      progress.value = 0
+    }
+    videoPlaying.value = start(progress.value) { pro, _ ->
+      if (pro != null) {
+        progress.value = pro
+      }
+      if ((pro == null || pro == duration.value) && duration.value != 0L) {
+        videoPlaying.value = false
+        if (pro == duration.value) {
+          progress.value = if (resetOnEnd) 0 else duration.value
+        }/* else if (state == TrackState.STOPPED) {
+          progress.value = 0 //
+        }*/
+      }
    }
  }

  override fun enableSound(enable: Boolean): Boolean {
-    /*TODO*/
-    return false
+    if (isReleased) return false
+    if (soundEnabled.value == enable) return false
+    soundEnabled.value = enable
+    player.audio().setVolume(if (enable) currentVolume else 0)
+    return true
  }

-  override fun release(remove: Boolean) {
-    /*TODO*/
+  override fun release(remove: Boolean) { withApi {
+    if (isReleased) return@withApi
+    isReleased = true
+    // TODO
+    /** [player.release] freezes thread for some reason. It happens periodically. So doing this we don't see the freeze, but it's still there */
+    if (player.isPlaying) player.stop()
+    CoroutineScope(Dispatchers.IO).launch { player.release() }
+    if (remove) {
+      VideoPlayerHolder.players.remove(uri to gallery)
+    }
+  }}
+
+  private val MediaPlayer.currentPosition: Int
+    get() = if (isReleased) 0 else max(0, player.status().time().toInt())
+
+  private suspend fun setPreviewAndDuration() {
+    // It freezes main thread, doing it in IO thread
+    CoroutineScope(Dispatchers.IO).launch {
+      val previewAndDuration = VideoPlayerHolder.previewsAndDurations.getOrPut(uri) { getBitmapFromVideo() }
+      withContext(Dispatchers.Main) {
+        preview.value = previewAndDuration.preview ?: defaultPreview
+        duration.value = (previewAndDuration.duration ?: 0)
+      }
+    }
+  }
+
+  private suspend fun getBitmapFromVideo(): VideoPlayerInterface.PreviewAndDuration {
+    val player = CallbackMediaPlayerComponent().mediaPlayer()
+    val filepath = getAppFilePath(uri)
+    if (filepath == null || !File(filepath).exists()) {
+      return VideoPlayerInterface.PreviewAndDuration(preview = defaultPreview, timestamp = 0L, duration = 0L)
+    }
+    player.media().startPaused(filepath)
+    val start = System.currentTimeMillis()
+    while (player.snapshots()?.get() == null && start + 5000 > System.currentTimeMillis()) {
+      delay(10)
+    }
+    val preview = player.snapshots()?.get()?.toComposeImageBitmap()
+    val duration = player.duration.toLong()
+    CoroutineScope(Dispatchers.IO).launch { player.release() }
+    return VideoPlayerInterface.PreviewAndDuration(preview = preview, timestamp = 0L, duration = duration)
+  }
+
+  private fun initializeMediaPlayerComponent(): Component {
+    return if (desktopPlatform.isMac()) {
+      CallbackMediaPlayerComponent()
+    } else {
+      EmbeddedMediaPlayerComponent()
+    }
+  }
+
+  private fun Component.mediaPlayer() = when (this) {
+    is CallbackMediaPlayerComponent -> mediaPlayer()
+    is EmbeddedMediaPlayerComponent -> mediaPlayer()
+    else -> error("mediaPlayer() can only be called on vlcj player components")
  }
 }
@@ -6,9 +6,7 @@ import androidx.compose.ui.unit.Dp
 import chat.simplex.common.platform.VideoPlayer

@Composable
-actual fun PlayerView(player: VideoPlayer, width: Dp, onClick: () -> Unit, onLongClick: () -> Unit, stop: () -> Unit) {
-  /* LALAL */
-}
+actual fun PlayerView(player: VideoPlayer, width: Dp, onClick: () -> Unit, onLongClick: () -> Unit, stop: () -> Unit) {}

@Composable
 actual fun LocalWindowWidth(): Dp {
@@ -1,14 +1,23 @@
 package chat.simplex.common.views.chat.item

-import androidx.compose.foundation.Image
-import androidx.compose.runtime.Composable
+import androidx.compose.foundation.*
+import androidx.compose.foundation.layout.*
+import androidx.compose.material.*
+import androidx.compose.runtime.*
+import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
+import androidx.compose.ui.awt.SwingPanel
 import androidx.compose.ui.graphics.*
 import androidx.compose.ui.layout.ContentScale
-import chat.simplex.common.platform.VideoPlayer
+import androidx.compose.ui.unit.dp
+import chat.simplex.common.platform.*
+import chat.simplex.common.simplexWindowState
 import chat.simplex.common.views.helpers.getBitmapFromByteArray
 import chat.simplex.res.MR
+import dev.icerock.moko.resources.compose.painterResource
 import dev.icerock.moko.resources.compose.stringResource
+import kotlinx.coroutines.delay
+import kotlin.math.max

@Composable
 actual fun FullScreenImageView(modifier: Modifier, data: ByteArray, imageBitmap: ImageBitmap) {
@@ -20,6 +29,43 @@ actual fun FullScreenImageView(modifier: Modifier, data: ByteArray, imageBitmap:
  )
 }
@Composable
-actual fun FullScreenVideoView(player: VideoPlayer, modifier: Modifier) {
-
+actual fun FullScreenVideoView(player: VideoPlayer, modifier: Modifier, close: () -> Unit) {
+  // Workaround. Without changing size of the window the screen flashes a lot even if it's not being recomposed
+  LaunchedEffect(Unit) {
+    simplexWindowState.windowState.size = simplexWindowState.windowState.size.copy(width = simplexWindowState.windowState.size.width + 1.dp)
+    delay(50)
+    player.play(true)
+    simplexWindowState.windowState.size = simplexWindowState.windowState.size.copy(width = simplexWindowState.windowState.size.width - 1.dp)
+  }
+  Box {
+    Box(Modifier.fillMaxSize().padding(bottom = 50.dp)) {
+      val factory = remember { { player.mediaPlayerComponent } }
+      SwingPanel(
+        background = Color.Transparent,
+        modifier = Modifier,
+        factory = factory
+      )
+    }
+    Controls(player, close)
+  }
+}
+
+@Composable
+private fun BoxScope.Controls(player: VideoPlayer, close: () -> Unit) {
+  val playing = remember(player) { player.videoPlaying }
+  val progress = remember(player) { player.progress }
+  val duration = remember(player) { player.duration }
+  Row(Modifier.fillMaxWidth().align(Alignment.BottomCenter).height(50.dp)) {
+    IconButton(onClick = { if (playing.value) player.player.pause() else player.play(true) },) {
+      Icon(painterResource(if (playing.value) MR.images.ic_pause_filled else MR.images.ic_play_arrow_filled), null, Modifier.size(30.dp), tint = MaterialTheme.colors.primary)
+    }
+    Slider(
+      value = progress.value.toFloat() / max(0.0001f, duration.value.toFloat()),
+      onValueChange = { player.player.seekTo((it * duration.value).toInt()) },
+      modifier = Modifier.fillMaxWidth().weight(1f)
+    )
+    IconButton(onClick = close,) {
+      Icon(painterResource(MR.images.ic_close), null, Modifier.size(30.dp), tint = MaterialTheme.colors.primary)
+    }
+  }
 }
@@ -133,7 +133,6 @@ actual suspend fun saveTempImageUncompressed(image: ImageBitmap, asPng: Boolean)
 }

 actual fun getBitmapFromVideo(uri: URI, timestamp: Long?, random: Boolean): VideoPlayerInterface.PreviewAndDuration {
-  // LALAL
  return VideoPlayerInterface.PreviewAndDuration(preview = null, timestamp = 0L, duration = 0L)
 }