From a9355c2d88261a5f3a3e25f5c95b02956eab4e4c Mon Sep 17 00:00:00 2001 From: Avently <7953703+avently@users.noreply.github.com> Date: Tue, 9 Apr 2024 02:10:05 +0700 Subject: [PATCH] moved audio playing to js --- .../simplex/common/helpers/SoundPlayer.kt | 31 ------------------ .../common/platform/RecAndPlay.android.kt | 1 - .../common/views/call/CallView.android.kt | 13 ++------ .../simplex/common/platform/RecAndPlay.kt | 9 +---- .../www/audio}/call_sound_before_answer.mp3 | Bin .../commonMain/resources/assets/www/call.js | 16 +++++++++ .../common/platform/RecAndPlay.desktop.kt | 30 ----------------- .../common/views/call/CallView.desktop.kt | 2 +- packages/simplex-chat-webrtc/copy | 6 ++-- .../src/audio}/call_sound_before_answer.mp3 | Bin packages/simplex-chat-webrtc/src/call.ts | 17 ++++++++++ 11 files changed, 42 insertions(+), 83 deletions(-) rename apps/multiplatform/common/src/{androidMain/res/raw => commonMain/resources/assets/www/audio}/call_sound_before_answer.mp3 (100%) rename {apps/multiplatform/common/src/desktopMain/resources/media => packages/simplex-chat-webrtc/src/audio}/call_sound_before_answer.mp3 (100%) diff --git a/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/helpers/SoundPlayer.kt b/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/helpers/SoundPlayer.kt index a43fac29e3..2c42af3f86 100644 --- a/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/helpers/SoundPlayer.kt +++ b/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/helpers/SoundPlayer.kt @@ -36,37 +36,6 @@ object SoundPlayer: SoundPlayerInterface { } } - override fun stop() { - playing = false - player?.stop() - } -} - -object CallSoundsPlayer: CallSoundsPlayerInterface { - private var player: MediaPlayer? = null - private var playing = false - - override fun startWaitingAnswerSound(scope: CoroutineScope) { - player?.reset() - player = MediaPlayer().apply { - setAudioAttributes( - AudioAttributes.Builder() - .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION) - .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION_SIGNALLING) - .build() - ) - setDataSource(androidAppContext, Uri.parse("android.resource://" + androidAppContext.packageName + "/" + R.raw.call_sound_before_answer)) - prepare() - } - playing = true - scope.launch { - while (playing) { - player?.start() - delay(2000) - } - } - } - override fun vibrate() { val vibrator = ContextCompat.getSystemService(androidAppContext, Vibrator::class.java) val effect = VibrationEffect.createOneShot(50, VibrationEffect.DEFAULT_AMPLITUDE) diff --git a/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/platform/RecAndPlay.android.kt b/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/platform/RecAndPlay.android.kt index 5cb10ff070..5b0d3c778f 100644 --- a/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/platform/RecAndPlay.android.kt +++ b/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/platform/RecAndPlay.android.kt @@ -296,7 +296,6 @@ actual object AudioPlayer: AudioPlayerInterface { } actual typealias SoundPlayer = chat.simplex.common.helpers.SoundPlayer -actual typealias CallSoundsPlayer = chat.simplex.common.helpers.CallSoundsPlayer class CryptoMediaSource(val data: ByteArray) : MediaDataSource() { override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int { diff --git a/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/call/CallView.android.kt b/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/call/CallView.android.kt index 2a11c09c95..ad9897406e 100644 --- a/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/call/CallView.android.kt +++ b/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/call/CallView.android.kt @@ -83,12 +83,9 @@ actual fun ActiveCallView() { val wasConnected = rememberSaveable { mutableStateOf(false) } LaunchedEffect(call) { if (call?.callState == CallState.Connected && !wasConnected.value) { - CallSoundsPlayer.vibrate() + SoundPlayer.vibrate() wasConnected.value = true } - if ((call?.callState ?: CallState.WaitCapabilities) >= CallState.OfferReceived) { - CallSoundsPlayer.stop() - } } DisposableEffect(Unit) { val am = androidAppContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager @@ -118,14 +115,9 @@ actual fun ActiveCallView() { } } am.registerAudioDeviceCallback(audioCallback, null) - if (call?.callState == CallState.WaitCapabilities || call?.callState == CallState.InvitationSent) { - setCallSound(call.soundSpeaker, audioViaBluetooth) - CallSoundsPlayer.startWaitingAnswerSound(scope) - } onDispose { - CallSoundsPlayer.stop() if (wasConnected.value) { - CallSoundsPlayer.vibrate() + SoundPlayer.vibrate() } dropAudioManagerOverrides() am.unregisterAudioDeviceCallback(audioCallback) @@ -154,6 +146,7 @@ actual fun ActiveCallView() { val callType = CallType(call.localMedia, r.capabilities) chatModel.controller.apiSendCallInvitation(callRh, call.contact, callType) updateActiveCall(call) { it.copy(callState = CallState.InvitationSent, localCapabilities = r.capabilities) } + setCallSound(call.soundSpeaker, audioViaBluetooth) } is WCallResponse.Offer -> withBGApi { chatModel.controller.apiSendCallOffer(callRh, call.contact, r.offer, r.iceCandidates, call.localMedia, r.capabilities) diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/platform/RecAndPlay.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/platform/RecAndPlay.kt index 66c6bad03f..049a006135 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/platform/RecAndPlay.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/platform/RecAndPlay.kt @@ -36,15 +36,8 @@ expect object AudioPlayer: AudioPlayerInterface interface SoundPlayerInterface { fun start(scope: CoroutineScope, sound: Boolean) - fun stop() -} - -interface CallSoundsPlayerInterface { - fun startWaitingAnswerSound(scope: CoroutineScope) - fun stop() fun vibrate() + fun stop() } expect object SoundPlayer: SoundPlayerInterface - -expect object CallSoundsPlayer: CallSoundsPlayerInterface diff --git a/apps/multiplatform/common/src/androidMain/res/raw/call_sound_before_answer.mp3 b/apps/multiplatform/common/src/commonMain/resources/assets/www/audio/call_sound_before_answer.mp3 similarity index 100% rename from apps/multiplatform/common/src/androidMain/res/raw/call_sound_before_answer.mp3 rename to apps/multiplatform/common/src/commonMain/resources/assets/www/audio/call_sound_before_answer.mp3 diff --git a/apps/multiplatform/common/src/commonMain/resources/assets/www/call.js b/apps/multiplatform/common/src/commonMain/resources/assets/www/call.js index 5ea2f062b0..c33eb73c31 100644 --- a/apps/multiplatform/common/src/commonMain/resources/assets/www/call.js +++ b/apps/multiplatform/common/src/commonMain/resources/assets/www/call.js @@ -34,6 +34,17 @@ var useWorker = false; var isDesktop = false; var localizedState = ""; var localizedDescription = ""; +let callSoundBeforeAnswer = new Audio("../audio/call_sound_before_answer.mp3"); +var callSoundBeforeAnswerStopped = false; +callSoundBeforeAnswer.addEventListener("ended", function () { + setTimeout(() => { + if (!callSoundBeforeAnswerStopped) { + this.currentTime = 0; + this.play(); + } + callSoundBeforeAnswerStopped = false; + }, 1500); +}); const processCommand = (function () { const defaultIceServers = [ { urls: ["stun:stun.simplex.im:443"] }, @@ -210,6 +221,9 @@ const processCommand = (function () { if (command.media) await getLocalMediaStream(command.media, VideoCamera.User); const encryption = supportsInsertableStreams(useWorker); + setTimeout(() => { + callSoundBeforeAnswer.play(); + }, 1500); resp = { type: "capabilities", capabilities: { encryption } }; break; case "start": { @@ -261,6 +275,8 @@ const processCommand = (function () { const answer = await pc.createAnswer(); await pc.setLocalDescription(answer); addIceCandidates(pc, remoteIceCandidates); + callSoundBeforeAnswerStopped = true; + callSoundBeforeAnswer.pause(); // same as command for caller to use resp = { type: "answer", diff --git a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/platform/RecAndPlay.desktop.kt b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/platform/RecAndPlay.desktop.kt index e936b35789..4619c13f3c 100644 --- a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/platform/RecAndPlay.desktop.kt +++ b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/platform/RecAndPlay.desktop.kt @@ -223,36 +223,6 @@ actual object SoundPlayer: SoundPlayerInterface { } } - override fun stop() { - playing = false - AudioPlayer.stop() - } -} - -actual object CallSoundsPlayer: CallSoundsPlayerInterface { - private var player: MediaPlayer? = null - private var playing = false - - override fun startWaitingAnswerSound(scope: CoroutineScope) { - val tmpFile = File(tmpDir, UUID.randomUUID().toString()) - tmpFile.deleteOnExit() - SoundPlayer::class.java.getResource("/media/call_sound_before_answer.mp3").openStream()!!.use { it.copyTo(tmpFile.outputStream()) } - SoundPlayer.playing = true - scope.launch { - while (SoundPlayer.playing) { - AudioPlayer.play(CryptoFile.plain(tmpFile.absolutePath), mutableStateOf(true), mutableStateOf(0), mutableStateOf(0), true) - delay(3500) - } - } - playing = true - scope.launch { - while (playing) { - player?.start() - delay(3500) - } - } - } - override fun vibrate() {} override fun stop() { diff --git a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/call/CallView.desktop.kt b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/call/CallView.desktop.kt index 87c2d3e8f2..9e91828d8e 100644 --- a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/call/CallView.desktop.kt +++ b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/call/CallView.desktop.kt @@ -203,7 +203,7 @@ fun startServer(onResponse: (WVAPIMessage) -> Unit): NanoWSD { override fun openWebSocket(session: IHTTPSession): WebSocket = MyWebSocket(onResponse, session) fun resourcesToResponse(path: String): Response { - val uri = Class.forName("chat.simplex.common.AppKt").getResource("/assets/www$path") ?: return resourceNotFound + val uri = Class.forName("chat.simplex.common.AppKt").getResource("/assets/www${path.replace("/simplex", "")}") ?: return resourceNotFound val response = newFixedLengthResponse( Status.OK, getMimeTypeForFile(uri.file), uri.openStream().readBytes() diff --git a/packages/simplex-chat-webrtc/copy b/packages/simplex-chat-webrtc/copy index 770547b2cd..5574f02f4f 100755 --- a/packages/simplex-chat-webrtc/copy +++ b/packages/simplex-chat-webrtc/copy @@ -1,7 +1,7 @@ #!/bin/sh # it can be tested in the browser from dist folder -mkdir -p dist/{android,desktop,desktop/images} 2>/dev/null +mkdir -p dist/{android,desktop,desktop/images,audio} 2>/dev/null cp ./src/android/call.html ./dist/android/call.html cp ./src/android/style.css ./dist/android/style.css cp ./src/desktop/call.html ./dist/desktop/call.html @@ -10,9 +10,10 @@ cp ./src/desktop/images/* ./dist/desktop/images/ cp ./node_modules/lz-string/libs/lz-string.min.js ./dist/lz-string.min.js cp ./src/webcall.html ./dist/webcall.html cp ./src/ui.js ./dist/ui.js +cp ./src/audio/call_sound_before_answer.mp3 ./dist/audio/call_sound_before_answer.mp3 # copy to android and desktop apps -mkdir -p ../../apps/multiplatform/common/src/commonMain/resources/assets/www/{android,desktop,desktop/images} 2>/dev/null +mkdir -p ../../apps/multiplatform/common/src/commonMain/resources/assets/www/{android,desktop,desktop/images,audio} 2>/dev/null cp ./src/android/call.html ../../apps/multiplatform/common/src/commonMain/resources/assets/www/android/call.html cp ./src/android/style.css ../../apps/multiplatform/common/src/commonMain/resources/assets/www/android/style.css cp ./src/desktop/call.html ../../apps/multiplatform/common/src/commonMain/resources/assets/www/desktop/call.html @@ -21,4 +22,5 @@ cp ./src/desktop/images/* ../../apps/multiplatform/common/src/commonMain/resourc cp ./dist/desktop/ui.js ../../apps/multiplatform/common/src/commonMain/resources/assets/www/desktop/ui.js cp ./dist/call.js ../../apps/multiplatform/common/src/commonMain/resources/assets/www/call.js +cp ./dist/audio/call_sound_before_answer.mp3 ../../apps/multiplatform/common/src/commonMain/resources/assets/www/audio/call_sound_before_answer.mp3 cp ./node_modules/lz-string/libs/lz-string.min.js ../../apps/multiplatform/common/src/commonMain/resources/assets/www/lz-string.min.js diff --git a/apps/multiplatform/common/src/desktopMain/resources/media/call_sound_before_answer.mp3 b/packages/simplex-chat-webrtc/src/audio/call_sound_before_answer.mp3 similarity index 100% rename from apps/multiplatform/common/src/desktopMain/resources/media/call_sound_before_answer.mp3 rename to packages/simplex-chat-webrtc/src/audio/call_sound_before_answer.mp3 diff --git a/packages/simplex-chat-webrtc/src/call.ts b/packages/simplex-chat-webrtc/src/call.ts index accbebb22d..67a03aaee2 100644 --- a/packages/simplex-chat-webrtc/src/call.ts +++ b/packages/simplex-chat-webrtc/src/call.ts @@ -220,6 +220,18 @@ var isDesktop = false var localizedState = "" var localizedDescription = "" +let callSoundBeforeAnswer = new Audio("../audio/call_sound_before_answer.mp3") +var callSoundBeforeAnswerStopped = false +callSoundBeforeAnswer.addEventListener("ended", function () { + setTimeout(() => { + if (!callSoundBeforeAnswerStopped) { + this.currentTime = 0 + this.play() + } + callSoundBeforeAnswerStopped = false + }, 1500) +}) + const processCommand = (function () { type RTCRtpSenderWithEncryption = RTCRtpSender & { createEncodedStreams: () => TransformStream @@ -422,6 +434,9 @@ const processCommand = (function () { // This request for local media stream is made to prompt for camera/mic permissions on call start if (command.media) await getLocalMediaStream(command.media, VideoCamera.User) const encryption = supportsInsertableStreams(useWorker) + setTimeout(() => { + callSoundBeforeAnswer.play() + }, 1500) resp = {type: "capabilities", capabilities: {encryption}} break case "start": { @@ -470,6 +485,8 @@ const processCommand = (function () { const answer = await pc.createAnswer() await pc.setLocalDescription(answer) addIceCandidates(pc, remoteIceCandidates) + callSoundBeforeAnswerStopped = true + callSoundBeforeAnswer.pause() // same as command for caller to use resp = { type: "answer",