From fc0879ebb7249bb5e6a02d1dc4f59a60c6887382 Mon Sep 17 00:00:00 2001 From: Stanislav Dmitrenko <7953703+avently@users.noreply.github.com> Date: Sat, 28 Sep 2024 04:04:16 +0700 Subject: [PATCH] android, desktop: fix Safari sound (#4947) * android, desktop: fix Safari sound * another approach * test * Revert "test" This reverts commit f89a30a88eef665b29191652b8ba4533f048b513. * Revert "another approach" This reverts commit 824ab7047c9b2c9c359bb16a6576b0cb51a9c1f6. * Revert "android, desktop: fix Safari sound" This reverts commit 80a866d4726f78c9760fb8a775dbe924a76123dd. * android, desktop: fix Safari sound * dependencies --- .../commonMain/resources/assets/www/call.js | 34 ++++++++++-------- packages/simplex-chat-webrtc/package.json | 4 +-- packages/simplex-chat-webrtc/src/call.ts | 36 +++++++++++-------- 3 files changed, 43 insertions(+), 31 deletions(-) diff --git a/apps/multiplatform/common/src/commonMain/resources/assets/www/call.js b/apps/multiplatform/common/src/commonMain/resources/assets/www/call.js index b5dcc9a2c2..97d157b762 100644 --- a/apps/multiplatform/common/src/commonMain/resources/assets/www/call.js +++ b/apps/multiplatform/common/src/commonMain/resources/assets/www/call.js @@ -186,6 +186,7 @@ const processCommand = (function () { localStream, localScreenStream, remoteStream, + remoteTracks: new Map(), remoteScreenStream, peerMediaSources: { mic: false, @@ -548,14 +549,6 @@ const processCommand = (function () { call.worker = new Worker(URL.createObjectURL(new Blob([workerCode], { type: "text/javascript" }))); call.worker.onerror = ({ error, filename, lineno, message }) => console.log({ error, filename, lineno, message }); // call.worker.onmessage = ({data}) => console.log(JSON.stringify({message: data})) - call.worker.onmessage = ({ data }) => { - console.log(JSON.stringify({ message: data })); - const transceiverMid = data.transceiverMid; - const mute = data.mute; - if (transceiverMid && mute != undefined) { - onMediaMuteUnmute(transceiverMid, mute); - } - }; } } } @@ -661,12 +654,7 @@ const processCommand = (function () { } setupMuteUnmuteListener(event.transceiver, track); const mediaSource = mediaSourceFromTransceiverMid(event.transceiver.mid); - if (mediaSource == CallMediaSource.ScreenAudio || mediaSource == CallMediaSource.ScreenVideo) { - call.remoteScreenStream.addTrack(track); - } - else { - call.remoteStream.addTrack(track); - } + call.remoteTracks.set(mediaSource, track); console.log(`ontrack success`); } catch (e) { @@ -1023,10 +1011,26 @@ const processCommand = (function () { if (!mute) videos.remoteScreen.play().catch((e) => console.log(e)); } + if (!mute) + addRemoteTracksWhenUnmuted(source, activeCall); localOrPeerMediaSourcesChanged(activeCall); // Make sure that remote camera and remote screen video in their places and shown/hidden based on layout type currently in use changeLayout(activeCall.layout); } + /* + When new remote tracks are coming, they don't get added to remote streams. They are stored in a map and once any of them "unmuted", + that track is added to the stream. Such workaround needed because Safari doesn't play one stream + if another one is not playing too, eg. no audio if only audio is playing while video track is present too but muted. + But we have possibility to have only one currently active track, even no active track at all. + */ + function addRemoteTracksWhenUnmuted(source, call) { + const track = call.remoteTracks.get(source); + if (track) { + const stream = source == CallMediaSource.Mic || source == CallMediaSource.Camera ? call.remoteStream : call.remoteScreenStream; + stream.addTrack(track); + call.remoteTracks.delete(source); + } + } async function getLocalMediaStream(mic, camera, facingMode) { if (!mic && !camera) return new MediaStream(); @@ -1136,7 +1140,7 @@ const processCommand = (function () { if (peerHasOldVersion) { console.log("The peer has an old version.", "Tracks size:", activeCall.remoteStream.getAudioTracks().length, activeCall.remoteStream.getVideoTracks().length); onMediaMuteUnmute("0", false); - if (activeCall.remoteStream.getVideoTracks().length > 0) { + if (activeCall.remoteStream.getVideoTracks().length > 0 || activeCall.remoteTracks.get(CallMediaSource.Camera)) { onMediaMuteUnmute("1", false); } if (activeCall.localMediaSources.camera && !activeCall.peerMediaSources.camera) { diff --git a/packages/simplex-chat-webrtc/package.json b/packages/simplex-chat-webrtc/package.json index f11ea36343..a6598b6ce7 100644 --- a/packages/simplex-chat-webrtc/package.json +++ b/packages/simplex-chat-webrtc/package.json @@ -27,7 +27,7 @@ "author": "SimpleX Chat", "license": "AGPL-3.0-or-later", "devDependencies": { - "@types/lz-string": "^1.3.34", + "@types/lz-string": "1.3.34", "husky": "^7.0.4", "isomorphic-webcrypto": "^2.3.8", "lint-staged": "^12.4.1", @@ -38,6 +38,6 @@ "**/*": "prettier --write --ignore-unknown" }, "dependencies": { - "lz-string": "^1.4.4" + "lz-string": "1.5.0" } } diff --git a/packages/simplex-chat-webrtc/src/call.ts b/packages/simplex-chat-webrtc/src/call.ts index 793bc8e5c2..cf91af07dc 100644 --- a/packages/simplex-chat-webrtc/src/call.ts +++ b/packages/simplex-chat-webrtc/src/call.ts @@ -248,7 +248,10 @@ interface Call { localCamera: VideoCamera localStream: MediaStream localScreenStream: MediaStream + // has no tracks in the beggining, see addRemoteTracksWhenUnmuted remoteStream: MediaStream + remoteTracks: Map + // has no tracks in the beggining too remoteScreenStream: MediaStream peerMediaSources: CallMediaSources aesKey?: string @@ -439,6 +442,7 @@ const processCommand = (function () { localStream, localScreenStream, remoteStream, + remoteTracks: new Map(), remoteScreenStream, peerMediaSources: { mic: false, @@ -794,14 +798,6 @@ const processCommand = (function () { call.worker = new Worker(URL.createObjectURL(new Blob([workerCode], {type: "text/javascript"}))) call.worker.onerror = ({error, filename, lineno, message}: ErrorEvent) => console.log({error, filename, lineno, message}) // call.worker.onmessage = ({data}) => console.log(JSON.stringify({message: data})) - call.worker.onmessage = ({data}) => { - console.log(JSON.stringify({message: data})) - const transceiverMid: string = data.transceiverMid - const mute: boolean = data.mute - if (transceiverMid && mute != undefined) { - onMediaMuteUnmute(transceiverMid, mute) - } - } } } } @@ -927,11 +923,7 @@ const processCommand = (function () { setupMuteUnmuteListener(event.transceiver, track) const mediaSource = mediaSourceFromTransceiverMid(event.transceiver.mid) - if (mediaSource == CallMediaSource.ScreenAudio || mediaSource == CallMediaSource.ScreenVideo) { - call.remoteScreenStream.addTrack(track) - } else { - call.remoteStream.addTrack(track) - } + call.remoteTracks.set(mediaSource, track) console.log(`ontrack success`) } catch (e) { console.log(`ontrack error: ${(e as Error).message}`) @@ -1296,11 +1288,27 @@ const processCommand = (function () { sendMessageToNative({resp: resp}) if (!mute) videos.remoteScreen.play().catch((e) => console.log(e)) } + if (!mute) addRemoteTracksWhenUnmuted(source, activeCall) localOrPeerMediaSourcesChanged(activeCall) // Make sure that remote camera and remote screen video in their places and shown/hidden based on layout type currently in use changeLayout(activeCall.layout) } + /* + When new remote tracks are coming, they don't get added to remote streams. They are stored in a map and once any of them "unmuted", + that track is added to the stream. Such workaround needed because Safari doesn't play one stream + if another one is not playing too, eg. no audio if only audio is playing while video track is present too but muted. + But we have possibility to have only one currently active track, even no active track at all. + */ + function addRemoteTracksWhenUnmuted(source: CallMediaSource, call: Call) { + const track = call.remoteTracks.get(source) + if (track) { + const stream = source == CallMediaSource.Mic || source == CallMediaSource.Camera ? call.remoteStream : call.remoteScreenStream + stream.addTrack(track) + call.remoteTracks.delete(source) + } + } + async function getLocalMediaStream(mic: boolean, camera: boolean, facingMode: VideoCamera): Promise { if (!mic && !camera) return new MediaStream() const constraints = callMediaConstraints(mic, camera, facingMode) @@ -1422,7 +1430,7 @@ const processCommand = (function () { activeCall.remoteStream.getVideoTracks().length ) onMediaMuteUnmute("0", false) - if (activeCall.remoteStream.getVideoTracks().length > 0) { + if (activeCall.remoteStream.getVideoTracks().length > 0 || activeCall.remoteTracks.get(CallMediaSource.Camera)) { onMediaMuteUnmute("1", false) } if (activeCall.localMediaSources.camera && !activeCall.peerMediaSources.camera) {