ui: reconnect call (#7711)

* Update Call.hs * Update Internal.hs * add reconnecting string * Update WebRTC.kt * desktop: show reconnecting state, keep call duration * android keep connection time correct wiith adding reconnecting * Update CallTypes.swift add reconnecting status on IOS * Update WebRTC.swift add reconnecting * add reconnecting and make sure call duration is correct * Update call.ts * core: regenerate call.js from call.ts * Update Call.hs revert call status change minimization * Update Internal.hs rever to minimize changes * Update CallTypes.swift minimize changes * Update WebRTC.kt minimize changes * Update CallView.desktop.kt desktop shows reconnecting without sending to core * Update CallView.android.kt show reconnecting without sending it to the core * Update ActiveCallView.swift show reconnecting to ui without sending to core * Update CallView.desktop.kt add reconnecting sound on reconnect * Update CallView.android.kt add reconnecting sound * Update ActiveCallView.swift play reconnecting sound when reconnecting * Update RecAndPlay.desktop.kt * rebuild * refactor * build * android, desktop: restart ICE to reconnect calls * ios: restart ICE to reconnect calls * webrtc: add call reconnection e2e tests * android, desktop: fix lost call commands --------- Co-authored-by: GuavaGoat <320862384+GuavaGoat@users.noreply.github.com> Co-authored-by: Evgeny Poberezkin <evgeny@poberezkin.com> Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com>
2026-10-22 13:11:55 +00:00 · 2026-10-19 12:17:16 +01:00
1 parent 634fa5ecff
commit 2ea1ec0d50
@@ -52,8 +52,7 @@ import dev.icerock.moko.resources.StringResource
 import dev.icerock.moko.resources.compose.painterResource
 import dev.icerock.moko.resources.compose.stringResource
 import kotlinx.coroutines.*
-import kotlinx.coroutines.flow.distinctUntilChanged
-import kotlinx.coroutines.flow.filterNotNull
+import kotlinx.coroutines.flow.first
 import kotlinx.datetime.Clock
 import kotlinx.serialization.encodeToString
 import java.io.Closeable
@@ -168,13 +167,24 @@ actual fun ActiveCallView() {
            CallSoundsPlayer.stop()
          }
          is WCallResponse.Ice -> withBGApi {
-            chatModel.controller.apiSendCallExtraInfo(callRh, call.contact, r.iceCandidates)
+            chatModel.controller.apiSendCallExtraInfo(callRh, call.contact, r.session, r.iceCandidates)
          }
          is WCallResponse.Connection ->
-            try {
+            // the call is restored and NOT sent to the core
+            // Makes sure the core DOES NOT record the duration wrong if a reconnect happens
+            if (r.state.connectionState == "reconnecting") {
+              if (call.callState != CallState.Reconnecting) {
+                CallSoundsPlayer.startConnectingCallSound(scope)
+              }
+              updateActiveCall(call) { it.copy(callState = CallState.Reconnecting) }
+            } else try {
              val callStatus = json.decodeFromString<WebRTCCallStatus>("\"${r.state.connectionState}\"")
              if (callStatus == WebRTCCallStatus.Connected) {
-                updateActiveCall(call) { it.copy(callState = CallState.Connected, connectedAt = Clock.System.now()) }
+                if (call.callState == CallState.Reconnecting) {
+                  CallSoundsPlayer.stop()
+                }
+                // connectedAt is only set ONCE, this way call duration is NOT reset when the call is restored from reconnecting
+                updateActiveCall(call) { it.copy(callState = CallState.Connected, connectedAt = it.connectedAt ?: Clock.System.now()) }
              }
              withBGApi { chatModel.controller.apiCallStatus(callRh, call.contact, callStatus) }
            } catch (e: Throwable) {
@@ -705,18 +715,17 @@ fun WebRTCView(callCommand: SnapshotStateList<WCallCommand>, onResponse: (WVAPIM
  val wv = webView.value
  if (wv != null) {
    LaunchedEffect(Unit) {
-      snapshotFlow { callCommand.firstOrNull() }
-        .distinctUntilChanged()
-        .filterNotNull()
-        .collect {
-          while (callCommand.isNotEmpty()) {
-            val cmd = callCommand.removeFirstOrNull()
-            Log.d(TAG, "WebRTCView LaunchedEffect executing ${cmd?.javaClass?.simpleName}")
-            if (cmd != null) {
-              processCommand(wv, cmd)
-            }
+      while (isActive) {
+        // waiting for the first command would miss a command equal to the previous one
+        snapshotFlow { callCommand.isNotEmpty() }.first { it }
+        while (callCommand.isNotEmpty()) {
+          val cmd = callCommand.removeFirstOrNull()
+          Log.d(TAG, "WebRTCView LaunchedEffect executing ${cmd?.javaClass?.simpleName}")
+          if (cmd != null) {
+            processCommand(wv, cmd)
          }
        }
+      }
    }
  }
  val assetLoader = WebViewAssetLoader.Builder()
@@ -2185,8 +2185,8 @@ object ChatController {
    return r.result is CR.CmdOk
  }

-  suspend fun apiSendCallExtraInfo(rh: Long?, contact: Contact, rtcIceCandidates: String): Boolean {
-    val extraInfo = WebRTCExtraInfo(rtcIceCandidates)
+  suspend fun apiSendCallExtraInfo(rh: Long?, contact: Contact, rtcSession: String?, rtcIceCandidates: String): Boolean {
+    val extraInfo = WebRTCExtraInfo(rtcSession, rtcIceCandidates)
    val r = sendCmd(rh, CC.ApiSendCallExtraInfo(contact, extraInfo))
    return r.result is CR.CmdOk
  }
@@ -3436,7 +3436,7 @@ object ChatController {
      }
      is CR.CallExtraInfo -> {
        withCall(r, r.contact) { _ ->
-          chatModel.callCommand.add(WCallCommand.Ice(iceCandidates = r.extraInfo.rtcIceCandidates))
+          chatModel.callCommand.add(WCallCommand.Ice(iceCandidates = r.extraInfo.rtcIceCandidates, session = r.extraInfo.rtcSession))
        }
      }
      is CR.CallEnded -> {
@@ -56,6 +56,7 @@ enum class CallState {
  AnswerReceived,
  Negotiated,
  Connected,
+  Reconnecting,
  Ended;

  val text: String get() = when(this) {
@@ -67,6 +68,7 @@ enum class CallState {
    AnswerReceived -> generalGetString(MR.strings.callstate_received_confirmation)
    Negotiated -> generalGetString(MR.strings.callstate_connecting)
    Connected -> generalGetString(MR.strings.callstate_connected)
+    Reconnecting -> generalGetString(MR.strings.callstate_reconnecting)
    Ended -> generalGetString(MR.strings.callstate_ended)
  }
 }
@@ -91,7 +93,7 @@ sealed class WCallCommand {
  @Serializable @SerialName("start") data class Start(val media: CallMediaType, val aesKey: String? = null, val iceServers: List<RTCIceServer>? = null, val relay: Boolean? = null): WCallCommand()
  @Serializable @SerialName("offer") data class Offer(val offer: String, val iceCandidates: String, val media: CallMediaType, val aesKey: String? = null, val iceServers: List<RTCIceServer>? = null, val relay: Boolean? = null): WCallCommand()
  @Serializable @SerialName("answer") data class Answer (val answer: String, val iceCandidates: String): WCallCommand()
-  @Serializable @SerialName("ice") data class Ice(val iceCandidates: String): WCallCommand()
+  @Serializable @SerialName("ice") data class Ice(val iceCandidates: String, val session: String? = null): WCallCommand()
  @Serializable @SerialName("media") data class Media(val source: CallMediaSource, val enable: Boolean): WCallCommand()
  @Serializable @SerialName("camera") data class Camera(val camera: VideoCamera): WCallCommand()
  @Serializable @SerialName("description") data class Description(val state: String, val description: String): WCallCommand()
@@ -104,7 +106,7 @@ sealed class WCallResponse {
  @Serializable @SerialName("capabilities") data class Capabilities(val capabilities: CallCapabilities): WCallResponse()
  @Serializable @SerialName("offer") data class Offer(val offer: String, val iceCandidates: String, val capabilities: CallCapabilities): WCallResponse()
  @Serializable @SerialName("answer") data class Answer(val answer: String, val iceCandidates: String): WCallResponse()
-  @Serializable @SerialName("ice") data class Ice(val iceCandidates: String): WCallResponse()
+  @Serializable @SerialName("ice") data class Ice(val iceCandidates: String, val session: String? = null): WCallResponse()
  @Serializable @SerialName("connection") data class Connection(val state: ConnectionState): WCallResponse()
  @Serializable @SerialName("connected") data class Connected(val connectionInfo: ConnectionInfo): WCallResponse()
  @Serializable @SerialName("peerMedia") data class PeerMedia(val source: CallMediaSource, val enabled: Boolean): WCallResponse()
@@ -116,7 +118,7 @@ sealed class WCallResponse {

@Serializable data class WebRTCCallOffer(val callType: CallType, val rtcSession: WebRTCSession)
@Serializable data class WebRTCSession(val rtcSession: String, val rtcIceCandidates: String)
-@Serializable data class WebRTCExtraInfo(val rtcIceCandidates: String)
+@Serializable data class WebRTCExtraInfo(val rtcSession: String? = null, val rtcIceCandidates: String)
@Serializable data class CallType(val media: CallMediaType, val capabilities: CallCapabilities)
@Serializable data class RcvCallInvitation(
  val remoteHostId: Long?,
@@ -1301,6 +1301,7 @@
  <string name="callstate_received_answer">received answer…</string>
  <string name="callstate_received_confirmation">received confirmation…</string>
  <string name="callstate_connecting">connecting…</string>
+  <string name="callstate_reconnecting">reconnecting…</string>
  <string name="callstate_connected">connected</string>
  <string name="callstate_ended">ended</string>

@@ -54,6 +54,8 @@ let inactiveCallMediaSources = {
 let activeCall;
 let notConnectedCall;
 let answerTimeout = 30000;
+let reconnectTimeout = 30000;
+let iceRestartInterval = 10000;
 var useWorker = false;
 var isDesktop = false;
 var localizedState = "";
@@ -75,7 +77,8 @@ const processCommand = (function () {
        return {
            peerConnectionConfig: {
                iceServers: iceServers !== null && iceServers !== void 0 ? iceServers : defaultIceServers,
-                iceCandidatePoolSize: 10,
+                // Chrome gathers no candidates on ICE restart when candidates pool is used
+                iceCandidatePoolSize: 0,
                encodedInsertableStreams,
                iceTransportPolicy: relay ? "relay" : "all",
                // needed for Android WebView >= 69 && <= 72 where default was "plan-b" which is incompatible with transceivers
@@ -202,10 +205,14 @@ const processCommand = (function () {
            layout: (_c = notConnectedCall === null || notConnectedCall === void 0 ? void 0 : notConnectedCall.layout) !== null && _c !== void 0 ? _c : LayoutType.Default,
            cameraTrackWasSetBefore: localStream.getVideoTracks().length > 0,
            peerHasOldVersion: false,
+            receiveRestartSession,
        };
        localOrPeerMediaSourcesChanged(call);
        await setupMediaStreams(call);
        let connectionTimeout = setTimeout(connectionHandler, answerTimeout);
+        let reconnectingTimeout;
+        let iceRestartTimeout;
+        let wasConnected = false;
        if (pc.connectionState) {
            pc.addEventListener("connectionstatechange", connectionStateChange);
        }
@@ -216,30 +223,49 @@ const processCommand = (function () {
        async function connectionStateChange() {
            // "failed" means the second party did not answer in time (15 sec timeout in Chrome WebView)
            // See https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/p2p/base/p2p_constants.cc;l=70)
-            if (pc.connectionState !== "failed")
+            if (pc.connectionState !== "failed" || wasConnected)
                connectionHandler();
        }
-        async function connectionHandler() {
+        async function connectionHandler(reconnectTimedOut = false) {
            var _a;
+            let state = (_a = pc.connectionState) !== null && _a !== void 0 ? _a : (pc.iceConnectionState != "completed" && pc.iceConnectionState != "checking"
+                ? pc.iceConnectionState
+                : pc.iceConnectionState == "completed"
+                    ? "connected"
+                    : "connecting"); /* webView 69-70 doesn't have connectionState yet */
+            if (wasConnected) {
+                if (reconnectTimedOut) {
+                    reconnectingTimeout = undefined;
+                    if (activeCall !== call || state == "connected")
+                        return;
+                    state = "disconnected";
+                }
+                else if (state == "disconnected" || state == "failed") {
+                    if (reconnectingTimeout !== undefined)
+                        return;
+                    reconnectingTimeout = setTimeout(() => connectionHandler(true), reconnectTimeout);
+                    if (isIceRestarter())
+                        restartIce();
+                    state = "reconnecting";
+                }
+                else if (state == "connecting" || state == "new") {
+                    return;
+                }
+            }
            sendMessageToNative({
                resp: {
                    type: "connection",
                    state: {
-                        connectionState: (_a = pc.connectionState) !== null && _a !== void 0 ? _a : (pc.iceConnectionState != "completed" && pc.iceConnectionState != "checking"
-                            ? pc.iceConnectionState
-                            : pc.iceConnectionState == "completed"
-                                ? "connected"
-                                : "connecting") /* webView 69-70 doesn't have connectionState yet */,
+                        connectionState: state,
                        iceConnectionState: pc.iceConnectionState,
                        iceGatheringState: pc.iceGatheringState,
                        signalingState: pc.signalingState,
                    },
                },
            });
-            if (pc.connectionState == "disconnected" ||
-                pc.connectionState == "failed" ||
-                (!pc.connectionState && (pc.iceConnectionState == "disconnected" || pc.iceConnectionState == "failed"))) {
+            if (state == "disconnected" || state == "failed") {
                clearConnectionTimeout();
+                clearReconnecting();
                if (pc.connectionState) {
                    pc.removeEventListener("connectionstatechange", connectionStateChange);
                }
@@ -251,8 +277,10 @@ const processCommand = (function () {
                }
                endCall();
            }
-            else if (pc.connectionState == "connected" || (!pc.connectionState && pc.iceConnectionState == "connected")) {
+            else if (state == "connected") {
                clearConnectionTimeout();
+                clearReconnecting();
+                wasConnected = true;
                const stats = (await pc.getStats());
                for (const stat of stats.values()) {
                    const { type, state } = stat;
@@ -278,6 +306,53 @@ const processCommand = (function () {
                connectionTimeout = undefined;
            }
        }
+        function clearReconnecting() {
+            clearTimeout(reconnectingTimeout);
+            reconnectingTimeout = undefined;
+            clearTimeout(iceRestartTimeout);
+            iceRestartTimeout = undefined;
+        }
+        // Only the party that sent the initial offer restarts ICE, so that restart offers never collide
+        function isIceRestarter() {
+            var _a;
+            return ((_a = pc.localDescription) === null || _a === void 0 ? void 0 : _a.type) == "offer";
+        }
+        async function restartIce() {
+            if (activeCall !== call || reconnectingTimeout === undefined)
+                return;
+            iceRestartTimeout = setTimeout(restartIce, iceRestartInterval);
+            // the previous restart offer is not answered yet
+            if (pc.signalingState != "stable")
+                return;
+            try {
+                const iceCandidates = getIceCandidates(pc, config);
+                const offer = await pc.createOffer({ iceRestart: true });
+                await pc.setLocalDescription(offer);
+                sendMessageToNative({ resp: { type: "ice", session: serialize(offer), iceCandidates: await iceCandidates } });
+            }
+            catch (e) {
+                console.log("ICE restart error", e);
+            }
+        }
+        async function receiveRestartSession(session, iceCandidates) {
+            const desc = parse(session);
+            const remoteDesc = new RTCSessionDescription(!webView69Or70() ? desc : adaptSdpToOldWebView(desc));
+            if (desc.type == "offer" && !isIceRestarter()) {
+                await pc.setRemoteDescription(remoteDesc);
+                const localIceCandidates = getIceCandidates(pc, config);
+                const answer = await pc.createAnswer();
+                await pc.setLocalDescription(answer);
+                addIceCandidates(pc, iceCandidates);
+                sendMessageToNative({ resp: { type: "ice", session: serialize(answer), iceCandidates: await localIceCandidates } });
+            }
+            else if (desc.type == "answer" && pc.signalingState == "have-local-offer") {
+                await pc.setRemoteDescription(remoteDesc);
+                addIceCandidates(pc, iceCandidates);
+            }
+            else {
+                throw new Error(`unexpected ICE restart ${desc.type} in ${pc.signalingState} state`);
+            }
+        }
    }
    function serialize(x) {
        return LZString.compressToBase64(JSON.stringify(x));
@@ -296,6 +371,8 @@ const processCommand = (function () {
                    console.log("starting outgoing call - capabilities");
                    if (activeCall)
                        endCall();
+                    // Stop a preview stream from an earlier pre-connect outgoing call being replaced (activeCall may be null here)
+                    stopNotConnectedCall();
                    let localStream = null;
                    try {
                        localStream = await getLocalMediaStream(true, command.media == CallMediaType.Video && (await browserHasCamera()), VideoCamera.User);
@@ -332,7 +409,8 @@ const processCommand = (function () {
                    if (activeCall)
                        endCall();
                    // It can be already defined on Android when switching calls (if the previous call was outgoing)
-                    notConnectedCall = undefined;
+                    // Stop its preview tracks before clearing, otherwise camera/mic stay live
+                    stopNotConnectedCall();
                    inactiveCallMediaSources.mic = true;
                    inactiveCallMediaSources.camera = command.media == CallMediaType.Video;
                    inactiveCallMediaSourcesChanged(inactiveCallMediaSources);
@@ -436,7 +514,11 @@ const processCommand = (function () {
                    break;
                case "ice":
                    const remoteIceCandidates = parse(command.iceCandidates);
-                    if (pc) {
+                    if (activeCall && command.session) {
+                        await activeCall.receiveRestartSession(command.session, remoteIceCandidates);
+                        resp = { type: "ok" };
+                    }
+                    else if (pc) {
                        addIceCandidates(pc, remoteIceCandidates);
                        resp = { type: "ok" };
                    }
@@ -1131,6 +1213,13 @@ const processCommand = (function () {
            activeCall.localScreenStream.getTracks().forEach((track) => track.stop());
        }
    }
+    // Call on any path that abandons notConnectedCall, otherwise its preview camera/mic tracks stay live.
+    function stopNotConnectedCall() {
+        if (notConnectedCall) {
+            notConnectedCall.localStream.getTracks().forEach((track) => track.stop());
+            notConnectedCall = undefined;
+        }
+    }
    function resetVideoElements() {
        const videos = getVideoElements();
        if (!videos)
@@ -2,7 +2,7 @@
 // Override defaults to enable worker on Chrome and Safari
 useWorker = typeof window.Worker !== "undefined";
 isDesktop = true;
-// Create WebSocket connection.
+// Create WebSocket connection. location.search carries the per-call ?token=... capability required by the server.
 const socket = new WebSocket(`ws://${location.host}${location.search}`);
 socket.addEventListener("open", (_event) => {
    console.log("Opened socket");
@@ -192,4 +192,4 @@ function updateCallInfoView(state, description) {
    document.getElementById("state").innerText = state;
    document.getElementById("description").innerText = description;
 }
-//# sourceMappingURL=ui.js.map
+//# sourceMappingURL=ui.js.map
@@ -313,7 +313,9 @@ actual object CallSoundsPlayer: CallSoundsPlayerInterface {
  override fun startConnectingCallSound(scope: CoroutineScope) {
    // Taken from https://github.com/TelegramOrg/Telegram-Android
    // https://github.com/TelegramOrg/Telegram-Android/blob/master/LICENSE
-    start("/media/connecting_call.mp3", 3000, scope)
+    // the delay is the length of the track exactly, so it repeats with no delay, which already happens on ios and android
+    // Android (isLooping) and iOS (which sleeps for the track duration)
+    start("/media/connecting_call.mp3", 3536, scope)
  }

  override fun startInCallSound(scope: CoroutineScope) {
@@ -8,6 +8,7 @@ import chat.simplex.common.platform.*
 import chat.simplex.common.views.helpers.*
 import chat.simplex.res.*
 import kotlinx.coroutines.delay
+import kotlinx.coroutines.isActive
 import kotlinx.coroutines.flow.*
 import kotlinx.datetime.Clock
 import kotlinx.serialization.encodeToString
@@ -55,13 +56,24 @@ actual fun ActiveCallView() {
          CallSoundsPlayer.stop()
        }
        is WCallResponse.Ice -> withBGApi {
-          chatModel.controller.apiSendCallExtraInfo(callRh, call.contact, r.iceCandidates)
+          chatModel.controller.apiSendCallExtraInfo(callRh, call.contact, r.session, r.iceCandidates)
        }
        is WCallResponse.Connection ->
-          try {
+          // the call is restored and NOT sent to the core
+          // Makes sure the core DOES NOT record the duration wrong if a reconnect happens
+          if (r.state.connectionState == "reconnecting") {
+            if (call.callState != CallState.Reconnecting) {
+              CallSoundsPlayer.startConnectingCallSound(scope)
+            }
+            chatModel.activeCall.value = call.copy(callState = CallState.Reconnecting)
+          } else try {
            val callStatus = json.decodeFromString<WebRTCCallStatus>("\"${r.state.connectionState}\"")
            if (callStatus == WebRTCCallStatus.Connected) {
-              chatModel.activeCall.value = call.copy(callState = CallState.Connected, connectedAt = Clock.System.now())
+              if (call.callState == CallState.Reconnecting) {
+                CallSoundsPlayer.stop()
+              }
+              // connectedAt is only set ONCE, this way call duration is NOT reset when the call is restored from reconnecting
+              chatModel.activeCall.value = call.copy(callState = CallState.Connected, connectedAt = call.connectedAt ?: Clock.System.now())
            }
            withBGApi { chatModel.controller.apiCallStatus(callRh, call.contact, callStatus) }
          } catch (e: Throwable) {
@@ -194,21 +206,20 @@ fun WebRTCController(callCommand: SnapshotStateList<WCallCommand>, onResponse: (
    }
  }
  LaunchedEffect(Unit) {
-    snapshotFlow { callCommand.firstOrNull() }
-      .distinctUntilChanged()
-      .filterNotNull()
-      .collect {
-        while (connections.isEmpty()) {
-          delay(100)
-        }
-        while (callCommand.isNotEmpty()) {
-          val cmd = callCommand.removeFirstOrNull()
-          Log.d(TAG, "WebRTCController LaunchedEffect executing ${cmd?.javaClass?.simpleName}")
-          if (cmd != null) {
-            processCommand(cmd)
-          }
+    while (isActive) {
+      // waiting for the first command would miss a command equal to the previous one
+      snapshotFlow { callCommand.isNotEmpty() }.first { it }
+      while (connections.isEmpty()) {
+        delay(100)
+      }
+      while (callCommand.isNotEmpty()) {
+        val cmd = callCommand.removeFirstOrNull()
+        Log.d(TAG, "WebRTCController LaunchedEffect executing ${cmd?.javaClass?.simpleName}")
+        if (cmd != null) {
+          processCommand(cmd)
        }
      }
+    }
  }
 }