From 5f67c450b10352cbbd8589089105fb4ab638386d Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Wed, 18 May 2022 17:20:43 +0100 Subject: [PATCH] mobile: webrtc calls fix encryption status, translate alerts, report connection stats on connection (#664) * mobile: webrtc calls fix encryption status, translate alerts, report connection stats on connection * refactor, remove logger, make property into getter --- apps/android/app/src/main/assets/www/call.js | 55 ++++++++++---- .../java/chat/simplex/app/model/SimpleXAPI.kt | 20 +++-- .../chat/simplex/app/views/call/CallView.kt | 13 ++-- .../chat/simplex/app/views/call/WebRTC.kt | 41 ++++++++-- .../chat/simplex/app/views/chat/ChatView.kt | 2 +- .../app/src/main/res/values-ru/strings.xml | 12 +++ .../app/src/main/res/values/strings.xml | 12 +++ apps/ios/Shared/Model/Shared/CallTypes.swift | 18 +++++ .../Shared/Model/Shared/Notifications.swift | 8 +- .../Shared/Views/Call/ActiveCallView.swift | 2 + apps/ios/Shared/Views/Call/WebRTC.swift | 76 +++++++++++++++---- apps/ios/Shared/Views/Call/WebRTCView.swift | 6 +- .../Shared/Views/ChatList/ChatListView.swift | 11 ++- packages/simplex-chat-webrtc/src/call.ts | 73 ++++++++++++++---- 14 files changed, 274 insertions(+), 75 deletions(-) diff --git a/apps/android/app/src/main/assets/www/call.js b/apps/android/app/src/main/assets/www/call.js index 722bdccae9..5a4ae8f983 100644 --- a/apps/android/app/src/main/assets/www/call.js +++ b/apps/android/app/src/main/assets/www/call.js @@ -18,16 +18,17 @@ var TransformOperation; })(TransformOperation || (TransformOperation = {})); const processCommand = (function () { let activeCall; - function defaultCallConfig(encodedInsertableStreams) { + const defaultIceServers = [ + { urls: ["stun:stun.simplex.chat:5349"] }, + { urls: ["turn:turn.simplex.chat:5349"], username: "private", credential: "yleob6AVkiNI87hpR94Z" }, + ]; + function getCallConfig(encodedInsertableStreams, iceServers, relay) { return { peerConnectionConfig: { - iceServers: [ - { urls: "stun:stun.simplex.chat:5349" }, - { urls: "turn:turn.simplex.chat:5349", username: "private", credential: "yleob6AVkiNI87hpR94Z" }, - ], + iceServers: iceServers !== null && iceServers !== void 0 ? iceServers : defaultIceServers, iceCandidatePoolSize: 10, encodedInsertableStreams, - // iceTransportPolicy: "relay", + iceTransportPolicy: relay ? "relay" : "all", }, iceCandidates: { delay: 2000, @@ -91,7 +92,7 @@ const processCommand = (function () { } }); return { connection: conn, iceCandidates, localMedia: mediaType, localStream }; - function connectionStateChange() { + async function connectionStateChange() { sendMessageToNative({ resp: { type: "connection", @@ -105,10 +106,29 @@ const processCommand = (function () { }); if (conn.connectionState == "disconnected" || conn.connectionState == "failed") { conn.removeEventListener("connectionstatechange", connectionStateChange); - sendMessageToNative({ resp: { type: "ended" } }); conn.close(); activeCall = undefined; resetVideoElements(); + setTimeout(() => sendMessageToNative({ resp: { type: "ended" } }), 0); + } + else if (conn.connectionState == "connected") { + const stats = (await conn.getStats()); + for (const stat of stats.values()) { + const { type, state } = stat; + if (type === "candidate-pair" && state === "succeeded") { + const iceCandidatePair = stat; + const resp = { + type: "connected", + connectionInfo: { + iceCandidatePair, + localCandidate: stats.get(iceCandidatePair.localCandidateId), + remoteCandidate: stats.get(iceCandidatePair.remoteCandidateId), + }, + }; + setTimeout(() => sendMessageToNative({ resp }), 0); + break; + } + } } } } @@ -131,13 +151,14 @@ const processCommand = (function () { case "start": console.log("starting call"); if (activeCall) { + // TODO cancel current call resp = { type: "error", message: "start: call already started" }; } else { - const { media, useWorker } = command; + const { media, useWorker, iceServers, relay } = command; const encryption = supportsInsertableStreams(useWorker); const aesKey = encryption ? command.aesKey : undefined; - activeCall = await initializeCall(defaultCallConfig(encryption && !!aesKey), media, aesKey, useWorker); + activeCall = await initializeCall(getCallConfig(encryption && !!aesKey, iceServers, relay), media, aesKey, useWorker); const pc = activeCall.connection; const offer = await pc.createOffer(); await pc.setLocalDescription(offer); @@ -146,8 +167,12 @@ const processCommand = (function () { // type: "offer", // offer: serialize(offer), // iceCandidates: await activeCall.iceCandidates, + // capabilities: {encryption}, // media, + // iceServers, + // relay, // aesKey, + // useWorker, // } resp = { type: "offer", @@ -167,8 +192,8 @@ const processCommand = (function () { else { const offer = parse(command.offer); const remoteIceCandidates = parse(command.iceCandidates); - const { media, aesKey, useWorker } = command; - activeCall = await initializeCall(defaultCallConfig(!!aesKey), media, aesKey, useWorker); + const { media, aesKey, useWorker, iceServers, relay } = command; + activeCall = await initializeCall(getCallConfig(!!aesKey, iceServers, relay), media, aesKey, useWorker); const pc = activeCall.connection; await pc.setRemoteDescription(new RTCSessionDescription(offer)); const answer = await pc.createAnswer(); @@ -279,7 +304,11 @@ const processCommand = (function () { console.log("set up decryption for receiving"); setupPeerTransform(TransformOperation.Decrypt, event.receiver, worker, aesKey, key); } - remoteStream.addTrack(event.track); + for (const stream of event.streams) { + for (const track of stream.getTracks()) { + remoteStream.addTrack(track); + } + } }; // We assume VP8 encoding in the decode/encode stages to get the initial // bytes to pass as plaintext so we enforce that here. diff --git a/apps/android/app/src/main/java/chat/simplex/app/model/SimpleXAPI.kt b/apps/android/app/src/main/java/chat/simplex/app/model/SimpleXAPI.kt index 036f6f5e1a..12719b7e58 100644 --- a/apps/android/app/src/main/java/chat/simplex/app/model/SimpleXAPI.kt +++ b/apps/android/app/src/main/java/chat/simplex/app/model/SimpleXAPI.kt @@ -10,6 +10,7 @@ import android.os.Build import android.os.PowerManager import android.provider.Settings import android.util.Log +import androidx.annotation.StringRes import androidx.compose.foundation.layout.* import androidx.compose.material.* import androidx.compose.material.icons.Icons @@ -505,26 +506,23 @@ open class ChatController(private val ctrl: ChatCtrl, private val ntfManager: Nt chatModel.activeCallInvitation.value = ContactRef(r.contact.apiId, r.contact.localDisplayName) } ntfManager.notifyCallInvitation(r.contact, invitation) - val encryptionText = if (r.sharedKey == null) "without e2e encryption" else "with e2e encryption" AlertManager.shared.showAlertDialog( - title = "Incoming call", - text = "${r.contact.displayName} wants to start ${r.callType.media} call with you (${encryptionText})", - confirmText = "Answer", // generalGetString(R.string.answer), + title = invitation.callTitle, + text = String.format(generalGetString(R.string.contact_wants_to_connect_via_call), r.contact.displayName) + " " + invitation.callTypeText + ".\n" + generalGetString(R.string.if_you_accept_this_call_your_ip_address_visible), + confirmText = generalGetString(R.string.answer), onConfirm = { - Log.e(TAG, "showAlertDialog onConfirm ${chatModel.activeCallInvitation.value}") if (chatModel.activeCallInvitation.value == null) { AlertManager.shared.hideAlert() - AlertManager.shared.showAlertMsg("Call already ended!") + AlertManager.shared.showAlertMsg(generalGetString(R.string.call_already_ended)) } else { - Log.e(TAG, "showAlertDialog onConfirm has activeCallInvitation ${chatModel.activeCallInvitation.value}") chatModel.activeCallInvitation.value = null chatModel.activeCall.value = Call( contact = r.contact, callState = CallState.InvitationReceived, - localMedia = invitation.peerMedia + localMedia = invitation.peerMedia, + sharedKey = invitation.sharedKey ) - chatModel.callCommand.value = WCallCommand.Start(invitation.peerMedia, invitation.sharedKey) - Log.e(TAG, "showAlertDialog onConfirm ${chatModel.callCommand.value}") + chatModel.callCommand.value = WCallCommand.Start(media = invitation.peerMedia, aesKey = invitation.sharedKey) chatModel.showCallView.value = true } }, @@ -554,7 +552,7 @@ open class ChatController(private val ctrl: ChatCtrl, private val ntfManager: Nt } is CR.CallEnded -> { withCall(r, r.contact) { call -> - chatModel.callCommand.value = WCallCommand.End() + chatModel.callCommand.value = WCallCommand.End chatModel.activeCall.value = null chatModel.activeCallInvitation.value = null chatModel.callCommand.value = null diff --git a/apps/android/app/src/main/java/chat/simplex/app/views/call/CallView.kt b/apps/android/app/src/main/java/chat/simplex/app/views/call/CallView.kt index b6a793bca7..366e9b6c07 100644 --- a/apps/android/app/src/main/java/chat/simplex/app/views/call/CallView.kt +++ b/apps/android/app/src/main/java/chat/simplex/app/views/call/CallView.kt @@ -35,7 +35,7 @@ import kotlinx.serialization.encodeToString @Composable fun ActiveCallView(chatModel: ChatModel) { val endCall = { - Log.e(TAG, "ActiveCallView: endCall") + Log.d(TAG, "ActiveCallView: endCall") chatModel.activeCall.value = null chatModel.activeCallInvitation.value = null chatModel.callCommand.value = null @@ -44,10 +44,10 @@ fun ActiveCallView(chatModel: ChatModel) { BackHandler(onBack = endCall) Box(Modifier.fillMaxSize()) { WebRTCView(chatModel.callCommand) { apiMsg -> - Log.e(TAG, "received from WebRTCView: $apiMsg") + Log.d(TAG, "received from WebRTCView: $apiMsg") val call = chatModel.activeCall.value if (call != null) { - Log.e(TAG, "has active call $call") + Log.d(TAG, "has active call $call") when (val r = apiMsg.resp) { is WCallResponse.Capabilities -> withApi { val callType = CallType(call.localMedia, r.capabilities) @@ -75,6 +75,9 @@ fun ActiveCallView(chatModel: ChatModel) { } catch (e: Error) { Log.d(TAG,"call status ${r.state.connectionState} not used") } + is WCallResponse.Connected -> { + chatModel.activeCall.value = call.copy(callState = CallState.Connected, connectionInfo = r.connectionInfo) + } is WCallResponse.Ended -> endCall() is WCallResponse.Ok -> when (val cmd = apiMsg.command) { is WCallCommand.Media -> { @@ -178,7 +181,7 @@ fun WebRTCView(callCommand: MutableState, onResponse: (WVAPIMessa lifecycleOwner.lifecycle.addObserver(observer) onDispose { val wv = webView.value - if (wv != null) processCommand(wv, WCallCommand.End()) + if (wv != null) processCommand(wv, WCallCommand.End) lifecycleOwner.lifecycle.removeObserver(observer) } } @@ -228,7 +231,7 @@ fun WebRTCView(callCommand: MutableState, onResponse: (WVAPIMessa } } ) { wv -> - Log.e(TAG, "WebRTCView: webview ready") + Log.d(TAG, "WebRTCView: webview ready") // for debugging // wv.evaluateJavascript("sendMessageToNative = ({resp}) => WebRTCInterface.postMessage(JSON.stringify({command: resp}))", null) withApi { diff --git a/apps/android/app/src/main/java/chat/simplex/app/views/call/WebRTC.kt b/apps/android/app/src/main/java/chat/simplex/app/views/call/WebRTC.kt index 019e1d8e2b..aae260ba8b 100644 --- a/apps/android/app/src/main/java/chat/simplex/app/views/call/WebRTC.kt +++ b/apps/android/app/src/main/java/chat/simplex/app/views/call/WebRTC.kt @@ -14,7 +14,8 @@ data class Call( val peerMedia: CallMediaType? = null, val sharedKey: String? = null, val audioEnabled: Boolean = true, - val videoEnabled: Boolean = localMedia == CallMediaType.Video + val videoEnabled: Boolean = localMedia == CallMediaType.Video, + val connectionInfo: ConnectionInfo? = null ) { val encrypted: Boolean get() = (localCapabilities?.encryption ?: false) && sharedKey != null } @@ -44,13 +45,13 @@ enum class CallState { @Serializable sealed class WCallCommand { - @Serializable @SerialName("capabilities") class Capabilities(): WCallCommand() - @Serializable @SerialName("start") class Start(val media: CallMediaType, val aesKey: String? = null): WCallCommand() - @Serializable @SerialName("offer") class Offer(val offer: String, val iceCandidates: String, val media: CallMediaType, val aesKey: String? = null): WCallCommand() + @Serializable @SerialName("capabilities") object Capabilities: WCallCommand() + @Serializable @SerialName("start") class Start(val media: CallMediaType, val aesKey: String? = null, val iceServers: List? = null, val relay: Boolean? = null): WCallCommand() + @Serializable @SerialName("offer") class Offer(val offer: String, val iceCandidates: String, val media: CallMediaType, val aesKey: String? = null, val iceServers: List? = null, val relay: Boolean? = null): WCallCommand() @Serializable @SerialName("answer") class Answer (val answer: String, val iceCandidates: String): WCallCommand() @Serializable @SerialName("ice") class Ice(val iceCandidates: String): WCallCommand() @Serializable @SerialName("media") class Media(val media: CallMediaType, val enable: Boolean): WCallCommand() - @Serializable @SerialName("end") class End(): WCallCommand() + @Serializable @SerialName("end") object End: WCallCommand() } @Serializable @@ -60,8 +61,9 @@ sealed class WCallResponse { @Serializable @SerialName("answer") class Answer(val answer: String, val iceCandidates: String): WCallResponse() @Serializable @SerialName("ice") class Ice(val iceCandidates: String): WCallResponse() @Serializable @SerialName("connection") class Connection(val state: ConnectionState): WCallResponse() - @Serializable @SerialName("ended") class Ended(): WCallResponse() - @Serializable @SerialName("ok") class Ok(): WCallResponse() + @Serializable @SerialName("connected") class Connected(val connectionInfo: ConnectionInfo): WCallResponse() + @Serializable @SerialName("ended") object Ended: WCallResponse() + @Serializable @SerialName("ok") object Ok: WCallResponse() @Serializable @SerialName("error") class Error(val message: String): WCallResponse() } @@ -69,8 +71,31 @@ sealed class WCallResponse { @Serializable class WebRTCSession(val rtcSession: String, val rtcIceCandidates: String) @Serializable class WebRTCExtraInfo(val rtcIceCandidates: String) @Serializable class CallType(val media: CallMediaType, val capabilities: CallCapabilities) -@Serializable class CallInvitation(val peerMedia: CallMediaType, val sharedKey: String?) +@Serializable class CallInvitation(val peerMedia: CallMediaType, val sharedKey: String?) { + val callTypeText: String get() = generalGetString(when(peerMedia) { + CallMediaType.Video -> if (sharedKey == null) R.string.video_call_no_encryption else R.string.encrypted_video_call + CallMediaType.Audio -> if (sharedKey == null) R.string.audio_call_no_encryption else R.string.encrypted_audio_call + }) + val callTitle: String get() = generalGetString(when(peerMedia) { + CallMediaType.Video -> R.string.incoming_video_call + CallMediaType.Audio -> R.string.incoming_audio_call + }) +} @Serializable class CallCapabilities(val encryption: Boolean) +@Serializable class ConnectionInfo(val localCandidate: RTCIceCandidate?, val remoteCandidate: RTCIceCandidate) +// https://developer.mozilla.org/en-US/docs/Web/API/RTCIceCandidate +@Serializable class RTCIceCandidate(val candidateType: RTCIceCandidateType?) +// https://developer.mozilla.org/en-US/docs/Web/API/RTCIceServer +@Serializable class RTCIceServer(val urls: List, val username: String? = null, val credential: String? = null) + +// https://developer.mozilla.org/en-US/docs/Web/API/RTCIceCandidate/type +@Serializable +enum class RTCIceCandidateType { + @SerialName("host") Host, + @SerialName("srflx") ServerReflexive, + @SerialName("prflx") PeerReflexive, + @SerialName("relay") Relay +} @Serializable enum class WebRTCCallStatus { diff --git a/apps/android/app/src/main/java/chat/simplex/app/views/chat/ChatView.kt b/apps/android/app/src/main/java/chat/simplex/app/views/chat/ChatView.kt index cc98299e02..8a2eecb865 100644 --- a/apps/android/app/src/main/java/chat/simplex/app/views/chat/ChatView.kt +++ b/apps/android/app/src/main/java/chat/simplex/app/views/chat/ChatView.kt @@ -124,7 +124,7 @@ fun ChatView(chatModel: ChatModel) { if (cInfo is ChatInfo.Direct) { chatModel.activeCall.value = Call(contact = cInfo.contact, callState = CallState.WaitCapabilities, localMedia = media) chatModel.showCallView.value = true - chatModel.callCommand.value = WCallCommand.Capabilities() + chatModel.callCommand.value = WCallCommand.Capabilities } } ) diff --git a/apps/android/app/src/main/res/values-ru/strings.xml b/apps/android/app/src/main/res/values-ru/strings.xml index 6116d4f8dd..e6d895c804 100644 --- a/apps/android/app/src/main/res/values-ru/strings.xml +++ b/apps/android/app/src/main/res/values-ru/strings.xml @@ -337,4 +337,16 @@ или Соединиться с разработчиками Чтобы задать вопросы и получать уведомления о SimpleX Chat. + + + Incoming video call + Incoming audio call + %1$s wants to connect with you via + video call (not e2e encrypted) + e2e encrypted video call + audio call (not e2e encrypted) + e2e encrypted audio call + If you accept this call, your IP address might be visible to your contact, unless you connect via relay. + Answer + Call already ended! diff --git a/apps/android/app/src/main/res/values/strings.xml b/apps/android/app/src/main/res/values/strings.xml index 119fe2247d..ea855d5641 100644 --- a/apps/android/app/src/main/res/values/strings.xml +++ b/apps/android/app/src/main/res/values/strings.xml @@ -338,4 +338,16 @@ or Connect with the developers To ask any questions and to receive SimpleX Chat updates. + + + Incoming video call + Incoming audio call + %1$s wants to connect with you via + video call (not e2e encrypted) + e2e encrypted video call + audio call (not e2e encrypted) + e2e encrypted audio call + If you accept this call, your IP address might be visible to your contact, unless you connect via relay. + Answer + Call already ended! diff --git a/apps/ios/Shared/Model/Shared/CallTypes.swift b/apps/ios/Shared/Model/Shared/CallTypes.swift index ad59e1cc36..6dc1793c7f 100644 --- a/apps/ios/Shared/Model/Shared/CallTypes.swift +++ b/apps/ios/Shared/Model/Shared/CallTypes.swift @@ -7,6 +7,7 @@ // import Foundation +import SwiftUI struct WebRTCCallOffer: Encodable { var callType: CallType @@ -25,6 +26,23 @@ struct WebRTCExtraInfo: Codable { struct CallInvitation { var peerMedia: CallMediaType var sharedKey: String? + var callTypeText: LocalizedStringKey { + get { + switch peerMedia { + case .video: return sharedKey == nil ? "video call (not e2e encrypted)." : "**e2e encrypted** video call." + case .audio: return sharedKey == nil ? "audio call (not e2e encrypted)." : "**e2e encrypted** audio call." + } + } + } + var callTitle: LocalizedStringKey { + get { + switch peerMedia { + case .video: return "Incoming video call" + case .audio: return "Incoming audio call" + } + } + } + var encryptionText: LocalizedStringKey { get { sharedKey == nil ? "no e2e encryption" : "with e2e encryption" } } } struct CallType: Codable { diff --git a/apps/ios/Shared/Model/Shared/Notifications.swift b/apps/ios/Shared/Model/Shared/Notifications.swift index 725dbb8553..7255d8ad60 100644 --- a/apps/ios/Shared/Model/Shared/Notifications.swift +++ b/apps/ios/Shared/Model/Shared/Notifications.swift @@ -8,6 +8,7 @@ import Foundation import UserNotifications +import SwiftUI let ntfCategoryContactRequest = "NTF_CAT_CONTACT_REQUEST" let ntfCategoryContactConnected = "NTF_CAT_CONTACT_CONNECTED" @@ -50,10 +51,13 @@ func createMessageReceivedNtf(_ cInfo: ChatInfo, _ cItem: ChatItem) -> UNMutable } func createCallInvitationNtf(_ contact: Contact, _ invitation: CallInvitation) -> UNMutableNotificationContent { - createNotification( + let text = invitation.peerMedia == .video + ? NSLocalizedString("Incoming video call", comment: "notification") + : NSLocalizedString("Incoming audio call", comment: "notification") + return createNotification( categoryIdentifier: ntfCategoryCallInvitation, title: "\(contact.chatViewName):", - body: String.localizedStringWithFormat(NSLocalizedString("Incoming %@ call", comment: "notification body"), invitation.peerMedia.rawValue), + body: text, targetContentIdentifier: nil, userInfo: ["chatId": contact.id] ) diff --git a/apps/ios/Shared/Views/Call/ActiveCallView.swift b/apps/ios/Shared/Views/Call/ActiveCallView.swift index 33628e685f..1c15a98151 100644 --- a/apps/ios/Shared/Views/Call/ActiveCallView.swift +++ b/apps/ios/Shared/Views/Call/ActiveCallView.swift @@ -64,6 +64,8 @@ struct ActiveCallView: View { m.activeCall = call.copy(callState: .connected) } try await apiCallStatus(call.contact, state.connectionState) + case let .connected(connectionInfo): + m.activeCall = call.copy(callState: .connected, connectionInfo: connectionInfo) case .ended: m.activeCall = nil m.activeCallInvitation = nil diff --git a/apps/ios/Shared/Views/Call/WebRTC.swift b/apps/ios/Shared/Views/Call/WebRTC.swift index 9b53bd1e35..bc7db73ca9 100644 --- a/apps/ios/Shared/Views/Call/WebRTC.swift +++ b/apps/ios/Shared/Views/Call/WebRTC.swift @@ -22,6 +22,7 @@ class Call: Equatable { var sharedKey: String? var audioEnabled: Bool var videoEnabled: Bool + var connectionInfo: ConnectionInfo? init( contact: Contact, @@ -31,7 +32,8 @@ class Call: Equatable { peerMedia: CallMediaType? = nil, sharedKey: String? = nil, audioEnabled: Bool? = nil, - videoEnabled: Bool? = nil + videoEnabled: Bool? = nil, + connectionInfo: ConnectionInfo? = nil ) { self.contact = contact self.callState = callState @@ -41,6 +43,7 @@ class Call: Equatable { self.sharedKey = sharedKey self.audioEnabled = audioEnabled ?? true self.videoEnabled = videoEnabled ?? (localMedia == .video) + self.connectionInfo = connectionInfo } func copy( @@ -51,7 +54,8 @@ class Call: Equatable { peerMedia: CallMediaType? = nil, sharedKey: String? = nil, audioEnabled: Bool? = nil, - videoEnabled: Bool? = nil + videoEnabled: Bool? = nil, + connectionInfo: ConnectionInfo? = nil ) -> Call { Call ( contact: contact ?? self.contact, @@ -61,13 +65,12 @@ class Call: Equatable { peerMedia: peerMedia ?? self.peerMedia, sharedKey: sharedKey ?? self.sharedKey, audioEnabled: audioEnabled ?? self.audioEnabled, - videoEnabled: videoEnabled ?? self.videoEnabled + videoEnabled: videoEnabled ?? self.videoEnabled, + connectionInfo: connectionInfo ?? self.connectionInfo ) } - var encrypted: Bool { - (localCapabilities?.encryption ?? false) && sharedKey != nil - } + var encrypted: Bool { get { (localCapabilities?.encryption ?? false) && sharedKey != nil } } } enum CallState { @@ -105,8 +108,8 @@ struct WVAPIMessage: Equatable, Decodable, Encodable { enum WCallCommand: Equatable, Encodable, Decodable { case capabilities(useWorker: Bool? = nil) - case start(media: CallMediaType, aesKey: String? = nil, useWorker: Bool? = nil) - case offer(offer: String, iceCandidates: String, media: CallMediaType, aesKey: String? = nil, useWorker: Bool? = nil) + case start(media: CallMediaType, aesKey: String? = nil, useWorker: Bool? = nil, iceServers: [RTCIceServer]? = nil, relay: Bool? = nil) + case offer(offer: String, iceCandidates: String, media: CallMediaType, aesKey: String? = nil, useWorker: Bool? = nil, iceServers: [RTCIceServer]? = nil, relay: Bool? = nil) case answer(answer: String, iceCandidates: String) case ice(iceCandidates: String) case media(media: CallMediaType, enable: Bool) @@ -121,6 +124,8 @@ enum WCallCommand: Equatable, Encodable, Decodable { case answer case iceCandidates case enable + case iceServers + case relay } var cmdType: String { @@ -143,18 +148,22 @@ enum WCallCommand: Equatable, Encodable, Decodable { case let .capabilities(useWorker): try container.encode("capabilities", forKey: .type) try container.encode(useWorker, forKey: .useWorker) - case let .start(media, aesKey, useWorker): + case let .start(media, aesKey, useWorker, iceServers, relay): try container.encode("start", forKey: .type) try container.encode(media, forKey: .media) try container.encode(aesKey, forKey: .aesKey) try container.encode(useWorker, forKey: .useWorker) - case let .offer(offer, iceCandidates, media, aesKey, useWorker): + try container.encode(iceServers, forKey: .iceServers) + try container.encode(relay, forKey: .relay) + case let .offer(offer, iceCandidates, media, aesKey, useWorker, iceServers, relay): try container.encode("offer", forKey: .type) try container.encode(offer, forKey: .offer) try container.encode(iceCandidates, forKey: .iceCandidates) try container.encode(media, forKey: .media) try container.encode(aesKey, forKey: .aesKey) try container.encode(useWorker, forKey: .useWorker) + try container.encode(iceServers, forKey: .iceServers) + try container.encode(relay, forKey: .relay) case let .answer(answer, iceCandidates): try container.encode("answer", forKey: .type) try container.encode(answer, forKey: .answer) @@ -182,14 +191,18 @@ enum WCallCommand: Equatable, Encodable, Decodable { let media = try container.decode(CallMediaType.self, forKey: CodingKeys.media) let aesKey = try? container.decode(String.self, forKey: CodingKeys.aesKey) let useWorker = try container.decode((Bool?).self, forKey: CodingKeys.useWorker) - self = .start(media: media, aesKey: aesKey, useWorker: useWorker) + let iceServers = try container.decode(([RTCIceServer]?).self, forKey: .iceServers) + let relay = try container.decode((Bool?).self, forKey: .relay) + self = .start(media: media, aesKey: aesKey, useWorker: useWorker, iceServers: iceServers, relay: relay) case "offer": let offer = try container.decode(String.self, forKey: CodingKeys.offer) let iceCandidates = try container.decode(String.self, forKey: CodingKeys.iceCandidates) let media = try container.decode(CallMediaType.self, forKey: CodingKeys.media) let aesKey = try? container.decode(String.self, forKey: CodingKeys.aesKey) let useWorker = try container.decode((Bool?).self, forKey: CodingKeys.useWorker) - self = .offer(offer: offer, iceCandidates: iceCandidates, media: media, aesKey: aesKey, useWorker: useWorker) + let iceServers = try container.decode(([RTCIceServer]?).self, forKey: .iceServers) + let relay = try container.decode((Bool?).self, forKey: .relay) + self = .offer(offer: offer, iceCandidates: iceCandidates, media: media, aesKey: aesKey, useWorker: useWorker, iceServers: iceServers, relay: relay) case "answer": let answer = try container.decode(String.self, forKey: CodingKeys.answer) let iceCandidates = try container.decode(String.self, forKey: CodingKeys.iceCandidates) @@ -216,6 +229,7 @@ enum WCallResponse: Equatable, Decodable { case answer(answer: String, iceCandidates: String) case ice(iceCandidates: String) case connection(state: ConnectionState) + case connected(connectionInfo: ConnectionInfo) case ended case ok case error(message: String) @@ -228,10 +242,8 @@ enum WCallResponse: Equatable, Decodable { case answer case iceCandidates case state + case connectionInfo case message - // TODO remove media, aesKey - case media - case aesKey } var respType: String { @@ -239,9 +251,10 @@ enum WCallResponse: Equatable, Decodable { switch self { case .capabilities: return("capabilities") case .offer: return("offer") - case .answer: return("answer (TODO remove)") + case .answer: return("answer") case .ice: return("ice") case .connection: return("connection") + case .connected: return("connected") case .ended: return("ended") case .ok: return("ok") case .error: return("error") @@ -273,6 +286,9 @@ enum WCallResponse: Equatable, Decodable { case "connection": let state = try container.decode(ConnectionState.self, forKey: CodingKeys.state) self = .connection(state: state) + case "connected": + let connectionInfo = try container.decode(ConnectionInfo.self, forKey: CodingKeys.connectionInfo) + self = .connected(connectionInfo: connectionInfo) case "ended": self = .ended case "ok": @@ -311,6 +327,9 @@ extension WCallResponse: Encodable { case let .connection(state): try container.encode("connection", forKey: .type) try container.encode(state, forKey: .state) + case let .connected(connectionInfo): + try container.encode("connected", forKey: .type) + try container.encode(connectionInfo, forKey: .connectionInfo) case .ended: try container.encode("ended", forKey: .type) case .ok: @@ -330,3 +349,28 @@ struct ConnectionState: Codable, Equatable { var iceGatheringState: String var signalingState: String } + +struct ConnectionInfo: Codable, Equatable { + var localCandidate: RTCIceCandidate? + var remoteCandidate: RTCIceCandidate? +} + +// https://developer.mozilla.org/en-US/docs/Web/API/RTCIceCandidate +struct RTCIceCandidate: Codable, Equatable { + var candidateType: RTCIceCandidateType? +} + +// https://developer.mozilla.org/en-US/docs/Web/API/RTCIceCandidate/type +enum RTCIceCandidateType: String, Codable { + case host = "host" + case serverReflexive = "srflx" + case peerReflexive = "prflx" + case relay = "relay" +} + +// https://developer.mozilla.org/en-US/docs/Web/API/RTCIceServer +struct RTCIceServer: Codable, Equatable { + var urls: [String] + var username: String? = nil + var credential: String? = nil +} diff --git a/apps/ios/Shared/Views/Call/WebRTCView.swift b/apps/ios/Shared/Views/Call/WebRTCView.swift index 39970f2513..fc4d7310b4 100644 --- a/apps/ios/Shared/Views/Call/WebRTCView.swift +++ b/apps/ios/Shared/Views/Call/WebRTCView.swift @@ -31,12 +31,14 @@ class WebRTCCoordinator: NSObject, WKNavigationDelegate, WKScriptMessageHandler didReceive message: WKScriptMessage ) { logger.debug("WebRTCCoordinator.userContentController") - logger.debug("\(String(describing: message.body as? String))") if let msgStr = message.body as? String, let msg: WVAPIMessage = decodeJSON(msgStr) { webViewMsg.wrappedValue = msg + if case .invalid = msg.resp { + logger.error("WebRTCCoordinator.userContentController: invalid message \(String(describing: message.body))") + } } else { - logger.error("WebRTCCoordinator.userContentController: invalid message \(String(describing: message.body))") + logger.error("WebRTCCoordinator.userContentController: message parsing error \(String(describing: message.body))") } } diff --git a/apps/ios/Shared/Views/ChatList/ChatListView.swift b/apps/ios/Shared/Views/ChatList/ChatListView.swift index d4bd2d1f43..1334a62373 100644 --- a/apps/ios/Shared/Views/ChatList/ChatListView.swift +++ b/apps/ios/Shared/Views/ChatList/ChatListView.swift @@ -98,8 +98,12 @@ struct ChatListView: View { } private func answerCallAlert(_ contact: Contact, _ invitation: CallInvitation) { - AlertManager.shared.showAlert(Alert( - title: Text("Incoming call"), + return AlertManager.shared.showAlert(Alert( + title: Text(invitation.callTitle), + message: Text(contact.profile.displayName).bold() + + Text(" wants to connect with you via ") + + Text(invitation.callTypeText) + + Text("\nIf you accept this call, your IP address might be visible to your contact, unless you connect via relay."), primaryButton: .default(Text("Answer")) { if chatModel.activeCallInvitation == nil { DispatchQueue.main.async { @@ -110,7 +114,8 @@ struct ChatListView: View { chatModel.activeCall = Call( contact: contact, callState: .invitationReceived, - localMedia: invitation.peerMedia + localMedia: invitation.peerMedia, + sharedKey: invitation.sharedKey ) showCallView = true chatModel.callCommand = .start(media: invitation.peerMedia, aesKey: invitation.sharedKey, useWorker: true) diff --git a/packages/simplex-chat-webrtc/src/call.ts b/packages/simplex-chat-webrtc/src/call.ts index a256965cd4..559d4348cd 100644 --- a/packages/simplex-chat-webrtc/src/call.ts +++ b/packages/simplex-chat-webrtc/src/call.ts @@ -15,6 +15,7 @@ type WCallResponse = | WCallAnswer | WCallIceCandidates | WRConnection + | WRCallConnected | WRCallEnded | WROk | WRError @@ -22,7 +23,7 @@ type WCallResponse = type WCallCommandTag = "capabilities" | "start" | "offer" | "answer" | "ice" | "media" | "end" -type WCallResponseTag = "capabilities" | "offer" | "answer" | "ice" | "connection" | "ended" | "ok" | "error" +type WCallResponseTag = "capabilities" | "offer" | "answer" | "ice" | "connection" | "connected" | "ended" | "ok" | "error" enum CallMediaType { Audio = "audio", @@ -47,6 +48,8 @@ interface WCStartCall extends IWCallCommand { media: CallMediaType aesKey?: string useWorker?: boolean + iceServers?: RTCIceServer[] + relay?: boolean } interface WCEndCall extends IWCallCommand { @@ -60,6 +63,8 @@ interface WCAcceptOffer extends IWCallCommand { media: CallMediaType aesKey?: string useWorker?: boolean + iceServers?: RTCIceServer[] + relay?: boolean } interface WCallOffer extends IWCallResponse { @@ -105,6 +110,11 @@ interface WRConnection extends IWCallResponse { } } +interface WRCallConnected extends IWCallResponse { + type: "connected" + connectionInfo: ConnectionInfo +} + interface WRCallEnded extends IWCallResponse { type: "ended" } @@ -118,6 +128,12 @@ interface WRError extends IWCallResponse { message: string } +interface ConnectionInfo { + iceCandidatePair: RTCIceCandidatePairStats + localCandidate?: RTCIceCandidate + remoteCandidate?: RTCIceCandidate +} + // for debugging // var sendMessageToNative = ({resp}: WVApiMessage) => console.log(JSON.stringify({command: resp})) var sendMessageToNative = (msg: WVApiMessage) => console.log(JSON.stringify(msg)) @@ -175,16 +191,18 @@ const processCommand = (function () { let activeCall: Call | undefined - function defaultCallConfig(encodedInsertableStreams: boolean): CallConfig { + const defaultIceServers: RTCIceServer[] = [ + {urls: ["stun:stun.simplex.chat:5349"]}, + {urls: ["turn:turn.simplex.chat:5349"], username: "private", credential: "yleob6AVkiNI87hpR94Z"}, + ] + + function getCallConfig(encodedInsertableStreams: boolean, iceServers?: RTCIceServer[], relay?: boolean): CallConfig { return { peerConnectionConfig: { - iceServers: [ - {urls: "stun:stun.simplex.chat:5349"}, - {urls: "turn:turn.simplex.chat:5349", username: "private", credential: "yleob6AVkiNI87hpR94Z"}, - ], + iceServers: iceServers ?? defaultIceServers, iceCandidatePoolSize: 10, encodedInsertableStreams, - // iceTransportPolicy: "relay", + iceTransportPolicy: relay ? "relay" : "all", }, iceCandidates: { delay: 2000, @@ -249,7 +267,7 @@ const processCommand = (function () { return {connection: conn, iceCandidates, localMedia: mediaType, localStream} - function connectionStateChange() { + async function connectionStateChange() { sendMessageToNative({ resp: { type: "connection", @@ -263,10 +281,28 @@ const processCommand = (function () { }) if (conn.connectionState == "disconnected" || conn.connectionState == "failed") { conn.removeEventListener("connectionstatechange", connectionStateChange) - sendMessageToNative({resp: {type: "ended"}}) conn.close() activeCall = undefined resetVideoElements() + setTimeout(() => sendMessageToNative({resp: {type: "ended"}}), 0) + } else if (conn.connectionState == "connected") { + const stats = (await conn.getStats()) as Map + for (const stat of stats.values()) { + const {type, state} = stat + if (type === "candidate-pair" && state === "succeeded") { + const iceCandidatePair = stat as RTCIceCandidatePairStats + const resp: WRCallConnected = { + type: "connected", + connectionInfo: { + iceCandidatePair, + localCandidate: stats.get(iceCandidatePair.localCandidateId), + remoteCandidate: stats.get(iceCandidatePair.remoteCandidateId), + }, + } + setTimeout(() => sendMessageToNative({resp}), 0) + break + } + } } } } @@ -292,12 +328,13 @@ const processCommand = (function () { case "start": console.log("starting call") if (activeCall) { + // TODO cancel current call resp = {type: "error", message: "start: call already started"} } else { - const {media, useWorker} = command + const {media, useWorker, iceServers, relay} = command const encryption = supportsInsertableStreams(useWorker) const aesKey = encryption ? command.aesKey : undefined - activeCall = await initializeCall(defaultCallConfig(encryption && !!aesKey), media, aesKey, useWorker) + activeCall = await initializeCall(getCallConfig(encryption && !!aesKey, iceServers, relay), media, aesKey, useWorker) const pc = activeCall.connection const offer = await pc.createOffer() await pc.setLocalDescription(offer) @@ -306,8 +343,12 @@ const processCommand = (function () { // type: "offer", // offer: serialize(offer), // iceCandidates: await activeCall.iceCandidates, + // capabilities: {encryption}, // media, + // iceServers, + // relay, // aesKey, + // useWorker, // } resp = { type: "offer", @@ -325,8 +366,8 @@ const processCommand = (function () { } else { const offer: RTCSessionDescriptionInit = parse(command.offer) const remoteIceCandidates: RTCIceCandidateInit[] = parse(command.iceCandidates) - const {media, aesKey, useWorker} = command - activeCall = await initializeCall(defaultCallConfig(!!aesKey), media, aesKey, useWorker) + const {media, aesKey, useWorker, iceServers, relay} = command + activeCall = await initializeCall(getCallConfig(!!aesKey, iceServers, relay), media, aesKey, useWorker) const pc = activeCall.connection await pc.setRemoteDescription(new RTCSessionDescription(offer)) const answer = await pc.createAnswer() @@ -439,7 +480,11 @@ const processCommand = (function () { console.log("set up decryption for receiving") setupPeerTransform(TransformOperation.Decrypt, event.receiver as RTCRtpReceiverWithEncryption, worker, aesKey, key) } - remoteStream.addTrack(event.track) + for (const stream of event.streams) { + for (const track of stream.getTracks()) { + remoteStream.addTrack(track) + } + } } // We assume VP8 encoding in the decode/encode stages to get the initial // bytes to pass as plaintext so we enforce that here.