mobile: webrtc calls state machine (#606)

* mobile: webrtc calls state machine

* android: call api types

* android: call api methods

* ios: connect calls via chat UI (WIP)

* ios: webrtc call connects via UI

* core: update call duration/status when x.call.end is received

* improve call UX/UI

* audio calls

* different overlay for audio calls

* toggle video/audio in the call
This commit is contained in:
Evgeny Poberezkin
2022-05-07 09:40:46 +04:00
committed by GitHub
parent 884231369f
commit 29990765e7
26 changed files with 1175 additions and 362 deletions
@@ -4,8 +4,8 @@
<link href="./style.css" rel="stylesheet" />
</head>
<body>
<video id="remote-video-stream" autoplay></video>
<video id="local-video-stream" muted autoplay></video>
<video id="remote-video-stream" autoplay playsinline></video>
<video id="local-video-stream" muted autoplay playsinline></video>
</body>
<footer>
<script src="./call.js"></script>
+44 -29
View File
@@ -11,7 +11,7 @@ const keyAlgorithm = {
length: 256,
};
const keyUsages = ["encrypt", "decrypt"];
let pc;
let activeCall;
const IV_LENGTH = 12;
const initialPlainTextRequired = {
key: 10,
@@ -86,7 +86,7 @@ async function initializeCall(config, mediaType, aesKey) {
sendMessageToNative({ resp: { type: "ice", iceCandidates } });
}
});
return { connection: conn, iceCandidates };
return { connection: conn, iceCandidates, localMedia: mediaType, localStream };
function connectionStateChange() {
sendMessageToNative({
resp: {
@@ -103,17 +103,15 @@ async function initializeCall(config, mediaType, aesKey) {
conn.removeEventListener("connectionstatechange", connectionStateChange);
sendMessageToNative({ resp: { type: "ended" } });
conn.close();
pc = undefined;
activeCall = undefined;
resetVideoElements();
}
}
}
function sendMessageToNative(msg) {
console.log(JSON.stringify(msg));
}
// TODO remove WCallCommand from result type
var sendMessageToNative = (msg) => console.log(JSON.stringify(msg));
async function processCommand(body) {
const { command, corrId } = body;
const { corrId, command } = body;
const pc = activeCall === null || activeCall === void 0 ? void 0 : activeCall.connection;
let resp;
try {
switch (command.type) {
@@ -123,26 +121,31 @@ async function processCommand(body) {
break;
case "start":
console.log("starting call");
if (pc) {
if (activeCall) {
resp = { type: "error", message: "start: call already started" };
}
else if (!supportsInsertableStreams() && command.aesKey) {
resp = { type: "error", message: "start: encryption is not supported" };
}
else {
const encryption = supportsInsertableStreams();
const { media, aesKey } = command;
const call = await initializeCall(defaultCallConfig(!!aesKey), media, aesKey);
const { connection, iceCandidates } = call;
pc = connection;
activeCall = await initializeCall(defaultCallConfig(encryption && !!aesKey), media, encryption ? aesKey : undefined);
const pc = activeCall.connection;
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
// for debugging, returning the command for callee to use
resp = { type: "accept", offer: JSON.stringify(offer), iceCandidates: await iceCandidates, media, aesKey };
// resp = {type: "offer", offer, iceCandidates: await iceCandidates}
// resp = {type: "accept", offer: JSON.stringify(offer), iceCandidates: await iceCandidates, media, aesKey}
resp = {
type: "offer",
offer: JSON.stringify(offer),
iceCandidates: await activeCall.iceCandidates,
capabilities: { encryption },
};
}
break;
case "accept":
if (pc) {
if (activeCall) {
resp = { type: "error", message: "accept: call already started" };
}
else if (!supportsInsertableStreams() && command.aesKey) {
@@ -151,15 +154,18 @@ async function processCommand(body) {
else {
const offer = JSON.parse(command.offer);
const remoteIceCandidates = command.iceCandidates.map((c) => JSON.parse(c));
const call = await initializeCall(defaultCallConfig(!!command.aesKey), command.media, command.aesKey);
const { connection, iceCandidates } = call;
pc = connection;
activeCall = await initializeCall(defaultCallConfig(!!command.aesKey), command.media, command.aesKey);
const pc = activeCall.connection;
await pc.setRemoteDescription(new RTCSessionDescription(offer));
const answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
addIceCandidates(pc, remoteIceCandidates);
// same as command for caller to use
resp = { type: "answer", answer: JSON.stringify(answer), iceCandidates: await iceCandidates };
resp = {
type: "answer",
answer: JSON.stringify(answer),
iceCandidates: await activeCall.iceCandidates,
};
}
break;
case "answer":
@@ -190,10 +196,22 @@ async function processCommand(body) {
resp = { type: "error", message: "ice: call not started" };
}
break;
case "media":
if (!activeCall) {
resp = { type: "error", message: "media: call not started" };
}
else if (activeCall.localMedia == CallMediaType.Audio && command.media == CallMediaType.Video) {
resp = { type: "error", message: "media: no video" };
}
else {
enableMedia(activeCall.localStream, command.media, command.enable);
resp = { type: "ok" };
}
break;
case "end":
if (pc) {
pc.close();
pc = undefined;
activeCall = undefined;
resetVideoElements();
resp = { type: "ok" };
}
@@ -209,7 +227,7 @@ async function processCommand(body) {
catch (e) {
resp = { type: "error", message: e.message };
}
const apiResp = { resp, corrId };
const apiResp = { corrId, resp, command };
sendMessageToNative(apiResp);
return apiResp;
}
@@ -321,14 +339,11 @@ function getVideoElements() {
// video.style.opacity = "1"
// }
// }
// what does it do?
// function toggleVideo(b) {
// if (b == "true") {
// localStream.getVideoTracks()[0].enabled = true
// } else {
// localStream.getVideoTracks()[0].enabled = false
// }
// }
function enableMedia(s, media, enable) {
const tracks = media == CallMediaType.Video ? s.getVideoTracks() : s.getAudioTracks();
for (const t of tracks)
t.enabled = enable;
}
/* Stream Transforms */
function setupPeerTransform(peer, transform) {
const streams = peer.createEncodedStreams();
@@ -5,6 +5,7 @@ html,
body {
padding: 0;
margin: 0;
background-color: black;
}
#remote-video-stream {
position: absolute;
@@ -16,6 +16,7 @@ import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.dp
import chat.simplex.app.*
import chat.simplex.app.R
import chat.simplex.app.views.call.*
import chat.simplex.app.views.helpers.*
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
@@ -320,6 +321,51 @@ open class ChatController(private val ctrl: ChatCtrl, private val ntfManager: Nt
return false
}
suspend fun apiSendCallInvitation(contact: Contact, callType: CallType): Boolean {
val r = sendCmd(CC.ApiSendCallInvitation(contact, callType))
return r is CR.CmdOk
}
suspend fun apiRejectCall(contact: Contact): Boolean {
val r = sendCmd(CC.ApiRejectCall(contact))
return r is CR.CmdOk
}
suspend fun apiSendCallOffer(contact: Contact, rtcSession: String, rtcIceCandidates: List<String>, media: CallMediaType, capabilities: CallCapabilities): Boolean {
val webRtcSession = WebRTCSession(rtcSession, rtcIceCandidates)
val callOffer = WebRTCCallOffer(CallType(media, capabilities), webRtcSession)
val r = sendCmd(CC.ApiSendCallOffer(contact, callOffer))
return r is CR.CmdOk
}
suspend fun apiSendCallAnswer(contact: Contact, rtcSession: String, rtcIceCandidates: List<String>): Boolean {
val answer = WebRTCSession(rtcSession, rtcIceCandidates)
val r = sendCmd(CC.ApiSendCallAnswer(contact, answer))
return r is CR.CmdOk
}
suspend fun apiSendCallExtraInfo(contact: Contact, rtcIceCandidates: List<String>): Boolean {
val extraInfo = WebRTCExtraInfo(rtcIceCandidates)
val r = sendCmd(CC.ApiSendCallExtraInfo(contact, extraInfo))
return r is CR.CmdOk
}
suspend fun apiEndCall(contact: Contact): Boolean {
val r = sendCmd(CC.ApiEndCall(contact))
return r is CR.CmdOk
}
suspend fun apiCallStatus(contact: Contact, status: String): Boolean {
try {
val callStatus = WebRTCCallStatus.valueOf(status)
val r = sendCmd(CC.ApiCallStatus(contact, callStatus))
return r is CR.CmdOk
} catch (e: Error) {
Log.d(TAG,"apiCallStatus: call status $status not used")
return false
}
}
suspend fun apiChatRead(type: ChatType, id: Long, range: CC.ItemRange): Boolean {
val r = sendCmd(CC.ApiChatRead(type, id, range))
if (r is CR.CmdOk) return true
@@ -562,6 +608,13 @@ sealed class CC {
class CreateMyAddress: CC()
class DeleteMyAddress: CC()
class ShowMyAddress: CC()
class ApiSendCallInvitation(val contact: Contact, val callType: CallType): CC()
class ApiRejectCall(val contact: Contact): CC()
class ApiSendCallOffer(val contact: Contact, val callOffer: WebRTCCallOffer): CC()
class ApiSendCallAnswer(val contact: Contact, val answer: WebRTCSession): CC()
class ApiSendCallExtraInfo(val contact: Contact, val extraInfo: WebRTCExtraInfo): CC()
class ApiEndCall(val contact: Contact): CC()
class ApiCallStatus(val contact: Contact, val callStatus: WebRTCCallStatus): CC()
class ApiAcceptContact(val contactReqId: Long): CC()
class ApiRejectContact(val contactReqId: Long): CC()
class ApiChatRead(val type: ChatType, val id: Long, val range: ItemRange): CC()
@@ -590,6 +643,13 @@ sealed class CC {
is ShowMyAddress -> "/show_address"
is ApiAcceptContact -> "/_accept $contactReqId"
is ApiRejectContact -> "/_reject $contactReqId"
is ApiSendCallInvitation -> "/_call invite @${contact.apiId} ${json.encodeToString(callType)}"
is ApiRejectCall -> "/_call reject @${contact.apiId}"
is ApiSendCallOffer -> "/_call offer @${contact.apiId} ${json.encodeToString(callOffer)}"
is ApiSendCallAnswer -> "/_call answer @${contact.apiId} ${json.encodeToString(answer)}"
is ApiSendCallExtraInfo -> "/_call extra @${contact.apiId} ${json.encodeToString(extraInfo)}"
is ApiEndCall -> "/_call end @${contact.apiId}"
is ApiCallStatus -> "/_call status @${contact.apiId} ${callStatus}"
is ApiChatRead -> "/_read chat ${chatRef(type, id)} from=${range.from} to=${range.to}"
is ReceiveFile -> "/freceive $fileId"
}
@@ -617,6 +677,13 @@ sealed class CC {
is ShowMyAddress -> "showMyAddress"
is ApiAcceptContact -> "apiAcceptContact"
is ApiRejectContact -> "apiRejectContact"
is ApiSendCallInvitation -> "apiSendCallInvitation"
is ApiRejectCall -> "apiRejectCall"
is ApiSendCallOffer -> "apiSendCallOffer"
is ApiSendCallAnswer -> "apiSendCallAnswer"
is ApiSendCallExtraInfo -> "apiSendCallExtraInfo"
is ApiEndCall -> "apiEndCall"
is ApiCallStatus -> "apiCallStatus"
is ApiChatRead -> "apiChatRead"
is ReceiveFile -> "receiveFile"
}
@@ -706,6 +773,11 @@ sealed class CR {
@Serializable @SerialName("sndFileCancelled") class SndFileCancelled(val chatItem: AChatItem, val sndFileTransfer: SndFileTransfer): CR()
@Serializable @SerialName("sndFileRcvCancelled") class SndFileRcvCancelled(val chatItem: AChatItem, val sndFileTransfer: SndFileTransfer): CR()
@Serializable @SerialName("sndGroupFileCancelled") class SndGroupFileCancelled(val chatItem: AChatItem, val fileTransferMeta: FileTransferMeta, val sndFileTransfers: List<SndFileTransfer>): CR()
@Serializable @SerialName("callInvitation") class CallInvitation(val contact: Contact, val callType: CallType, val sharedKey: String?): CR()
@Serializable @SerialName("callOffer") class CallOffer(val contact: Contact, val callType: CallType, val offer: WebRTCSession, val sharedKey: String?, val askConfirmation: Boolean): CR()
@Serializable @SerialName("callAnswer") class CallAnswer(val contact: Contact, val answer: WebRTCSession): CR()
@Serializable @SerialName("callExtraInfo") class CallExtraInfo(val contact: Contact, val extraInfo: WebRTCExtraInfo): CR()
@Serializable @SerialName("callEnded") class CallEnded(val contact: Contact): CR()
@Serializable @SerialName("newContactConnection") class NewContactConnection(val connection: PendingContactConnection): CR()
@Serializable @SerialName("contactConnectionDeleted") class ContactConnectionDeleted(val connection: PendingContactConnection): CR()
@Serializable @SerialName("cmdOk") class CmdOk: CR()
@@ -758,6 +830,11 @@ sealed class CR {
is SndFileRcvCancelled -> "sndFileRcvCancelled"
is SndFileStart -> "sndFileStart"
is SndGroupFileCancelled -> "sndGroupFileCancelled"
is CallInvitation -> "callInvitation"
is CallOffer -> "callOffer"
is CallAnswer -> "callAnswer"
is CallExtraInfo -> "callExtraInfo"
is CallEnded -> "callEnded"
is NewContactConnection -> "newContactConnection"
is ContactConnectionDeleted -> "contactConnectionDeleted"
is CmdOk -> "cmdOk"
@@ -811,6 +888,11 @@ sealed class CR {
is SndFileRcvCancelled -> json.encodeToString(chatItem)
is SndFileStart -> json.encodeToString(chatItem)
is SndGroupFileCancelled -> json.encodeToString(chatItem)
is CallInvitation -> "contact: ${contact.id}\ncallType: $callType\nsharedKey: ${sharedKey ?: ""}"
is CallOffer -> "contact: ${contact.id}\ncallType: $callType\nsharedKey: ${sharedKey ?: ""}\naskConfirmation: $askConfirmation\noffer: ${json.encodeToString(offer)}"
is CallAnswer -> "contact: ${contact.id}\nanswer: ${json.encodeToString(answer)}"
is CallExtraInfo -> "contact: ${contact.id}\nextraInfo: ${json.encodeToString(extraInfo)}"
is CallEnded -> "contact: ${contact.id}"
is NewContactConnection -> json.encodeToString(connection)
is ContactConnectionDeleted -> json.encodeToString(connection)
is CmdOk -> noDetails()
@@ -0,0 +1,66 @@
package chat.simplex.app.views.call
import chat.simplex.app.model.CR
import chat.simplex.app.model.User
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
@Serializable
sealed class WCallCommand {
@Serializable @SerialName("capabilities") class Capabilities(): WCallCommand()
@Serializable @SerialName("start") class Start(val media: CallMediaType, val aesKey: String? = null): WCallCommand()
@Serializable @SerialName("accept") class Accept(val offer: String, val iceCandidates: List<String>, val media: CallMediaType, val aesKey: String? = null): WCallCommand()
@Serializable @SerialName("answer") class Answer (val answer: String, val iceCandidates: List<String>): WCallCommand()
@Serializable @SerialName("ice") class Ice(val iceCandidates: List<String>): WCallCommand()
@Serializable @SerialName("end") class End(): WCallCommand()
}
@Serializable
sealed class WCallResponse {
@Serializable @SerialName("capabilities") class Capabilities(val capabilities: CallCapabilities): WCallResponse()
@Serializable @SerialName("offer") class Offer(val offer: String, val iceCandidates: List<String>): WCallResponse()
// TODO remove accept, it is needed for debugging
@Serializable @SerialName("accept") class Accept(val offer: String, val iceCandidates: List<String>, val media: CallMediaType, val aesKey: String? = null): WCallResponse()
@Serializable @SerialName("answer") class Answer(val answer: String, val iceCandidates: List<String>): WCallResponse()
@Serializable @SerialName("ice") class Ice(val iceCandidates: List<String>): WCallResponse()
@Serializable @SerialName("connection") class Connection(val state: ConnectionState): WCallResponse()
@Serializable @SerialName("ended") class Ended(): WCallResponse()
@Serializable @SerialName("ok") class Ok(): WCallResponse()
@Serializable @SerialName("error") class Error(val message: String): WCallResponse()
@Serializable class Invalid(val str: String): WCallResponse()
}
@Serializable
class WebRTCCallOffer(val callType: CallType, val rtcSession: WebRTCSession)
@Serializable
class WebRTCSession(val rtcSession: String, val rtcIceCandidates: List<String>)
@Serializable
class WebRTCExtraInfo(val rtcIceCandidates: List<String>)
@Serializable
class CallType(val media: CallMediaType, val capabilities: CallCapabilities)
enum class WebRTCCallStatus(val status: String) {
Connected("connected"),
Disconnected("disconnected"),
Failed("failed")
}
@Serializable
enum class CallMediaType(val media: String) {
Video("video"),
Audio("audio")
}
@Serializable
class CallCapabilities(val encryption: Boolean)
@Serializable
class ConnectionState(
val connectionState: String,
val iceConnectionState: String,
val iceGatheringState: String,
val signalingState: String
)
+5
View File
@@ -25,6 +25,11 @@ final class ChatModel: ObservableObject {
@Published var appOpenUrl: URL?
@Published var deviceToken: String?
@Published var tokenStatus = NtfTknStatus.new
// current WebRTC call
@Published var callInvitations: Dictionary<String, CallInvitation> = [:]
@Published var activeCallInvitation: ContactRef?
@Published var activeCall: Call?
@Published var callCommand: WCallCommand?
var messageDelivery: Dictionary<Int64, () -> Void> = [:]
+48 -3
View File
@@ -10,7 +10,9 @@ import Foundation
import UserNotifications
import UIKit
let ntfActionAccept = "NTF_ACT_ACCEPT"
let ntfActionAcceptContact = "NTF_ACT_ACCEPT_CONTACT"
let ntfActionAcceptCall = "NTF_ACT_ACCEPT_CALL"
let ntfActionRejectCall = "NTF_ACT_REJECT_CALL"
private let ntfTimeInterval: TimeInterval = 1
@@ -28,10 +30,33 @@ class NtfManager: NSObject, UNUserNotificationCenterDelegate, ObservableObject {
logger.debug("NtfManager.userNotificationCenter: didReceive")
let content = response.notification.request.content
let chatModel = ChatModel.shared
if content.categoryIdentifier == ntfCategoryContactRequest && response.actionIdentifier == ntfActionAccept,
let action = response.actionIdentifier
if content.categoryIdentifier == ntfCategoryContactRequest && action == ntfActionAcceptContact,
let chatId = content.userInfo["chatId"] as? String,
case let .contactRequest(contactRequest) = chatModel.getChat(chatId)?.chatInfo {
Task { await acceptContactRequest(contactRequest) }
} else if content.categoryIdentifier == ntfCategoryCallInvitation && (action == ntfActionAcceptCall || action == ntfActionRejectCall),
let chatId = content.userInfo["chatId"] as? String,
case let .direct(contact) = chatModel.getChat(chatId)?.chatInfo,
let invitation = chatModel.callInvitations[chatId] {
if action == ntfActionAcceptCall {
chatModel.activeCall = Call(contact: contact, callState: .invitationReceived, localMedia: invitation.peerMedia)
chatModel.chatId = nil
chatModel.callCommand = .start(media: invitation.peerMedia, aesKey: invitation.sharedKey)
} else {
Task {
do {
try await apiRejectCall(contact)
if chatModel.activeCall?.contact.id == chatId {
DispatchQueue.main.async {
chatModel.callCommand = .end
chatModel.activeCall = nil
}
}
}
}
}
chatModel.callInvitations.removeValue(forKey: chatId)
} else {
chatModel.chatId = content.targetContentIdentifier
}
@@ -88,7 +113,7 @@ class NtfManager: NSObject, UNUserNotificationCenterDelegate, ObservableObject {
UNNotificationCategory(
identifier: ntfCategoryContactRequest,
actions: [UNNotificationAction(
identifier: ntfActionAccept,
identifier: ntfActionAcceptContact,
title: NSLocalizedString("Accept", comment: "accept contact request via notification")
)],
intentIdentifiers: [],
@@ -106,6 +131,21 @@ class NtfManager: NSObject, UNUserNotificationCenterDelegate, ObservableObject {
intentIdentifiers: [],
hiddenPreviewsBodyPlaceholder: NSLocalizedString("New message", comment: "notification")
),
UNNotificationCategory(
identifier: ntfCategoryCallInvitation,
actions: [
UNNotificationAction(
identifier: ntfActionAcceptCall,
title: NSLocalizedString("Answer", comment: "accept incoming call via notification")
),
UNNotificationAction(
identifier: ntfActionRejectCall,
title: NSLocalizedString("Ignore", comment: "ignore incoming call via notification")
)
],
intentIdentifiers: [],
hiddenPreviewsBodyPlaceholder: NSLocalizedString("Incoming call", comment: "notification")
),
// TODO remove
UNNotificationCategory(
identifier: ntfCategoryCheckingMessages,
@@ -154,6 +194,11 @@ class NtfManager: NSObject, UNUserNotificationCenterDelegate, ObservableObject {
addNotification(createMessageReceivedNtf(cInfo, cItem))
}
func notifyCallInvitation(_ contact: Contact, _ invitation: CallInvitation) {
logger.debug("NtfManager.notifyCallInvitation")
addNotification(createCallInvitationNtf(contact, invitation))
}
// TODO remove
func notifyCheckingMessages() {
logger.debug("NtfManager.notifyCheckingMessages")
+49 -6
View File
@@ -37,6 +37,14 @@ enum ChatCommand {
case showMyAddress
case apiAcceptContact(contactReqId: Int64)
case apiRejectContact(contactReqId: Int64)
// WebRTC calls
case apiSendCallInvitation(contact: Contact, callType: CallType)
case apiRejectCall(contact: Contact)
case apiSendCallOffer(contact: Contact, callOffer: WebRTCCallOffer)
case apiSendCallAnswer(contact: Contact, answer: WebRTCSession)
case apiSendCallExtraInfo(contact: Contact, extraInfo: WebRTCExtraInfo)
case apiEndCall(contact: Contact)
case apiCallStatus(contact: Contact, callStatus: WebRTCCallStatus)
case apiChatRead(type: ChatType, id: Int64, itemRange: (Int64, Int64))
case receiveFile(fileId: Int64)
case string(String)
@@ -71,6 +79,13 @@ enum ChatCommand {
case .showMyAddress: return "/show_address"
case let .apiAcceptContact(contactReqId): return "/_accept \(contactReqId)"
case let .apiRejectContact(contactReqId): return "/_reject \(contactReqId)"
case let .apiSendCallInvitation(contact, callType): return "/_call invite @\(contact.apiId) \(encodeJSON(callType))"
case let .apiRejectCall(contact): return "/_call reject @\(contact.apiId)"
case let .apiSendCallOffer(contact, callOffer): return "/_call offer @\(contact.apiId) \(encodeJSON(callOffer))"
case let .apiSendCallAnswer(contact, answer): return "/_call answer @\(contact.apiId) \(encodeJSON(answer))"
case let .apiSendCallExtraInfo(contact, extraInfo): return "/_call extra @\(contact.apiId) \(encodeJSON(extraInfo))"
case let .apiEndCall(contact): return "/_call end @\(contact.apiId)"
case let .apiCallStatus(contact, callStatus): return "/_call status @\(contact.apiId) \(callStatus.rawValue)"
case let .apiChatRead(type, id, itemRange: (from, to)): return "/_read chat \(ref(type, id)) from=\(from) to=\(to)"
case let .receiveFile(fileId): return "/freceive \(fileId)"
case let .string(str): return str
@@ -106,6 +121,13 @@ enum ChatCommand {
case .showMyAddress: return "showMyAddress"
case .apiAcceptContact: return "apiAcceptContact"
case .apiRejectContact: return "apiRejectContact"
case .apiSendCallInvitation: return "apiSendCallInvitation"
case .apiRejectCall: return "apiRejectCall"
case .apiSendCallOffer: return "apiSendCallOffer"
case .apiSendCallAnswer: return "apiSendCallAnswer"
case .apiSendCallExtraInfo: return "apiSendCallExtraInfo"
case .apiEndCall: return "apiEndCall"
case .apiCallStatus: return "apiCallStatus"
case .apiChatRead: return "apiChatRead"
case .receiveFile: return "receiveFile"
case .string: return "console command"
@@ -173,6 +195,11 @@ enum ChatResponse: Decodable, Error {
case sndFileCancelled(chatItem: AChatItem, sndFileTransfer: SndFileTransfer)
case sndFileRcvCancelled(chatItem: AChatItem, sndFileTransfer: SndFileTransfer)
case sndGroupFileCancelled(chatItem: AChatItem, fileTransferMeta: FileTransferMeta, sndFileTransfers: [SndFileTransfer])
case callInvitation(contact: Contact, callType: CallType, sharedKey: String?)
case callOffer(contact: Contact, callType: CallType, offer: WebRTCSession, sharedKey: String?, askConfirmation: Bool)
case callAnswer(contact: Contact, answer: WebRTCSession)
case callExtraInfo(contact: Contact, extraInfo: WebRTCExtraInfo)
case callEnded(contact: Contact)
case ntfTokenStatus(status: NtfTknStatus)
case newContactConnection(connection: PendingContactConnection)
case contactConnectionDeleted(connection: PendingContactConnection)
@@ -227,6 +254,11 @@ enum ChatResponse: Decodable, Error {
case .sndFileCancelled: return "sndFileCancelled"
case .sndFileRcvCancelled: return "sndFileRcvCancelled"
case .sndGroupFileCancelled: return "sndGroupFileCancelled"
case .callInvitation: return "callInvitation"
case .callOffer: return "callOffer"
case .callAnswer: return "callAnswer"
case .callExtraInfo: return "callExtraInfo"
case .callEnded: return "callEnded"
case .ntfTokenStatus: return "ntfTokenStatus"
case .newContactConnection: return "newContactConnection"
case .contactConnectionDeleted: return "contactConnectionDeleted"
@@ -284,6 +316,11 @@ enum ChatResponse: Decodable, Error {
case let .sndFileCancelled(chatItem, _): return String(describing: chatItem)
case let .sndFileRcvCancelled(chatItem, _): return String(describing: chatItem)
case let .sndGroupFileCancelled(chatItem, _, _): return String(describing: chatItem)
case let .callInvitation(contact, callType, sharedKey): return "contact: \(contact.id)\ncallType: \(String(describing: callType))\nsharedKey: \(sharedKey ?? "")"
case let .callOffer(contact, callType, offer, sharedKey, askConfirmation): return "contact: \(contact.id)\ncallType: \(String(describing: callType))\nsharedKey: \(sharedKey ?? "")\naskConfirmation: \(askConfirmation)\noffer: \(String(describing: offer))"
case let .callAnswer(contact, answer): return "contact: \(contact.id)\nanswer: \(String(describing: answer))"
case let .callExtraInfo(contact, extraInfo): return "contact: \(contact.id)\nextraInfo: \(String(describing: extraInfo))"
case let .callEnded(contact): return "contact: \(contact.id)"
case let .ntfTokenStatus(status): return String(describing: status)
case let .newContactConnection(connection): return String(describing: connection)
case let .contactConnectionDeleted(connection): return String(describing: connection)
@@ -303,12 +340,18 @@ struct ComposedMessage: Encodable {
var msgContent: MsgContent
}
private func decodeCJSON<T: Decodable>(_ cjson: UnsafePointer<CChar>) -> T? {
let s = String.init(cString: cjson)
let d = s.data(using: .utf8)!
// let p = UnsafeMutableRawPointer.init(mutating: UnsafeRawPointer(cjson))
// let d = Data.init(bytesNoCopy: p, count: strlen(cjson), deallocator: .free)
return try? jsonDecoder.decode(T.self, from: d)
func decodeJSON<T: Decodable>(_ json: String) -> T? {
if let data = json.data(using: .utf8) {
return try? jsonDecoder.decode(T.self, from: data)
}
return nil
}
func decodeCJSON<T: Decodable>(_ cjson: UnsafePointer<CChar>) -> T? {
// TODO is there a way to do it without copying the data? e.g:
// let p = UnsafeMutableRawPointer.init(mutating: UnsafeRawPointer(cjson))
// let d = Data.init(bytesNoCopy: p, count: strlen(cjson), deallocator: .free)
decodeJSON(String.init(cString: cjson))
}
private func getJSONObject(_ cjson: UnsafePointer<CChar>) -> NSDictionary? {
@@ -0,0 +1,48 @@
//
// CallTypes.swift
// SimpleX (iOS)
//
// Created by Evgeny on 05/05/2022.
// Copyright © 2022 SimpleX Chat. All rights reserved.
//
import Foundation
struct WebRTCCallOffer: Encodable {
var callType: CallType
var rtcSession: WebRTCSession
}
struct WebRTCSession: Codable {
var rtcSession: String
var rtcIceCandidates: [String]
}
struct WebRTCExtraInfo: Codable {
var rtcIceCandidates: [String]
}
struct CallInvitation {
var peerMedia: CallMediaType
var sharedKey: String?
}
struct CallType: Codable {
var media: CallMediaType
var capabilities: CallCapabilities
}
enum CallMediaType: String, Codable, Equatable {
case video = "video"
case audio = "audio"
}
struct CallCapabilities: Codable, Equatable {
var encryption: Bool
}
enum WebRTCCallStatus: String, Encodable {
case connected = "connected"
case disconnected = "disconnected"
case failed = "failed"
}
+38 -1
View File
@@ -218,7 +218,7 @@ struct Contact: Identifiable, Decodable, NamedChat {
)
}
struct ContactRef: Decodable {
struct ContactRef: Decodable, Equatable {
var contactId: Int64
var localDisplayName: ContactName
@@ -460,6 +460,14 @@ struct ChatItem: Identifiable, Decodable {
}
}
func isCall() -> Bool {
switch content {
case .sndCall: return true
case .rcvCall: return true
default: return false
}
}
var memberDisplayName: String? {
get {
if case let .groupRcv(groupMember) = chatDir {
@@ -578,6 +586,8 @@ enum CIContent: Decodable, ItemContent {
case rcvMsgContent(msgContent: MsgContent)
case sndDeleted(deleteMode: CIDeleteMode)
case rcvDeleted(deleteMode: CIDeleteMode)
case sndCall(status: CICallStatus, duration: Int)
case rcvCall(status: CICallStatus, duration: Int)
var text: String {
get {
@@ -586,6 +596,8 @@ enum CIContent: Decodable, ItemContent {
case let .rcvMsgContent(mc): return mc.text
case .sndDeleted: return NSLocalizedString("deleted", comment: "deleted chat item")
case .rcvDeleted: return NSLocalizedString("deleted", comment: "deleted chat item")
case let .sndCall(status, duration): return status.text(duration)
case let .rcvCall(status, duration): return status.text(duration)
}
}
}
@@ -838,3 +850,28 @@ struct SndFileTransfer: Decodable {
struct FileTransferMeta: Decodable {
}
enum CICallStatus: String, Decodable {
case pending
case missed
case rejected
case accepted
case negotiated
case progress
case ended
case error
func text(_ sec: Int) -> String {
switch self {
case .pending: return "calling..."
case .negotiated: return "connecting..."
case .progress: return "in progress"
case .ended: return "ended \(duration(sec))"
default: return self.rawValue
}
}
func duration(_ sec: Int) -> String {
String(format: "%02d:%02d", sec / 60, sec % 60)
}
}
@@ -12,6 +12,7 @@ import UserNotifications
let ntfCategoryContactRequest = "NTF_CAT_CONTACT_REQUEST"
let ntfCategoryContactConnected = "NTF_CAT_CONTACT_CONNECTED"
let ntfCategoryMessageReceived = "NTF_CAT_MESSAGE_RECEIVED"
let ntfCategoryCallInvitation = "NTF_CAT_CALL_INVITATION"
let ntfCategoryCheckMessage = "NTF_CAT_CHECK_MESSAGE"
// TODO remove
let ntfCategoryCheckingMessages = "NTF_CAT_CHECKING_MESSAGES"
@@ -48,6 +49,16 @@ func createMessageReceivedNtf(_ cInfo: ChatInfo, _ cItem: ChatItem) -> UNMutable
)
}
func createCallInvitationNtf(_ contact: Contact, _ invitation: CallInvitation) -> UNMutableNotificationContent {
createNotification(
categoryIdentifier: ntfCategoryCallInvitation,
title: "\(contact.chatViewName):",
body: String.localizedStringWithFormat(NSLocalizedString("Incoming %@ call", comment: "notification body"), invitation.peerMedia.rawValue),
targetContentIdentifier: nil,
userInfo: ["chatId": contact.id]
)
}
func createNotification(categoryIdentifier: String, title: String, subtitle: String? = nil, body: String? = nil,
targetContentIdentifier: String? = nil, userInfo: [AnyHashable : Any] = [:]) -> UNMutableNotificationContent {
let content = UNMutableNotificationContent()
+75 -1
View File
@@ -360,6 +360,42 @@ func rejectContactRequest(_ contactRequest: UserContactRequest) async {
}
}
func apiSendCallInvitation(_ contact: Contact, _ callType: CallType) async throws {
try await sendCommandOkResp(.apiSendCallInvitation(contact: contact, callType: callType))
}
func apiRejectCall(_ contact: Contact) async throws {
try await sendCommandOkResp(.apiRejectCall(contact: contact))
}
func apiSendCallOffer(_ contact: Contact, _ rtcSession: String, _ rtcIceCandidates: [String], media: CallMediaType, capabilities: CallCapabilities) async throws {
let webRtcSession = WebRTCSession(rtcSession: rtcSession, rtcIceCandidates: rtcIceCandidates)
let callOffer = WebRTCCallOffer(callType: CallType(media: media, capabilities: capabilities), rtcSession: webRtcSession)
try await sendCommandOkResp(.apiSendCallOffer(contact: contact, callOffer: callOffer))
}
func apiSendCallAnswer(_ contact: Contact, _ rtcSession: String, _ rtcIceCandidates: [String]) async throws {
let answer = WebRTCSession(rtcSession: rtcSession, rtcIceCandidates: rtcIceCandidates)
try await sendCommandOkResp(.apiSendCallAnswer(contact: contact, answer: answer))
}
func apiSendCallExtraInfo(_ contact: Contact, _ rtcIceCandidates: [String]) async throws {
let extraInfo = WebRTCExtraInfo(rtcIceCandidates: rtcIceCandidates)
try await sendCommandOkResp(.apiSendCallExtraInfo(contact: contact, extraInfo: extraInfo))
}
func apiEndCall(_ contact: Contact) async throws {
try await sendCommandOkResp(.apiEndCall(contact: contact))
}
func apiCallStatus(_ contact: Contact, _ status: String) async throws {
if let callStatus = WebRTCCallStatus.init(rawValue: status) {
try await sendCommandOkResp(.apiCallStatus(contact: contact, callStatus: callStatus))
} else {
logger.debug("apiCallStatus: call status \(status) not used")
}
}
func markChatRead(_ chat: Chat) async {
do {
let minItemId = chat.chatStats.minUnreadItemId
@@ -486,7 +522,9 @@ func processReceivedMsg(_ res: ChatResponse) {
await receiveFile(fileId: file.fileId)
}
}
NtfManager.shared.notifyMessageReceived(cInfo, cItem)
if !cItem.isCall() {
NtfManager.shared.notifyMessageReceived(cInfo, cItem)
}
case let .chatItemStatusUpdated(aChatItem):
let cInfo = aChatItem.chatInfo
let cItem = aChatItem.chatItem
@@ -530,9 +568,45 @@ func processReceivedMsg(_ res: ChatResponse) {
let fileName = cItem.file?.filePath {
removeFile(fileName)
}
case let .callInvitation(contact, callType, sharedKey):
let invitation = CallInvitation(peerMedia: callType.media, sharedKey: sharedKey)
m.callInvitations[contact.id] = invitation
if (m.activeCallInvitation == nil) {
m.activeCallInvitation = ContactRef(contactId: contact.apiId, localDisplayName: contact.localDisplayName)
}
NtfManager.shared.notifyCallInvitation(contact, invitation)
case let .callOffer(contact, callType, offer, sharedKey, _):
// TODO askConfirmation?
// TODO check encryption is compatible
withCall(contact) { call in
m.activeCall = call.copy(callState: .offerReceived, peerMedia: callType.media, sharedKey: sharedKey)
m.callCommand = .accept(offer: offer.rtcSession, iceCandidates: offer.rtcIceCandidates, media: callType.media, aesKey: sharedKey)
}
case let .callAnswer(contact, answer):
withCall(contact) { call in
m.activeCall = call.copy(callState: .negotiated)
m.callCommand = .answer(answer: answer.rtcSession, iceCandidates: answer.rtcIceCandidates)
}
case let .callExtraInfo(contact, extraInfo):
withCall(contact) { _ in
m.callCommand = .ice(iceCandidates: extraInfo.rtcIceCandidates)
}
case let .callEnded(contact):
m.activeCallInvitation = nil
withCall(contact) { _ in
m.callCommand = .end
}
default:
logger.debug("unsupported event: \(res.responseType)")
}
func withCall(_ contact: Contact, _ perform: (Call) -> Void) {
if let call = m.activeCall, call.contact.apiId == contact.apiId {
perform(call)
} else {
logger.debug("processReceivedMsg: ignoring \(res.responseType), not in call with the contact \(contact.id)")
}
}
}
}
@@ -0,0 +1,188 @@
//
// ActiveCallView.swift
// SimpleX (iOS)
//
// Created by Evgeny on 05/05/2022.
// Copyright © 2022 SimpleX Chat. All rights reserved.
//
import SwiftUI
struct ActiveCallView: View {
@EnvironmentObject var chatModel: ChatModel
@Environment(\.dismiss) private var dismiss
@Binding var showCallView: Bool
@State private var coordinator: WebRTCCoordinator? = nil
@State private var webViewReady: Bool = false
@State private var webViewMsg: WVAPIMessage? = nil
var body: some View {
ZStack(alignment: .bottom) {
WebRTCView(coordinator: $coordinator, webViewReady: $webViewReady, webViewMsg: $webViewMsg)
.onAppear() { sendCommandToWebView() }
.onChange(of: chatModel.callCommand) { _ in sendCommandToWebView() }
.onChange(of: webViewReady) { _ in sendCommandToWebView() }
.onChange(of: webViewMsg) { _ in processWebViewMessage() }
.background(.black)
ActiveCallOverlay(call: chatModel.activeCall, dismiss: { dismiss() })
}
}
private func sendCommandToWebView() {
if chatModel.activeCall != nil && webViewReady,
let cmd = chatModel.callCommand,
let c = coordinator {
chatModel.callCommand = nil
logger.debug("ActiveCallView: command \(cmd.cmdType)")
c.sendCommand(command: cmd)
}
}
private func processWebViewMessage() {
let m = chatModel
if let msg = webViewMsg,
let call = chatModel.activeCall {
logger.debug("ActiveCallView: response \(msg.resp.respType)")
Task {
switch msg.resp {
case let .capabilities(capabilities):
let callType = CallType(media: call.localMedia, capabilities: capabilities)
try await apiSendCallInvitation(call.contact, callType)
m.activeCall = call.copy(callState: .invitationSent, localCapabilities: capabilities)
case let .offer(offer, iceCandidates, capabilities):
try await apiSendCallOffer(call.contact, offer, iceCandidates,
media: call.localMedia, capabilities: capabilities)
m.activeCall = call.copy(callState: .offerSent, localCapabilities: capabilities)
case let .answer(answer, iceCandidates):
try await apiSendCallAnswer(call.contact, answer, iceCandidates)
m.activeCall = call.copy(callState: .negotiated)
case let .ice(iceCandidates):
try await apiSendCallExtraInfo(call.contact, iceCandidates)
case let .connection(state):
if let callStatus = WebRTCCallStatus.init(rawValue: state.connectionState),
case .connected = callStatus {
m.activeCall = call.copy(callState: .connected)
}
try await apiCallStatus(call.contact, state.connectionState)
case .ended:
m.activeCall = nil
m.activeCallInvitation = nil
m.callCommand = nil
showCallView = false
case .ok:
switch msg.command {
case let .media(media, enable):
switch media {
case .video: m.activeCall = call.copy(videoEnabled: enable)
case .audio: m.activeCall = call.copy(audioEnabled: enable)
}
case .end:
m.activeCall = nil
m.activeCallInvitation = nil
m.callCommand = nil
showCallView = false
default: ()
}
case let .error(message):
logger.debug("ActiveCallView: command error: \(message)")
case let .invalid(type):
logger.debug("ActiveCallView: invalid response: \(type)")
}
}
}
}
}
struct ActiveCallOverlay: View {
@EnvironmentObject var chatModel: ChatModel
var call: Call?
var dismiss: () -> Void
var body: some View {
VStack {
if let call = call {
switch call.localMedia {
case .video:
callInfoView(call, .leading)
.foregroundColor(.white)
.opacity(0.8)
.padding()
case .audio:
VStack {
ProfileImage(imageStr: call.contact.profile.image)
.scaledToFit()
.frame(width: 192, height: 192)
callInfoView(call, .center)
}
.foregroundColor(.white)
.opacity(0.8)
.padding()
.frame(maxHeight: .infinity)
}
Spacer()
ZStack(alignment: .bottom) {
VStack(alignment: .leading) {
if call.localMedia == .video {
callButton(call.videoEnabled ? "video.fill" : "video.slash", size: 48) {
chatModel.callCommand = .media(media: .video, enable: !call.videoEnabled)
}
.foregroundColor(.white)
.opacity(0.85)
}
callButton(call.audioEnabled ? "mic.fill" : "mic.slash", size: 48) {
chatModel.callCommand = .media(media: .audio, enable: !call.audioEnabled)
}
.foregroundColor(.white)
.opacity(0.85)
.frame(maxWidth: .infinity, alignment: .leading)
.padding(.top)
}
callButton("phone.down.fill", size: 60) { dismiss() }
.foregroundColor(.red)
}
.padding(.bottom, 60)
.padding(.horizontal, 48)
}
}
.frame(maxWidth: .infinity)
}
private func callInfoView(_ call: Call, _ alignment: Alignment) -> some View {
VStack {
Text(call.contact.chatViewName)
.lineLimit(1)
.font(.title)
.frame(maxWidth: .infinity, alignment: alignment)
let status = call.callState == .connected
? call.encrypted
? "end-to-end encrypted"
: "no end-to-end encryption"
: call.callState.text
Text(status)
.font(.subheadline)
.frame(maxWidth: .infinity, alignment: alignment)
}
}
private func callButton(_ imageName: String, size: CGFloat, perform: @escaping () -> Void) -> some View {
Button {
perform()
} label: {
Image(systemName: imageName)
.resizable()
.scaledToFit()
.frame(maxWidth: size, maxHeight: size)
}
}
}
struct ActiveCallOverlay_Previews: PreviewProvider {
static var previews: some View {
Group{
ActiveCallOverlay(call: Call(contact: Contact.sampleData, callState: .offerSent, localMedia: .video), dismiss: {})
.background(.black)
ActiveCallOverlay(call: Call(contact: Contact.sampleData, callState: .offerSent, localMedia: .audio), dismiss: {})
.background(.black)
}
}
}
-183
View File
@@ -1,183 +0,0 @@
//
// CallView.swift
// SimpleX (iOS)
//
// Created by Ian Davies on 29/04/2022.
// Copyright © 2022 SimpleX Chat. All rights reserved.
//
import SwiftUI
import WebKit
class WebRTCCoordinator: NSObject, WKNavigationDelegate, WKScriptMessageHandler {
var webView: WKWebView!
var corrId = 0
var pendingCommands: Dictionary<Int, CheckedContinuation<WCallResponse, Never>> = [:]
func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) {
webView.allowsBackForwardNavigationGestures = false
self.webView = webView
}
// receive message from WKWebView
func userContentController(
_ userContentController: WKUserContentController,
didReceive message: WKScriptMessage
) {
logger.debug("WebRTCCoordinator.userContentController")
if let data = (message.body as? String)?.data(using: .utf8),
let msg = try? jsonDecoder.decode(WVAPIMessage.self, from: data) {
if let corrId = msg.corrId, let cont = pendingCommands.removeValue(forKey: corrId) {
cont.resume(returning: msg.resp)
} else {
// TODO pass messages to call view via binding
// print(msg.resp)
}
} else {
logger.error("WebRTCCoordinator.userContentController: invalid message \(String(describing: message.body))")
}
}
func messageToWebview(msg: String) {
logger.debug("WebRTCCoordinator.messageToWebview")
self.webView.evaluateJavaScript("webkit.messageHandlers.logHandler.postMessage('\(msg)')")
}
func processCommand(command: WCallCommand) async -> WCallResponse {
await withCheckedContinuation { cont in
logger.debug("WebRTCCoordinator.processCommand")
let corrId_ = corrId
corrId = corrId + 1
pendingCommands[corrId_] = cont
let apiCmd = encodeJSON(WVAPICall(corrId: corrId_, command: command))
DispatchQueue.main.async {
logger.debug("WebRTCCoordinator.processCommand DispatchQueue.main.async")
let js = "processCommand(\(apiCmd))"
self.webView.evaluateJavaScript(js)
}
}
}
}
struct WebRTCView: UIViewRepresentable {
@Binding var coordinator: WebRTCCoordinator?
func makeCoordinator() -> WebRTCCoordinator {
WebRTCCoordinator()
}
func makeUIView(context: Context) -> WKWebView {
let _coordinator = makeCoordinator()
DispatchQueue.main.async {
coordinator = _coordinator
}
let userContentController = WKUserContentController()
let cfg = WKWebViewConfiguration()
cfg.userContentController = userContentController
cfg.mediaTypesRequiringUserActionForPlayback = []
cfg.allowsInlineMediaPlayback = true
// Enable us to capture calls to console.log in the xcode logs
let source = "console.log = (msg) => webkit.messageHandlers.logHandler.postMessage(msg)"
let script = WKUserScript(source: source, injectionTime: .atDocumentStart, forMainFrameOnly: false)
cfg.userContentController.addUserScript(script)
cfg.userContentController.add(_coordinator, name: "logHandler")
let _wkwebview = WKWebView(frame: .zero, configuration: cfg)
_wkwebview.navigationDelegate = _coordinator
guard let path: String = Bundle.main.path(forResource: "call", ofType: "html", inDirectory: "www") else {
logger.error("WebRTCView.makeUIView call.html not found")
return _wkwebview
}
let localHTMLUrl = URL(fileURLWithPath: path, isDirectory: false)
_wkwebview.loadFileURL(localHTMLUrl, allowingReadAccessTo: localHTMLUrl)
return _wkwebview
}
func updateUIView(_ webView: WKWebView, context: Context) {
logger.debug("WebRTCView.updateUIView")
}
}
struct CallView: View {
@State var coordinator: WebRTCCoordinator? = nil
@State var commandStr = ""
@FocusState private var keyboardVisible: Bool
var body: some View {
VStack(spacing: 30) {
WebRTCView(coordinator: $coordinator).frame(maxHeight: 260)
TextEditor(text: $commandStr)
.focused($keyboardVisible)
.disableAutocorrection(true)
.textInputAutocapitalization(.never)
.padding(.horizontal, 5)
.padding(.top, 2)
.frame(height: 112)
.overlay(
RoundedRectangle(cornerRadius: 10)
.strokeBorder(.secondary, lineWidth: 0.3, antialiased: true)
)
HStack(spacing: 20) {
Button("Copy") {
UIPasteboard.general.string = commandStr
}
Button("Paste") {
commandStr = UIPasteboard.general.string ?? ""
}
Button("Clear") {
commandStr = ""
}
Button("Send") {
do {
let command = try jsonDecoder.decode(WCallCommand.self, from: commandStr.data(using: .utf8)!)
if let c = coordinator {
Task {
let resp = await c.processCommand(command: command)
print(encodeJSON(resp))
}
}
} catch {
print(error)
}
}
}
HStack(spacing: 20) {
Button("Capabilities") {
}
Button("Start") {
if let c = coordinator {
Task {
let resp = await c.processCommand(command: .start(media: .video))
print(encodeJSON(resp))
}
}
}
Button("Accept") {
}
Button("Answer") {
}
Button("ICE") {
}
Button("End") {
}
}
}
}
}
struct CallView_Previews: PreviewProvider {
static var previews: some View {
CallView()
}
}
+169 -70
View File
@@ -7,23 +7,109 @@
//
import Foundation
import SwiftUI
class Call: Equatable {
static func == (lhs: Call, rhs: Call) -> Bool {
lhs.contact.apiId == rhs.contact.apiId
}
var contact: Contact
var callState: CallState
var localMedia: CallMediaType
var localCapabilities: CallCapabilities?
var peerMedia: CallMediaType?
var sharedKey: String?
var audioEnabled: Bool
var videoEnabled: Bool
init(
contact: Contact,
callState: CallState,
localMedia: CallMediaType,
localCapabilities: CallCapabilities? = nil,
peerMedia: CallMediaType? = nil,
sharedKey: String? = nil,
audioEnabled: Bool? = nil,
videoEnabled: Bool? = nil
) {
self.contact = contact
self.callState = callState
self.localMedia = localMedia
self.localCapabilities = localCapabilities
self.peerMedia = peerMedia
self.sharedKey = sharedKey
self.audioEnabled = audioEnabled ?? true
self.videoEnabled = videoEnabled ?? (localMedia == .video)
}
func copy(
contact: Contact? = nil,
callState: CallState? = nil,
localMedia: CallMediaType? = nil,
localCapabilities: CallCapabilities? = nil,
peerMedia: CallMediaType? = nil,
sharedKey: String? = nil,
audioEnabled: Bool? = nil,
videoEnabled: Bool? = nil
) -> Call {
Call (
contact: contact ?? self.contact,
callState: callState ?? self.callState,
localMedia: localMedia ?? self.localMedia,
localCapabilities: localCapabilities ?? self.localCapabilities,
peerMedia: peerMedia ?? self.peerMedia,
sharedKey: sharedKey ?? self.sharedKey,
audioEnabled: audioEnabled ?? self.audioEnabled,
videoEnabled: videoEnabled ?? self.videoEnabled
)
}
var encrypted: Bool {
(localCapabilities?.encryption ?? false) && sharedKey != nil
}
}
enum CallState {
case waitCapabilities
case invitationSent
case invitationReceived
case offerSent
case offerReceived
case negotiated
case connected
var text: LocalizedStringKey {
switch self {
case .waitCapabilities: return "starting..."
case .invitationSent: return "waiting for answer..."
case .invitationReceived: return "starting..."
case .offerSent: return "waiting for confirmation..."
case .offerReceived: return "received answer..."
case .negotiated: return "connecting..."
case .connected: return "connected"
}
}
}
struct WVAPICall: Encodable {
var corrId: Int
var corrId: Int? = nil
var command: WCallCommand
}
struct WVAPIMessage: Decodable {
struct WVAPIMessage: Equatable, Decodable {
var corrId: Int?
var resp: WCallResponse
var command: WCallCommand?
}
enum WCallCommand {
enum WCallCommand: Equatable, Encodable, Decodable {
case capabilities
case start(media: CallMediaType, aesKey: String? = nil)
case accept(offer: String, iceCandidates: [String], media: CallMediaType, aesKey: String? = nil)
case answer(answer: String, iceCandidates: [String])
case ice(iceCandidates: [String])
case media(media: CallMediaType, enable: Bool)
case end
enum CodingKeys: String, CodingKey {
@@ -33,10 +119,23 @@ enum WCallCommand {
case offer
case answer
case iceCandidates
case enable
}
var cmdType: String {
get {
switch self {
case .capabilities: return "capabilities"
case .start: return "start"
case .accept: return "accept"
case .answer: return "answer"
case .ice: return "ice"
case .media: return "media"
case .end: return "end"
}
}
}
}
extension WCallCommand: Encodable {
func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
switch self {
@@ -59,14 +158,15 @@ extension WCallCommand: Encodable {
case let .ice(iceCandidates):
try container.encode("ice", forKey: .type)
try container.encode(iceCandidates, forKey: .iceCandidates)
case let .media(media, enable):
try container.encode("media", forKey: .type)
try container.encode(media, forKey: .media)
try container.encode(enable, forKey: .enable)
case .end:
try container.encode("end", forKey: .type)
}
}
}
// This protocol is only needed for debugging
extension WCallCommand: Decodable {
init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
let type = try container.decode(String.self, forKey: CodingKeys.type)
@@ -90,6 +190,10 @@ extension WCallCommand: Decodable {
case "ice":
let iceCandidates = try container.decode([String].self, forKey: CodingKeys.iceCandidates)
self = .ice(iceCandidates: iceCandidates)
case "media":
let media = try container.decode(CallMediaType.self, forKey: CodingKeys.media)
let enable = try container.decode(Bool.self, forKey: CodingKeys.enable)
self = .media(media: media, enable: enable)
case "end":
self = .end
default:
@@ -99,11 +203,11 @@ extension WCallCommand: Decodable {
}
enum WCallResponse {
enum WCallResponse: Equatable, Decodable {
case capabilities(capabilities: CallCapabilities)
case offer(offer: String, iceCandidates: [String])
case offer(offer: String, iceCandidates: [String], capabilities: CallCapabilities)
// TODO remove accept, it is needed for debugging
case accept(offer: String, iceCandidates: [String], media: CallMediaType, aesKey: String? = nil)
// case accept(offer: String, iceCandidates: [String], media: CallMediaType, aesKey: String? = nil)
case answer(answer: String, iceCandidates: [String])
case ice(iceCandidates: [String])
case connection(state: ConnectionState)
@@ -124,9 +228,24 @@ enum WCallResponse {
case media
case aesKey
}
}
extension WCallResponse: Decodable {
var respType: String {
get {
switch self {
case .capabilities: return("capabilities")
case .offer: return("offer")
// case .accept: return("accept")
case .answer: return("answer (TODO remove)")
case .ice: return("ice")
case .connection: return("connection")
case .ended: return("ended")
case .ok: return("ok")
case .error: return("error")
case .invalid: return("invalid")
}
}
}
init(from decoder: Decoder) throws {
do {
let container = try decoder.container(keyedBy: CodingKeys.self)
@@ -138,14 +257,8 @@ extension WCallResponse: Decodable {
case "offer":
let offer = try container.decode(String.self, forKey: CodingKeys.offer)
let iceCandidates = try container.decode([String].self, forKey: CodingKeys.iceCandidates)
self = .offer(offer: offer, iceCandidates: iceCandidates)
// TODO remove accept
case "accept":
let offer = try container.decode(String.self, forKey: CodingKeys.offer)
let iceCandidates = try container.decode([String].self, forKey: CodingKeys.iceCandidates)
let media = try container.decode(CallMediaType.self, forKey: CodingKeys.media)
let aesKey = try? container.decode(String.self, forKey: CodingKeys.aesKey)
self = .accept(offer: offer, iceCandidates: iceCandidates, media: media, aesKey: aesKey)
let capabilities = try container.decode(CallCapabilities.self, forKey: CodingKeys.capabilities)
self = .offer(offer: offer, iceCandidates: iceCandidates, capabilities: capabilities)
case "answer":
let answer = try container.decode(String.self, forKey: CodingKeys.answer)
let iceCandidates = try container.decode([String].self, forKey: CodingKeys.iceCandidates)
@@ -172,56 +285,42 @@ extension WCallResponse: Decodable {
}
}
// This protocol is only needed for debugging
extension WCallResponse: Encodable {
func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
switch self {
case .capabilities:
try container.encode("capabilities", forKey: .type)
case let .offer(offer, iceCandidates):
try container.encode("offer", forKey: .type)
try container.encode(offer, forKey: .offer)
try container.encode(iceCandidates, forKey: .iceCandidates)
case let .accept(offer, iceCandidates, media, aesKey):
try container.encode("accept", forKey: .type)
try container.encode(offer, forKey: .offer)
try container.encode(iceCandidates, forKey: .iceCandidates)
try container.encode(media, forKey: .media)
try container.encode(aesKey, forKey: .aesKey)
case let .answer(answer, iceCandidates):
try container.encode("answer", forKey: .type)
try container.encode(answer, forKey: .answer)
try container.encode(iceCandidates, forKey: .iceCandidates)
case let .ice(iceCandidates):
try container.encode("ice", forKey: .type)
try container.encode(iceCandidates, forKey: .iceCandidates)
case let .connection(state):
try container.encode("connection", forKey: .type)
try container.encode(state, forKey: .state)
case .ended:
try container.encode("ended", forKey: .type)
case .ok:
try container.encode("ok", forKey: .type)
case let .error(message):
try container.encode("error", forKey: .type)
try container.encode(message, forKey: .message)
case let .invalid(type):
try container.encode(type, forKey: .type)
}
}
}
// This protocol is for debugging
//extension WCallResponse: Encodable {
// func encode(to encoder: Encoder) throws {
// var container = encoder.container(keyedBy: CodingKeys.self)
// switch self {
// case .capabilities:
// try container.encode("capabilities", forKey: .type)
// case let .offer(offer, iceCandidates, capabilities):
// try container.encode("offer", forKey: .type)
// try container.encode(offer, forKey: .offer)
// try container.encode(iceCandidates, forKey: .iceCandidates)
// try container.encode(capabilities, forKey: .capabilities)
// case let .answer(answer, iceCandidates):
// try container.encode("answer", forKey: .type)
// try container.encode(answer, forKey: .answer)
// try container.encode(iceCandidates, forKey: .iceCandidates)
// case let .ice(iceCandidates):
// try container.encode("ice", forKey: .type)
// try container.encode(iceCandidates, forKey: .iceCandidates)
// case let .connection(state):
// try container.encode("connection", forKey: .type)
// try container.encode(state, forKey: .state)
// case .ended:
// try container.encode("ended", forKey: .type)
// case .ok:
// try container.encode("ok", forKey: .type)
// case let .error(message):
// try container.encode("error", forKey: .type)
// try container.encode(message, forKey: .message)
// case let .invalid(type):
// try container.encode(type, forKey: .type)
// }
// }
//}
enum CallMediaType: String, Codable {
case video = "video"
case audio = "audio"
}
struct CallCapabilities: Codable {
var encryption: Bool
}
struct ConnectionState: Codable {
struct ConnectionState: Codable, Equatable {
var connectionState: String
var iceConnectionState: String
var iceGatheringState: String
+169
View File
@@ -0,0 +1,169 @@
//
// WebRTCView.swift
// SimpleX (iOS)
//
// Created by Ian Davies on 29/04/2022.
// Copyright © 2022 SimpleX Chat. All rights reserved.
//
import SwiftUI
import WebKit
class WebRTCCoordinator: NSObject, WKNavigationDelegate, WKScriptMessageHandler {
var webViewReady: Binding<Bool>
var webViewMsg: Binding<WVAPIMessage?>
private var webView: WKWebView?
internal init(webViewReady: Binding<Bool>, webViewMsg: Binding<WVAPIMessage?>) {
self.webViewReady = webViewReady
self.webViewMsg = webViewMsg
}
func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) {
webView.allowsBackForwardNavigationGestures = false
self.webView = webView
webViewReady.wrappedValue = true
}
// receive message from WKWebView
func userContentController(
_ userContentController: WKUserContentController,
didReceive message: WKScriptMessage
) {
logger.debug("WebRTCCoordinator.userContentController")
if let msgStr = message.body as? String,
let msg: WVAPIMessage = decodeJSON(msgStr) {
webViewMsg.wrappedValue = msg
} else {
logger.error("WebRTCCoordinator.userContentController: invalid message \(String(describing: message.body))")
}
}
func sendCommand(command: WCallCommand) {
if let webView = webView {
logger.debug("WebRTCCoordinator.sendCommand")
let apiCmd = encodeJSON(WVAPICall(command: command))
let js = "processCommand(\(apiCmd))"
webView.evaluateJavaScript(js)
}
}
}
struct WebRTCView: UIViewRepresentable {
@Binding var coordinator: WebRTCCoordinator?
@Binding var webViewReady: Bool
@Binding var webViewMsg: WVAPIMessage?
func makeCoordinator() -> WebRTCCoordinator {
WebRTCCoordinator(webViewReady: $webViewReady, webViewMsg: $webViewMsg)
}
func makeUIView(context: Context) -> WKWebView {
let wkCoordinator = makeCoordinator()
DispatchQueue.main.async { coordinator = wkCoordinator }
let wkController = WKUserContentController()
let cfg = WKWebViewConfiguration()
cfg.userContentController = wkController
cfg.mediaTypesRequiringUserActionForPlayback = []
cfg.allowsInlineMediaPlayback = true
let source = "sendMessageToNative = (msg) => webkit.messageHandlers.webrtc.postMessage(JSON.stringify(msg))"
let script = WKUserScript(source: source, injectionTime: .atDocumentEnd, forMainFrameOnly: false)
wkController.addUserScript(script)
wkController.add(wkCoordinator, name: "webrtc")
let wkWebView = WKWebView(frame: .zero, configuration: cfg)
wkWebView.navigationDelegate = wkCoordinator
guard let path: String = Bundle.main.path(forResource: "call", ofType: "html", inDirectory: "www") else {
logger.error("WebRTCView.makeUIView call.html not found")
return wkWebView
}
let localHTMLUrl = URL(fileURLWithPath: path, isDirectory: false)
wkWebView.loadFileURL(localHTMLUrl, allowingReadAccessTo: localHTMLUrl)
return wkWebView
}
func updateUIView(_ webView: WKWebView, context: Context) {
logger.debug("WebRTCView.updateUIView")
}
}
//struct CallViewDebug: View {
// @State var coordinator: WebRTCCoordinator? = nil
// @State var commandStr = ""
// @State private var webViewMsg: WCallResponse? = nil
// @FocusState private var keyboardVisible: Bool
//
// var body: some View {
// VStack(spacing: 30) {
// WebRTCView(coordinator: $coordinator, webViewMsg: $webViewMsg).frame(maxHeight: 260)
// .onChange(of: webViewMsg) { _ in
// if let resp = webViewMsg {
// commandStr = encodeJSON(resp)
// }
// }
// TextEditor(text: $commandStr)
// .focused($keyboardVisible)
// .disableAutocorrection(true)
// .textInputAutocapitalization(.never)
// .padding(.horizontal, 5)
// .padding(.top, 2)
// .frame(height: 112)
// .overlay(
// RoundedRectangle(cornerRadius: 10)
// .strokeBorder(.secondary, lineWidth: 0.3, antialiased: true)
// )
// HStack(spacing: 20) {
// Button("Copy") {
// UIPasteboard.general.string = commandStr
// }
// Button("Paste") {
// commandStr = UIPasteboard.general.string ?? ""
// }
// Button("Clear") {
// commandStr = ""
// }
// Button("Send") {
// do {
// if let c = coordinator,
// let command: WCallCommand = decodeJSON(commandStr) {
// c.sendCommand(command: command)
// }
// } catch {
// print(error)
// }
// }
// }
// HStack(spacing: 20) {
// Button("Capabilities") {
//
// }
// Button("Start") {
// if let c = coordinator {
// c.sendCommand(command: .start(media: .video))
// }
// }
// Button("Accept") {
//
// }
// Button("Answer") {
//
// }
// Button("ICE") {
//
// }
// Button("End") {
//
// }
// }
// }
// }
//}
//
//struct CallViewDebug_Previews: PreviewProvider {
// static var previews: some View {
// CallViewDebug()
// }
//}
@@ -22,6 +22,8 @@ struct ChatItemView: View {
}
} else if chatItem.isDeletedContent() {
DeletedItemView(chatItem: chatItem, showMember: showMember)
} else if chatItem.isCall() {
FramedItemView(chatItem: chatItem, showMember: showMember, maxWidth: maxWidth)
}
}
}
+29 -4
View File
@@ -14,8 +14,9 @@ struct ChatView: View {
@EnvironmentObject var chatModel: ChatModel
@Environment(\.colorScheme) var colorScheme
@ObservedObject var chat: Chat
@State var composeState = ComposeState()
@State var deletingItem: ChatItem? = nil
@Binding var showCallView: Bool
@State private var composeState = ComposeState()
@State private var deletingItem: ChatItem? = nil
@FocusState private var keyboardVisible: Bool
@State private var showChatInfo = false
@State private var showDeleteMessage = false
@@ -105,10 +106,32 @@ struct ChatView: View {
ChatInfoView(chat: chat, showChatInfo: $showChatInfo)
}
}
ToolbarItem(placement: .navigationBarTrailing) {
if case let .direct(contact) = cInfo {
HStack {
callButton(contact, .audio, imageName: "phone")
callButton(contact, .video, imageName: "video")
}
}
}
}
.navigationBarBackButtonHidden(true)
}
private func callButton(_ contact: Contact, _ media: CallMediaType, imageName: String) -> some View {
Button {
chatModel.activeCall = Call(
contact: contact,
callState: .waitCapabilities,
localMedia: media
)
showCallView = true
chatModel.callCommand = .capabilities
} label: {
Image(systemName: imageName)
}
}
private func chatItemWithMenu(_ ci: ChatItem, _ maxWidth: CGFloat, showMember: Bool = false) -> some View {
let alignment: Alignment = ci.chatDir.sent ? .trailing : .leading
return ChatItemView(chatItem: ci, showMember: showMember, maxWidth: maxWidth)
@@ -166,7 +189,8 @@ struct ChatView: View {
}
if let di = deletingItem {
if di.meta.editable {
Button("Delete for everyone",role: .destructive) { deleteMessage(.cidmBroadcast)
Button("Delete for everyone",role: .destructive) {
deleteMessage(.cidmBroadcast)
}
}
}
@@ -236,6 +260,7 @@ struct ChatView: View {
struct ChatView_Previews: PreviewProvider {
static var previews: some View {
@State var showCallView = false
let chatModel = ChatModel()
chatModel.chatId = "@1"
chatModel.chatItems = [
@@ -249,7 +274,7 @@ struct ChatView_Previews: PreviewProvider {
ChatItem.getSample(8, .directSnd, .now, "👍👍👍👍"),
ChatItem.getSample(9, .directSnd, .now, "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.")
]
return ChatView(chat: Chat(chatInfo: ChatInfo.sampleData.direct, chatItems: []))
return ChatView(chat: Chat(chatInfo: ChatInfo.sampleData.direct, chatItems: []), showCallView: $showCallView)
.environmentObject(chatModel)
}
}
@@ -11,6 +11,7 @@ import SwiftUI
struct ChatListNavLink: View {
@EnvironmentObject var chatModel: ChatModel
@State var chat: Chat
@Binding var showCallView: Bool
@State private var showContactRequestDialog = false
var body: some View {
@@ -27,7 +28,7 @@ struct ChatListNavLink: View {
}
private func chatView() -> some View {
ChatView(chat: chat)
ChatView(chat: chat, showCallView: $showCallView)
.onAppear {
do {
let cInfo = chat.chatInfo
@@ -252,19 +253,20 @@ struct ChatListNavLink: View {
struct ChatListNavLink_Previews: PreviewProvider {
static var previews: some View {
@State var chatId: String? = "@1"
@State var showCallView = false
return Group {
ChatListNavLink(chat: Chat(
chatInfo: ChatInfo.sampleData.direct,
chatItems: [ChatItem.getSample(1, .directSnd, .now, "hello")]
))
), showCallView: $showCallView)
ChatListNavLink(chat: Chat(
chatInfo: ChatInfo.sampleData.direct,
chatItems: [ChatItem.getSample(1, .directSnd, .now, "hello")]
))
), showCallView: $showCallView)
ChatListNavLink(chat: Chat(
chatInfo: ChatInfo.sampleData.contactRequest,
chatItems: []
))
), showCallView: $showCallView)
}
.previewLayout(.fixed(width: 360, height: 80))
}
@@ -13,6 +13,7 @@ struct ChatListView: View {
// not really used in this view
@State private var showSettings = false
@State private var searchText = ""
@State private var showCallView = false
@AppStorage("pendingConnections") private var pendingConnections = true
var user: User
@@ -24,7 +25,7 @@ struct ChatListView: View {
ChatHelp(showSettings: $showSettings)
} else {
ForEach(filteredChats()) { chat in
ChatListNavLink(chat: chat)
ChatListNavLink(chat: chat, showCallView: $showCallView)
.padding(.trailing, -16)
}
}
@@ -53,6 +54,29 @@ struct ChatListView: View {
NewChatButton()
}
}
.fullScreenCover(isPresented: $showCallView) {
ActiveCallView(showCallView: $showCallView)
}
.onChange(of: showCallView) { _ in
if (showCallView) { return }
if let call = chatModel.activeCall {
Task {
do {
try await apiEndCall(call.contact)
} catch {
logger.error("ChatListView apiEndCall error: \(error.localizedDescription)")
}
}
}
chatModel.callCommand = .end
}
.onChange(of: chatModel.activeCallInvitation) { _ in
if let contactRef = chatModel.activeCallInvitation,
case let .direct(contact) = chatModel.getChat(contactRef.id)?.chatInfo,
let invitation = chatModel.callInvitations.removeValue(forKey: contactRef.id) {
answerCallAlert(contact, invitation)
}
}
}
.navigationViewStyle(.stack)
@@ -99,6 +123,29 @@ struct ChatListView: View {
return Alert(title: Text("Error: URL is invalid"))
}
}
private func answerCallAlert(_ contact: Contact, _ invitation: CallInvitation) {
AlertManager.shared.showAlert(Alert(
title: Text("Incoming call"),
primaryButton: .default(Text("Answer")) {
if chatModel.activeCallInvitation == nil {
DispatchQueue.main.async {
AlertManager.shared.showAlertMsg(title: "Call already ended!")
}
} else {
chatModel.activeCallInvitation = nil
chatModel.activeCall = Call(
contact: contact,
callState: .invitationReceived,
localMedia: invitation.peerMedia
)
showCallView = true
chatModel.callCommand = .start(media: invitation.peerMedia, aesKey: invitation.sharedKey)
}
},
secondaryButton: .cancel()
))
}
}
struct ChatListView_Previews: PreviewProvider {
@@ -143,12 +143,12 @@ struct SettingsView: View {
notificationsToggle(token)
}
}
NavigationLink {
CallView()
.frame(maxHeight: .infinity, alignment: .top)
} label: {
// NavigationLink {
// CallViewDebug()
// .frame(maxHeight: .infinity, alignment: .top)
// } label: {
Text("v\(appVersion ?? "?") (\(appBuild ?? "?"))")
}
// }
}
}
.navigationTitle("Your settings")
+14 -4
View File
@@ -7,7 +7,7 @@
objects = {
/* Begin PBXBuildFile section */
3C714777281C081000CB4D4B /* CallView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3C714776281C081000CB4D4B /* CallView.swift */; };
3C714777281C081000CB4D4B /* WebRTCView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3C714776281C081000CB4D4B /* WebRTCView.swift */; };
3C71477A281C0F6800CB4D4B /* www in Resources */ = {isa = PBXBuildFile; fileRef = 3C714779281C0F6800CB4D4B /* www */; };
3C8C548928133C84000A3EC7 /* PasteToConnectView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3C8C548828133C84000A3EC7 /* PasteToConnectView.swift */; };
3CDBCF4227FAE51000354CDD /* ComposeLinkView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3CDBCF4127FAE51000354CDD /* ComposeLinkView.swift */; };
@@ -32,6 +32,9 @@
5C545C8F2812A834007A6B96 /* libffi.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 5C545C8A2812A833007A6B96 /* libffi.a */; };
5C545C902812A834007A6B96 /* libHSsimplex-chat-1.6.0-IWA13KO27d7KsfAsFQIqFK-ghc8.10.7.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 5C545C8B2812A833007A6B96 /* libHSsimplex-chat-1.6.0-IWA13KO27d7KsfAsFQIqFK-ghc8.10.7.a */; };
5C577F7D27C83AA10006112D /* MarkdownHelp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C577F7C27C83AA10006112D /* MarkdownHelp.swift */; };
5C5E5D3B2824468B00B0488A /* ActiveCallView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C5E5D3A2824468B00B0488A /* ActiveCallView.swift */; };
5C5E5D3D282447AB00B0488A /* CallTypes.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C5E5D3C282447AB00B0488A /* CallTypes.swift */; };
5C5E5D3E282447BF00B0488A /* CallTypes.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C5E5D3C282447AB00B0488A /* CallTypes.swift */; };
5C5F2B6D27EBC3FE006A9D5F /* ImagePicker.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C5F2B6C27EBC3FE006A9D5F /* ImagePicker.swift */; };
5C5F2B7027EBC704006A9D5F /* ProfileImage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C5F2B6F27EBC704006A9D5F /* ProfileImage.swift */; };
5C6AD81327A834E300348BD7 /* NewChatButton.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C6AD81227A834E300348BD7 /* NewChatButton.swift */; };
@@ -134,7 +137,7 @@
/* End PBXCopyFilesBuildPhase section */
/* Begin PBXFileReference section */
3C714776281C081000CB4D4B /* CallView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CallView.swift; sourceTree = "<group>"; };
3C714776281C081000CB4D4B /* WebRTCView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebRTCView.swift; sourceTree = "<group>"; };
3C714779281C0F6800CB4D4B /* www */ = {isa = PBXFileReference; lastKnownFileType = folder; name = www; path = ../android/app/src/main/assets/www; sourceTree = "<group>"; };
3C8C548828133C84000A3EC7 /* PasteToConnectView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PasteToConnectView.swift; sourceTree = "<group>"; };
3CDBCF4127FAE51000354CDD /* ComposeLinkView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ComposeLinkView.swift; sourceTree = "<group>"; };
@@ -161,6 +164,8 @@
5C545C8A2812A833007A6B96 /* libffi.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libffi.a; sourceTree = "<group>"; };
5C545C8B2812A833007A6B96 /* libHSsimplex-chat-1.6.0-IWA13KO27d7KsfAsFQIqFK-ghc8.10.7.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = "libHSsimplex-chat-1.6.0-IWA13KO27d7KsfAsFQIqFK-ghc8.10.7.a"; sourceTree = "<group>"; };
5C577F7C27C83AA10006112D /* MarkdownHelp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MarkdownHelp.swift; sourceTree = "<group>"; };
5C5E5D3A2824468B00B0488A /* ActiveCallView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ActiveCallView.swift; sourceTree = "<group>"; };
5C5E5D3C282447AB00B0488A /* CallTypes.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CallTypes.swift; sourceTree = "<group>"; };
5C5F2B6C27EBC3FE006A9D5F /* ImagePicker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImagePicker.swift; sourceTree = "<group>"; };
5C5F2B6F27EBC704006A9D5F /* ProfileImage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileImage.swift; sourceTree = "<group>"; };
5C6AD81227A834E300348BD7 /* NewChatButton.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NewChatButton.swift; sourceTree = "<group>"; };
@@ -268,8 +273,9 @@
3C714775281C080100CB4D4B /* Call */ = {
isa = PBXGroup;
children = (
3C714776281C081000CB4D4B /* CallView.swift */,
3C714776281C081000CB4D4B /* WebRTCView.swift */,
5C9D13A2282187BB00AB8B43 /* WebRTC.swift */,
5C5E5D3A2824468B00B0488A /* ActiveCallView.swift */,
);
path = Call;
sourceTree = "<group>";
@@ -466,6 +472,7 @@
5CDCAD5228186F9500503DA2 /* GroupDefaults.swift */,
5CDCAD7228188CFF00503DA2 /* ChatTypes.swift */,
5CDCAD7428188D2900503DA2 /* APITypes.swift */,
5C5E5D3C282447AB00B0488A /* CallTypes.swift */,
5C9FD96A27A56D4D0075386C /* JSON.swift */,
5CDCAD7D2818941F00503DA2 /* API.swift */,
5CDCAD80281A7E2700503DA2 /* Notifications.swift */,
@@ -691,10 +698,12 @@
5C764E89279CBCB3000C6508 /* ChatModel.swift in Sources */,
5C971E1D27AEBEF600C8A3CE /* ChatInfoView.swift in Sources */,
5CC1C99527A6CF7F000D9FF6 /* ShareSheet.swift in Sources */,
5C5E5D3B2824468B00B0488A /* ActiveCallView.swift in Sources */,
5C2E260727A2941F00F70299 /* SimpleXAPI.swift in Sources */,
5CB924D427A853F100ACCCDD /* SettingsButton.swift in Sources */,
3C714777281C081000CB4D4B /* CallView.swift in Sources */,
3C714777281C081000CB4D4B /* WebRTCView.swift in Sources */,
5C5F2B7027EBC704006A9D5F /* ProfileImage.swift in Sources */,
5C5E5D3D282447AB00B0488A /* CallTypes.swift in Sources */,
64AA1C6C27F3537400AC7277 /* DeletedItemView.swift in Sources */,
5CE4407227ADB1D0007B033A /* Emoji.swift in Sources */,
5C1A4C1E27A715B700EAD5AD /* ChatItemView.swift in Sources */,
@@ -716,6 +725,7 @@
buildActionMask = 2147483647;
files = (
5CDCAD7928188FD600503DA2 /* APITypes.swift in Sources */,
5C5E5D3E282447BF00B0488A /* CallTypes.swift in Sources */,
5CDCAD5828187C7500503DA2 /* dummy.m in Sources */,
5CDCAD482818589900503DA2 /* NotificationService.swift in Sources */,
5CDCAD82281A7E2700503DA2 /* Notifications.swift in Sources */,
+2 -2
View File
@@ -4,8 +4,8 @@
<link href="./style.css" rel="stylesheet" />
</head>
<body>
<video id="remote-video-stream" autoplay></video>
<video id="local-video-stream" muted autoplay></video>
<video id="remote-video-stream" autoplay playsinline></video>
<video id="local-video-stream" muted autoplay playsinline></video>
</body>
<footer>
<script src="./call.js"></script>
+59 -40
View File
@@ -1,24 +1,22 @@
// Inspired by
// https://github.com/webrtc/samples/blob/gh-pages/src/content/insertable-streams/endtoend-encryption
// type WCallMessage = WCallCommand | WCallResponse
interface WebViewAPICall {
corrId: number
interface WVAPICall {
corrId?: number
command: WCallCommand
}
// TODO remove WCallCommand from resp type
interface WebViewMessage {
interface WVApiMessage {
corrId?: number
resp: WCallResponse | WCallCommand
resp: WCallResponse
command?: WCallCommand
}
type WCallCommand = WCCapabilities | WCStartCall | WCAcceptOffer | WCallAnswer | WCallIceCandidates | WCEndCall
type WCallCommand = WCCapabilities | WCStartCall | WCAcceptOffer | WCallAnswer | WCallIceCandidates | WCEnableMedia | WCEndCall
type WCallResponse = WRCapabilities | WCallOffer | WCallAnswer | WCallIceCandidates | WRConnection | WRCallEnded | WROk | WRError
type WCallCommandTag = "capabilities" | "start" | "accept" | "answer" | "ice" | "end"
type WCallCommandTag = "capabilities" | "start" | "accept" | "answer" | "ice" | "media" | "end"
type WCallResponseTag = "capabilities" | "offer" | "answer" | "ice" | "connection" | "ended" | "ok" | "error"
@@ -61,6 +59,7 @@ interface WCallOffer extends IWCallResponse {
type: "offer"
offer: string // JSON string for RTCSessionDescriptionInit
iceCandidates: string[] // JSON strings for RTCIceCandidateInit
capabilities: CallCapabilities
}
interface WCallAnswer extends IWCallCommand, IWCallResponse {
@@ -74,6 +73,12 @@ interface WCallIceCandidates extends IWCallCommand, IWCallResponse {
iceCandidates: string[] // JSON strings for RTCIceCandidateInit
}
interface WCEnableMedia extends IWCallCommand {
type: "media"
media: CallMediaType
enable: boolean
}
interface WRCapabilities extends IWCallResponse {
type: "capabilities"
capabilities: CallCapabilities
@@ -125,7 +130,7 @@ const keyAlgorithm: AesKeyAlgorithm = {
const keyUsages: KeyUsage[] = ["encrypt", "decrypt"]
let pc: RTCPeerConnection | undefined
let activeCall: Call | undefined
const IV_LENGTH = 12
@@ -138,6 +143,8 @@ const initialPlainTextRequired = {
interface Call {
connection: RTCPeerConnection
iceCandidates: Promise<string[]> // JSON strings for RTCIceCandidate
localMedia: CallMediaType
localStream: MediaStream
}
interface CallConfig {
@@ -217,7 +224,7 @@ async function initializeCall(config: CallConfig, mediaType: CallMediaType, aesK
}
})
return {connection: conn, iceCandidates}
return {connection: conn, iceCandidates, localMedia: mediaType, localStream}
function connectionStateChange() {
sendMessageToNative({
@@ -235,20 +242,18 @@ async function initializeCall(config: CallConfig, mediaType: CallMediaType, aesK
conn.removeEventListener("connectionstatechange", connectionStateChange)
sendMessageToNative({resp: {type: "ended"}})
conn.close()
pc = undefined
activeCall = undefined
resetVideoElements()
}
}
}
function sendMessageToNative(msg: WebViewMessage) {
console.log(JSON.stringify(msg))
}
var sendMessageToNative = (msg: WVApiMessage) => console.log(JSON.stringify(msg))
// TODO remove WCallCommand from result type
async function processCommand(body: WebViewAPICall): Promise<WebViewMessage> {
const {command, corrId} = body
let resp: WCallResponse | WCallCommand
async function processCommand(body: WVAPICall): Promise<WVApiMessage> {
const {corrId, command} = body
const pc = activeCall?.connection
let resp: WCallResponse
try {
switch (command.type) {
case "capabilities":
@@ -257,39 +262,47 @@ async function processCommand(body: WebViewAPICall): Promise<WebViewMessage> {
break
case "start":
console.log("starting call")
if (pc) {
if (activeCall) {
resp = {type: "error", message: "start: call already started"}
} else if (!supportsInsertableStreams() && command.aesKey) {
resp = {type: "error", message: "start: encryption is not supported"}
} else {
const encryption = supportsInsertableStreams()
const {media, aesKey} = command
const call = await initializeCall(defaultCallConfig(!!aesKey), media, aesKey)
const {connection, iceCandidates} = call
pc = connection
activeCall = await initializeCall(defaultCallConfig(encryption && !!aesKey), media, encryption ? aesKey : undefined)
const pc = activeCall.connection
const offer = await pc.createOffer()
await pc.setLocalDescription(offer)
// for debugging, returning the command for callee to use
resp = {type: "accept", offer: JSON.stringify(offer), iceCandidates: await iceCandidates, media, aesKey}
// resp = {type: "offer", offer, iceCandidates: await iceCandidates}
// resp = {type: "accept", offer: JSON.stringify(offer), iceCandidates: await iceCandidates, media, aesKey}
resp = {
type: "offer",
offer: JSON.stringify(offer),
iceCandidates: await activeCall.iceCandidates,
capabilities: {encryption},
}
}
break
case "accept":
if (pc) {
if (activeCall) {
resp = {type: "error", message: "accept: call already started"}
} else if (!supportsInsertableStreams() && command.aesKey) {
resp = {type: "error", message: "accept: encryption is not supported"}
} else {
const offer = JSON.parse(command.offer)
const remoteIceCandidates = command.iceCandidates.map((c) => JSON.parse(c))
const call = await initializeCall(defaultCallConfig(!!command.aesKey), command.media, command.aesKey)
const {connection, iceCandidates} = call
pc = connection
activeCall = await initializeCall(defaultCallConfig(!!command.aesKey), command.media, command.aesKey)
const pc = activeCall.connection
await pc.setRemoteDescription(new RTCSessionDescription(offer))
const answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
addIceCandidates(pc, remoteIceCandidates)
// same as command for caller to use
resp = {type: "answer", answer: JSON.stringify(answer), iceCandidates: await iceCandidates}
resp = {
type: "answer",
answer: JSON.stringify(answer),
iceCandidates: await activeCall.iceCandidates,
}
}
break
case "answer":
@@ -316,10 +329,20 @@ async function processCommand(body: WebViewAPICall): Promise<WebViewMessage> {
resp = {type: "error", message: "ice: call not started"}
}
break
case "media":
if (!activeCall) {
resp = {type: "error", message: "media: call not started"}
} else if (activeCall.localMedia == CallMediaType.Audio && command.media == CallMediaType.Video) {
resp = {type: "error", message: "media: no video"}
} else {
enableMedia(activeCall.localStream, command.media, command.enable)
resp = {type: "ok"}
}
break
case "end":
if (pc) {
pc.close()
pc = undefined
activeCall = undefined
resetVideoElements()
resp = {type: "ok"}
} else {
@@ -333,7 +356,7 @@ async function processCommand(body: WebViewAPICall): Promise<WebViewMessage> {
} catch (e) {
resp = {type: "error", message: (e as Error).message}
}
const apiResp = {resp, corrId}
const apiResp = {corrId, resp, command}
sendMessageToNative(apiResp)
return apiResp
}
@@ -460,14 +483,10 @@ function getVideoElements(): VideoElements | undefined {
// }
// }
// what does it do?
// function toggleVideo(b) {
// if (b == "true") {
// localStream.getVideoTracks()[0].enabled = true
// } else {
// localStream.getVideoTracks()[0].enabled = false
// }
// }
function enableMedia(s: MediaStream, media: CallMediaType, enable: boolean) {
const tracks = media == CallMediaType.Video ? s.getVideoTracks() : s.getAudioTracks()
for (const t of tracks) t.enabled = enable
}
/* Stream Transforms */
function setupPeerTransform(
@@ -5,6 +5,7 @@ html,
body {
padding: 0;
margin: 0;
background-color: black;
}
#remote-video-stream {
position: absolute;
+14 -7
View File
@@ -453,8 +453,13 @@ processChatCommand = \case
APISendCallExtraInfo contactId rtcExtraInfo ->
-- any call party
withCurrentCall contactId $ \_ ct call@Call {callId, callState} -> case callState of
CallOfferSent {localCallType, peerCallType, localCallSession, sharedKey} -> do
-- TODO update the list of ice servers in localCallSession
_ <- sendDirectContactMessage ct (XCallExtra callId CallExtraInfo {rtcExtraInfo})
let callState' = CallOfferSent {localCallType, peerCallType, localCallSession, sharedKey}
pure $ Just call {callState = callState'}
CallNegotiated {localCallType, peerCallType, localCallSession, peerCallSession, sharedKey} -> do
-- TODO update the list of ice servers
-- TODO update the list of ice servers in localCallSession
_ <- sendDirectContactMessage ct (XCallExtra callId CallExtraInfo {rtcExtraInfo})
let callState' = CallNegotiated {localCallType, peerCallType, localCallSession, peerCallSession, sharedKey}
pure $ Just call {callState = callState'}
@@ -1630,8 +1635,13 @@ processAgentMessage (Just user@User {userId, profile}) agentConnId agentMessage
xCallExtra ct callId CallExtraInfo {rtcExtraInfo} msg msgMeta = do
msgCurrentCall ct callId "x.call.extra" msg msgMeta $
\call -> case callState call of
CallOfferReceived {localCallType, peerCallType, peerCallSession, sharedKey} -> do
-- TODO update the list of ice servers in peerCallSession
let callState' = CallOfferReceived {localCallType, peerCallType, peerCallSession, sharedKey}
toView $ CRCallExtraInfo ct rtcExtraInfo
pure (Just call {callState = callState'}, Nothing)
CallNegotiated {localCallType, peerCallType, localCallSession, peerCallSession, sharedKey} -> do
-- TODO update the list of ice servers
-- TODO update the list of ice servers in peerCallSession
let callState' = CallNegotiated {localCallType, peerCallType, localCallSession, peerCallSession, sharedKey}
toView $ CRCallExtraInfo ct rtcExtraInfo
pure (Just call {callState = callState'}, Nothing)
@@ -1641,13 +1651,10 @@ processAgentMessage (Just user@User {userId, profile}) agentConnId agentMessage
-- to any call party
xCallEnd :: Contact -> CallId -> RcvMessage -> MsgMeta -> m ()
xCallEnd ct@Contact {contactId} callId msg msgMeta =
xCallEnd ct callId msg msgMeta =
msgCurrentCall ct callId "x.call.end" msg msgMeta $ \Call {chatItemId} -> do
toView $ CRCallEnded ct
CChatItem msgDir _ <- withStore $ \st -> getDirectChatItem st userId contactId chatItemId
pure $ case msgDir of
SMDSnd -> (Nothing, Just . ACIContent SMDSnd $ CISndCall CISCallEnded 0)
SMDRcv -> (Nothing, Just . ACIContent SMDRcv $ CIRcvCall CISCallEnded 0)
(Nothing,) <$> callStatusItemContent userId ct chatItemId WCSDisconnected
msgCurrentCall :: Contact -> CallId -> Text -> RcvMessage -> MsgMeta -> (Call -> m (Maybe Call, Maybe ACIContent)) -> m ()
msgCurrentCall ct@Contact {contactId = ctId'} callId' eventName RcvMessage {msgId} msgMeta action = do