mirror of
https://github.com/simplex-chat/simplex-chat.git
synced 2026-10-22 10:12:25 +00:00
ui: reconnect call (#7711)
* Update Call.hs * Update Internal.hs * add reconnecting string * Update WebRTC.kt * desktop: show reconnecting state, keep call duration * android keep connection time correct wiith adding reconnecting * Update CallTypes.swift add reconnecting status on IOS * Update WebRTC.swift add reconnecting * add reconnecting and make sure call duration is correct * Update call.ts * core: regenerate call.js from call.ts * Update Call.hs revert call status change minimization * Update Internal.hs rever to minimize changes * Update CallTypes.swift minimize changes * Update WebRTC.kt minimize changes * Update CallView.desktop.kt desktop shows reconnecting without sending to core * Update CallView.android.kt show reconnecting without sending it to the core * Update ActiveCallView.swift show reconnecting to ui without sending to core * Update CallView.desktop.kt add reconnecting sound on reconnect * Update CallView.android.kt add reconnecting sound * Update ActiveCallView.swift play reconnecting sound when reconnecting * Update RecAndPlay.desktop.kt * rebuild * refactor * build * android, desktop: restart ICE to reconnect calls * ios: restart ICE to reconnect calls * webrtc: add call reconnection e2e tests * android, desktop: fix lost call commands --------- Co-authored-by: GuavaGoat <320862384+GuavaGoat@users.noreply.github.com> Co-authored-by: Evgeny Poberezkin <evgeny@poberezkin.com> Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com>
This commit is contained in:
27 files changed
+1247
-136
No files matched your search
@@ -1850,8 +1850,8 @@ func apiSendCallAnswer(_ contact: Contact, _ rtcSession: String, _ rtcIceCandida
|
||||
try await sendCommandOkResp(.apiSendCallAnswer(contact: contact, answer: answer))
|
||||
}
|
||||
|
||||
func apiSendCallExtraInfo(_ contact: Contact, _ rtcIceCandidates: String) async throws {
|
||||
let extraInfo = WebRTCExtraInfo(rtcIceCandidates: rtcIceCandidates)
|
||||
func apiSendCallExtraInfo(_ contact: Contact, _ rtcSession: String?, _ rtcIceCandidates: String) async throws {
|
||||
let extraInfo = WebRTCExtraInfo(rtcSession: rtcSession, rtcIceCandidates: rtcIceCandidates)
|
||||
try await sendCommandOkResp(.apiSendCallExtraInfo(contact: contact, extraInfo: extraInfo))
|
||||
}
|
||||
|
||||
@@ -2987,7 +2987,7 @@ func processReceivedMsg(_ res: ChatEvent) async {
|
||||
}
|
||||
case let .callExtraInfo(_, contact, extraInfo):
|
||||
await withCall(contact) { _ in
|
||||
await m.callCommand.processCommand(.ice(iceCandidates: extraInfo.rtcIceCandidates))
|
||||
await m.callCommand.processCommand(.ice(iceCandidates: extraInfo.rtcIceCandidates, session: extraInfo.rtcSession))
|
||||
}
|
||||
case let .callEnded(_, contact):
|
||||
if let invitation = await MainActor.run(body: { m.callInvitations.removeValue(forKey: contact.id) }) {
|
||||
|
||||
@@ -161,10 +161,10 @@ struct ActiveCallView: View {
|
||||
CallSoundsPlayer.shared.stop()
|
||||
}
|
||||
}
|
||||
case let .ice(iceCandidates):
|
||||
case let .ice(iceCandidates, session):
|
||||
Task {
|
||||
do {
|
||||
try await apiSendCallExtraInfo(call.contact, iceCandidates)
|
||||
try await apiSendCallExtraInfo(call.contact, session, iceCandidates)
|
||||
} catch {
|
||||
logger.error("apiSendCallExtraInfo \(responseError(error))")
|
||||
}
|
||||
@@ -172,15 +172,24 @@ struct ActiveCallView: View {
|
||||
case let .connection(state):
|
||||
if let callStatus = WebRTCCallStatus.init(rawValue: state.connectionState),
|
||||
case .connected = callStatus {
|
||||
call.direction == .outgoing
|
||||
? CallController.shared.reportOutgoingCall(call: call, connectedAt: nil)
|
||||
: CallController.shared.reportIncomingCall(call: call, connectedAt: nil)
|
||||
// connectedAt is kept on reconnection, so that call duration is not reset
|
||||
if call.connectedAt == nil {
|
||||
call.direction == .outgoing
|
||||
? CallController.shared.reportOutgoingCall(call: call, connectedAt: nil)
|
||||
: CallController.shared.reportIncomingCall(call: call, connectedAt: nil)
|
||||
call.connectedAt = .now
|
||||
}
|
||||
call.callState = .connected
|
||||
call.connectedAt = .now
|
||||
CallSoundsPlayer.shared.stop()
|
||||
if !wasConnected {
|
||||
CallSoundsPlayer.shared.vibrate(long: false)
|
||||
wasConnected = true
|
||||
}
|
||||
} else if state.connectionState == "reconnecting" {
|
||||
if call.callState != .reconnecting {
|
||||
CallSoundsPlayer.shared.startConnectingCallSound()
|
||||
}
|
||||
call.callState = .reconnecting
|
||||
}
|
||||
if state.connectionState == "closed" {
|
||||
closeCallView(client)
|
||||
@@ -190,17 +199,22 @@ struct ActiveCallView: View {
|
||||
m.activeCall = nil
|
||||
m.activeCallViewIsCollapsed = false
|
||||
}
|
||||
Task {
|
||||
do {
|
||||
try await apiCallStatus(call.contact, state.connectionState)
|
||||
} catch {
|
||||
logger.error("apiCallStatus \(responseError(error))")
|
||||
// "reconnecting" is not a core call status
|
||||
if state.connectionState != "reconnecting" {
|
||||
Task {
|
||||
do {
|
||||
try await apiCallStatus(call.contact, state.connectionState)
|
||||
} catch {
|
||||
logger.error("apiCallStatus \(responseError(error))")
|
||||
}
|
||||
}
|
||||
}
|
||||
case let .connected(connectionInfo):
|
||||
call.callState = .connected
|
||||
call.connectionInfo = connectionInfo
|
||||
call.connectedAt = .now
|
||||
if call.connectedAt == nil {
|
||||
call.connectedAt = .now
|
||||
}
|
||||
if !wasConnected {
|
||||
CallSoundsPlayer.shared.vibrate(long: false)
|
||||
wasConnected = true
|
||||
|
||||
@@ -77,6 +77,7 @@ enum CallState {
|
||||
case answerReceived // incoming - webrtc answer received via API
|
||||
case negotiated // outgoing - webrtc offer processed and answer sent, incoming - webrtc answer processed
|
||||
case connected
|
||||
case reconnecting
|
||||
case ended
|
||||
|
||||
var text: LocalizedStringKey {
|
||||
@@ -89,6 +90,7 @@ enum CallState {
|
||||
case .answerReceived: return "received confirmation…"
|
||||
case .negotiated: return "connecting…"
|
||||
case .connected: return "connected"
|
||||
case .reconnecting: return "reconnecting…"
|
||||
case .ended: return "ended"
|
||||
}
|
||||
}
|
||||
@@ -119,7 +121,7 @@ enum WCallCommand: Equatable, Encodable, Decodable {
|
||||
case start(media: CallMediaType, aesKey: String? = nil, iceServers: [RTCIceServer]? = nil, relay: Bool? = nil)
|
||||
case offer(offer: String, iceCandidates: String, media: CallMediaType, aesKey: String? = nil, iceServers: [RTCIceServer]? = nil, relay: Bool? = nil)
|
||||
case answer(answer: String, iceCandidates: String)
|
||||
case ice(iceCandidates: String)
|
||||
case ice(iceCandidates: String, session: String? = nil)
|
||||
case media(source: CallMediaSource, enable: Bool)
|
||||
case end
|
||||
|
||||
@@ -131,6 +133,7 @@ enum WCallCommand: Equatable, Encodable, Decodable {
|
||||
case offer
|
||||
case answer
|
||||
case iceCandidates
|
||||
case session
|
||||
case enable
|
||||
case iceServers
|
||||
case relay
|
||||
@@ -174,9 +177,10 @@ enum WCallCommand: Equatable, Encodable, Decodable {
|
||||
try container.encode("answer", forKey: .type)
|
||||
try container.encode(answer, forKey: .answer)
|
||||
try container.encode(iceCandidates, forKey: .iceCandidates)
|
||||
case let .ice(iceCandidates):
|
||||
case let .ice(iceCandidates, session):
|
||||
try container.encode("ice", forKey: .type)
|
||||
try container.encode(iceCandidates, forKey: .iceCandidates)
|
||||
try container.encodeIfPresent(session, forKey: .session)
|
||||
case let .media(source, enable):
|
||||
try container.encode("media", forKey: .type)
|
||||
try container.encode(source, forKey: .media)
|
||||
@@ -213,7 +217,8 @@ enum WCallCommand: Equatable, Encodable, Decodable {
|
||||
self = .answer(answer: answer, iceCandidates: iceCandidates)
|
||||
case "ice":
|
||||
let iceCandidates = try container.decode(String.self, forKey: CodingKeys.iceCandidates)
|
||||
self = .ice(iceCandidates: iceCandidates)
|
||||
let session = try container.decodeIfPresent(String.self, forKey: CodingKeys.session)
|
||||
self = .ice(iceCandidates: iceCandidates, session: session)
|
||||
case "media":
|
||||
let source = try container.decode(CallMediaSource.self, forKey: CodingKeys.source)
|
||||
let enable = try container.decode(Bool.self, forKey: CodingKeys.enable)
|
||||
@@ -231,7 +236,7 @@ enum WCallResponse: Equatable, Decodable {
|
||||
case capabilities(capabilities: CallCapabilities)
|
||||
case offer(offer: String, iceCandidates: String, capabilities: CallCapabilities)
|
||||
case answer(answer: String, iceCandidates: String)
|
||||
case ice(iceCandidates: String)
|
||||
case ice(iceCandidates: String, session: String? = nil)
|
||||
case connection(state: ConnectionState)
|
||||
case connected(connectionInfo: ConnectionInfo)
|
||||
case peerMedia(source: CallMediaSource, enabled: Bool)
|
||||
@@ -246,6 +251,7 @@ enum WCallResponse: Equatable, Decodable {
|
||||
case offer
|
||||
case answer
|
||||
case iceCandidates
|
||||
case session
|
||||
case state
|
||||
case connectionInfo
|
||||
case message
|
||||
@@ -290,7 +296,8 @@ enum WCallResponse: Equatable, Decodable {
|
||||
self = .answer(answer: answer, iceCandidates: iceCandidates)
|
||||
case "ice":
|
||||
let iceCandidates = try container.decode(String.self, forKey: CodingKeys.iceCandidates)
|
||||
self = .ice(iceCandidates: iceCandidates)
|
||||
let session = try container.decodeIfPresent(String.self, forKey: CodingKeys.session)
|
||||
self = .ice(iceCandidates: iceCandidates, session: session)
|
||||
case "connection":
|
||||
let state = try container.decode(ConnectionState.self, forKey: CodingKeys.state)
|
||||
self = .connection(state: state)
|
||||
@@ -333,9 +340,10 @@ extension WCallResponse: Encodable {
|
||||
try container.encode("answer", forKey: .type)
|
||||
try container.encode(answer, forKey: .answer)
|
||||
try container.encode(iceCandidates, forKey: .iceCandidates)
|
||||
case let .ice(iceCandidates):
|
||||
case let .ice(iceCandidates, session):
|
||||
try container.encode("ice", forKey: .type)
|
||||
try container.encode(iceCandidates, forKey: .iceCandidates)
|
||||
try container.encodeIfPresent(session, forKey: .session)
|
||||
case let .connection(state):
|
||||
try container.encode("connection", forKey: .type)
|
||||
try container.encode(state, forKey: .state)
|
||||
|
||||
@@ -20,6 +20,8 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
|
||||
}()
|
||||
private static let ivTagBytes: Int = 28
|
||||
private static let enableEncryption: Bool = true
|
||||
private static let reconnectTimeout: UInt64 = 30_000_000_000
|
||||
private static let iceRestartInterval: UInt64 = 10_000_000_000
|
||||
private var chat_ctrl = getChatCtrl()
|
||||
|
||||
struct Call {
|
||||
@@ -37,6 +39,9 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
|
||||
var frameEncryptor: RTCFrameEncryptor?
|
||||
var frameDecryptor: RTCFrameDecryptor?
|
||||
var peerHasOldVersion: Bool
|
||||
var wasConnected = false
|
||||
var reconnection: Task<Void, Never>? = nil
|
||||
var iceRestarts: Task<Void, Never>? = nil
|
||||
}
|
||||
|
||||
struct NotConnectedCall {
|
||||
@@ -257,11 +262,15 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
|
||||
resp = .error(message: "answer: remote description is not set")
|
||||
}
|
||||
}
|
||||
case let .ice(iceCandidates):
|
||||
case let .ice(iceCandidates, session):
|
||||
if let pc = pc,
|
||||
let remoteIceCandidates: [RTCIceCandidate] = decodeJSON(decompressFromBase64(input: iceCandidates)) {
|
||||
addIceCandidates(pc, remoteIceCandidates)
|
||||
resp = .ok
|
||||
if let session {
|
||||
resp = await receiveRestartSession(pc, session, remoteIceCandidates)
|
||||
} else {
|
||||
addIceCandidates(pc, remoteIceCandidates)
|
||||
resp = .ok
|
||||
}
|
||||
} else {
|
||||
resp = .error(message: "ice: call not started")
|
||||
}
|
||||
@@ -301,6 +310,101 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
|
||||
}
|
||||
}
|
||||
|
||||
private func startReconnecting(_ connection: RTCPeerConnection) async {
|
||||
activeCall?.reconnection = Task {
|
||||
_ = try? await Task.sleep(nanoseconds: WebRTCClient.reconnectTimeout)
|
||||
if Task.isCancelled { return }
|
||||
await sendConnectionState(connection, "disconnected")
|
||||
endCall()
|
||||
}
|
||||
// Only the party that sent the initial offer restarts ICE, so that restart offers never collide
|
||||
if connection.localDescription?.type == .offer {
|
||||
activeCall?.iceRestarts = Task {
|
||||
while !Task.isCancelled {
|
||||
await restartIce(connection)
|
||||
_ = try? await Task.sleep(nanoseconds: WebRTCClient.iceRestartInterval)
|
||||
}
|
||||
}
|
||||
}
|
||||
await sendConnectionState(connection, "reconnecting")
|
||||
}
|
||||
|
||||
private func stopReconnecting() {
|
||||
activeCall?.reconnection?.cancel()
|
||||
activeCall?.reconnection = nil
|
||||
activeCall?.iceRestarts?.cancel()
|
||||
activeCall?.iceRestarts = nil
|
||||
}
|
||||
|
||||
private func restartIce(_ connection: RTCPeerConnection) async {
|
||||
// the previous restart offer is not answered yet
|
||||
if connection.signalingState != .stable { return }
|
||||
let (offer, error) = await connection.offer(iceRestart: true)
|
||||
if let offer {
|
||||
await sendRestartSession(offer)
|
||||
} else {
|
||||
logger.error("WebRTCClient: ICE restart error: \(error?.localizedDescription ?? "unknown error")")
|
||||
}
|
||||
}
|
||||
|
||||
private func receiveRestartSession(_ pc: RTCPeerConnection, _ session: String, _ remoteIceCandidates: [RTCIceCandidate]) async -> WCallResponse {
|
||||
guard let desc: CustomRTCSessionDescription = decodeJSON(decompressFromBase64(input: session)),
|
||||
let type = desc.type, let sdp = desc.sdp else {
|
||||
return .error(message: "ice: invalid session")
|
||||
}
|
||||
let remoteDesc = RTCSessionDescription(type: type.toWebRTCSdpType(), sdp: sdp)
|
||||
if type == .offer && pc.localDescription?.type != .offer {
|
||||
if (try? await pc.setRemoteDescription(remoteDesc)) == nil {
|
||||
return .error(message: "ice: remote offer is not set")
|
||||
}
|
||||
let (answer, error) = await pc.answer()
|
||||
guard let answer else {
|
||||
return .error(message: "ice: answer error: \(error?.localizedDescription ?? "unknown error")")
|
||||
}
|
||||
addIceCandidates(pc, remoteIceCandidates)
|
||||
await sendRestartSession(answer)
|
||||
} else if type == .answer && pc.signalingState == .haveLocalOffer {
|
||||
if (try? await pc.setRemoteDescription(remoteDesc)) == nil {
|
||||
return .error(message: "ice: remote answer is not set")
|
||||
}
|
||||
addIceCandidates(pc, remoteIceCandidates)
|
||||
} else {
|
||||
return .error(message: "ice: unexpected ICE restart \(type.rawValue)")
|
||||
}
|
||||
return .ok
|
||||
}
|
||||
|
||||
private func sendRestartSession(_ desc: RTCSessionDescription) async {
|
||||
await sendCallResponse(.init(
|
||||
corrId: nil,
|
||||
resp: .ice(
|
||||
iceCandidates: compressToBase64(input: encodeJSON(await getInitialIceCandidates())),
|
||||
session: compressToBase64(input: encodeJSON(CustomRTCSessionDescription(type: desc.type.toSdpType(), sdp: desc.sdp)))
|
||||
),
|
||||
command: nil)
|
||||
)
|
||||
waitForMoreIceCandidates()
|
||||
}
|
||||
|
||||
private func sendConnectionState(_ connection: RTCPeerConnection, _ connectionState: String) async {
|
||||
guard let iceConnectionState = connection.iceConnectionState.toString(),
|
||||
let iceGatheringState = connection.iceGatheringState.toString(),
|
||||
let signalingState = connection.signalingState.toString()
|
||||
else {
|
||||
return
|
||||
}
|
||||
await sendCallResponse(.init(
|
||||
corrId: nil,
|
||||
resp: .connection(state: ConnectionState(
|
||||
connectionState: connectionState,
|
||||
iceConnectionState: iceConnectionState,
|
||||
iceGatheringState: iceGatheringState,
|
||||
signalingState: signalingState)
|
||||
),
|
||||
command: nil)
|
||||
)
|
||||
}
|
||||
|
||||
// Spec: spec/services/calls.md#sendIceCandidates
|
||||
func sendIceCandidates(_ candidates: [RTCIceCandidate]) async {
|
||||
await self.sendCallResponse(.init(
|
||||
@@ -657,6 +761,8 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
|
||||
(notConnectedCall?.localCameraAndTrack?.0 as? RTCCameraVideoCapturer)?.stopCapture()
|
||||
guard let call = activeCall else { return }
|
||||
logger.debug("WebRTCClient: ending the call")
|
||||
call.reconnection?.cancel()
|
||||
call.iceRestarts?.cancel()
|
||||
call.connection.close()
|
||||
call.connection.delegate = nil
|
||||
call.frameEncryptor?.delegate = nil
|
||||
@@ -678,16 +784,18 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
|
||||
|
||||
|
||||
extension WebRTC.RTCPeerConnection {
|
||||
func mediaConstraints() -> RTCMediaConstraints {
|
||||
RTCMediaConstraints(
|
||||
mandatoryConstraints: [kRTCMediaConstraintsOfferToReceiveAudio: kRTCMediaConstraintsValueTrue,
|
||||
kRTCMediaConstraintsOfferToReceiveVideo: kRTCMediaConstraintsValueTrue],
|
||||
optionalConstraints: nil)
|
||||
func mediaConstraints(iceRestart: Bool = false) -> RTCMediaConstraints {
|
||||
var constraints = [kRTCMediaConstraintsOfferToReceiveAudio: kRTCMediaConstraintsValueTrue,
|
||||
kRTCMediaConstraintsOfferToReceiveVideo: kRTCMediaConstraintsValueTrue]
|
||||
if iceRestart {
|
||||
constraints[kRTCMediaConstraintsIceRestart] = kRTCMediaConstraintsValueTrue
|
||||
}
|
||||
return RTCMediaConstraints(mandatoryConstraints: constraints, optionalConstraints: nil)
|
||||
}
|
||||
|
||||
func offer() async -> (RTCSessionDescription?, Error?) {
|
||||
func offer(iceRestart: Bool = false) async -> (RTCSessionDescription?, Error?) {
|
||||
await withCheckedContinuation { cont in
|
||||
offer(for: mediaConstraints()) { (sdp, error) in
|
||||
offer(for: mediaConstraints(iceRestart: iceRestart)) { (sdp, error) in
|
||||
self.processSDP(cont, sdp, error)
|
||||
}
|
||||
}
|
||||
@@ -766,24 +874,20 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
|
||||
func peerConnection(_ connection: RTCPeerConnection, didChange newState: RTCIceConnectionState) {
|
||||
debugPrint("Connection new connection state: \(newState.toString() ?? "" + newState.rawValue.description) \(connection.receivers)")
|
||||
|
||||
guard let connectionStateString = newState.toString(),
|
||||
let iceConnectionStateString = connection.iceConnectionState.toString(),
|
||||
let iceGatheringStateString = connection.iceGatheringState.toString(),
|
||||
let signalingStateString = connection.signalingState.toString()
|
||||
else {
|
||||
return
|
||||
}
|
||||
guard let connectionStateString = newState.toString() else { return }
|
||||
Task {
|
||||
await sendCallResponse(.init(
|
||||
corrId: nil,
|
||||
resp: .connection(state: ConnectionState(
|
||||
connectionState: connectionStateString,
|
||||
iceConnectionState: iceConnectionStateString,
|
||||
iceGatheringState: iceGatheringStateString,
|
||||
signalingState: signalingStateString)
|
||||
),
|
||||
command: nil)
|
||||
)
|
||||
if activeCall?.wasConnected == true {
|
||||
switch newState {
|
||||
case .disconnected, .failed:
|
||||
if activeCall?.reconnection == nil {
|
||||
await startReconnecting(connection)
|
||||
}
|
||||
return
|
||||
case .new, .checking: return
|
||||
default: ()
|
||||
}
|
||||
}
|
||||
await sendConnectionState(connection, connectionStateString)
|
||||
|
||||
switch newState {
|
||||
case .checking:
|
||||
@@ -792,7 +896,10 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
|
||||
}
|
||||
let enableSpeaker: Bool = ChatModel.shared.activeCall?.localMediaSources.hasVideo == true
|
||||
setSpeakerEnabledAndConfigureSession(enableSpeaker)
|
||||
case .connected: sendConnectedEvent(connection)
|
||||
case .connected:
|
||||
stopReconnecting()
|
||||
activeCall?.wasConnected = true
|
||||
sendConnectedEvent(connection)
|
||||
case .disconnected, .failed: endCall()
|
||||
default: ()
|
||||
}
|
||||
|
||||
@@ -34,10 +34,12 @@ public struct WebRTCSession: Codable {
|
||||
|
||||
// Spec: spec/services/calls.md#WebRTCExtraInfo
|
||||
public struct WebRTCExtraInfo: Codable {
|
||||
public init(rtcIceCandidates: String) {
|
||||
public init(rtcSession: String?, rtcIceCandidates: String) {
|
||||
self.rtcSession = rtcSession
|
||||
self.rtcIceCandidates = rtcIceCandidates
|
||||
}
|
||||
|
||||
public var rtcSession: String?
|
||||
public var rtcIceCandidates: String
|
||||
}
|
||||
|
||||
|
||||
@@ -103,6 +103,14 @@ All signaling messages are exchanged via E2E encrypted SMP messages (no central
|
||||
5. `CallViewRenderers` manages WebRTC video rendering surfaces.
|
||||
6. Call status updates are sent via `apiCallStatus(contact:status:)`.
|
||||
|
||||
### 6.1 Reconnection
|
||||
|
||||
1. A connected call loses its connection, e.g. when the device switches between Wi-Fi and mobile network.
|
||||
2. The call shows "reconnecting…" and plays the connecting sound. Call duration continues, as this state is not reported to the core.
|
||||
3. The callee (the party that sent the initial offer) restarts ICE every 10 seconds, sending the new offer via `apiSendCallExtraInfo` with `rtcSession`; the caller answers the same way.
|
||||
4. When the connection is restored, the call returns to connected state.
|
||||
5. If the connection is not restored in 30 seconds, the call ends. Clients without ICE restart support ignore the restart offer, so the call ends in this case.
|
||||
|
||||
### 7. Audio Routing
|
||||
|
||||
1. `CallAudioDeviceManager` handles audio device selection.
|
||||
@@ -141,7 +149,7 @@ All signaling messages are exchanged via E2E encrypted SMP messages (no central
|
||||
| `RcvCallInvitation` | `SimpleXChat/CallTypes.swift` | Incoming call: user, contact, callType, sharedKey, callUUID, callTs |
|
||||
| `WebRTCCallOffer` | `SimpleXChat/CallTypes.swift` | SDP offer with call type and WebRTC session data |
|
||||
| `WebRTCSession` | `SimpleXChat/CallTypes.swift` | `rtcSession` (SDP) and `rtcIceCandidates` (serialized) |
|
||||
| `WebRTCExtraInfo` | `SimpleXChat/CallTypes.swift` | Additional ICE candidates sent after initial offer/answer |
|
||||
| `WebRTCExtraInfo` | `SimpleXChat/CallTypes.swift` | Additional ICE candidates sent after initial offer/answer, and ICE restart offer/answer |
|
||||
| `WebRTCCallStatus` | `SimpleXChat/CallTypes.swift` | Call lifecycle states for status reporting |
|
||||
| `CallMediaSource` | `SimpleXChat/CallTypes.swift` | `.mic`, `.camera`, `.screenAudio`, `.screenVideo`, `.unknown` |
|
||||
| `VideoCamera` | `SimpleXChat/CallTypes.swift` | `.user` (front) or `.environment` (rear) camera |
|
||||
@@ -152,7 +160,7 @@ All signaling messages are exchanged via E2E encrypted SMP messages (no central
|
||||
|-------|-------|----------|
|
||||
| Chat not ready on CallKit answer | App suspended, slow startup | `waitUntilChatStarted` with 30s timeout; `action.fail()` on timeout |
|
||||
| Call invitation not found | Race condition between notification and event processing | `justRefreshCallInvitations()` retry |
|
||||
| WebRTC peer connection failure | NAT traversal, network issues | Call ends with error status |
|
||||
| WebRTC peer connection failure | NAT traversal, network issues | Before connection: call ends with error status; after connection: reconnection with ICE restart, call ends if not reconnected in 30 seconds |
|
||||
| CallKit action fail | Internal state mismatch | `action.fail()` called, call cleaned up |
|
||||
| No camera/mic permission | User denied permissions | Permission request dialog shown |
|
||||
|
||||
|
||||
@@ -179,7 +179,7 @@ The `ChatCommand` enum ([`AppAPITypes.swift` L15](../Shared/Model/AppAPITypes.sw
|
||||
| `apiRejectCall` | `contact` | Reject incoming call | [L158](../Shared/Model/AppAPITypes.swift#L158) |
|
||||
| `apiSendCallOffer` | `contact, callOffer: WebRTCCallOffer` | Send SDP offer | [L159](../Shared/Model/AppAPITypes.swift#L159) |
|
||||
| `apiSendCallAnswer` | `contact, answer: WebRTCSession` | Send SDP answer | [L160](../Shared/Model/AppAPITypes.swift#L160) |
|
||||
| `apiSendCallExtraInfo` | `contact, extraInfo: WebRTCExtraInfo` | Send ICE candidates | [L161](../Shared/Model/AppAPITypes.swift#L161) |
|
||||
| `apiSendCallExtraInfo` | `contact, extraInfo: WebRTCExtraInfo` | Send ICE candidates or ICE restart offer/answer | [L171](../Shared/Model/AppAPITypes.swift#L171) |
|
||||
| `apiEndCall` | `contact` | End active call | [L162](../Shared/Model/AppAPITypes.swift#L162) |
|
||||
| `apiGetCallInvitations` | -- | Get pending call invitations | [L163](../Shared/Model/AppAPITypes.swift#L163) |
|
||||
| `apiCallStatus` | `contact, callStatus` | Report call status change | [L164](../Shared/Model/AppAPITypes.swift#L164) |
|
||||
|
||||
@@ -116,7 +116,10 @@ Manages the WebRTC peer connection, media streams, and data channels.
|
||||
| [`sendCallCommand`](../../Shared/Views/Call/WebRTCClient.swift#L176) | Dispatches WCallCommand (offer/answer/ICE) | L176 |
|
||||
| [`addIceCandidates`](../../Shared/Views/Call/WebRTCClient.swift#L165) | `peerConnection.add(RTCIceCandidate)` | L165 |
|
||||
| [`getInitialIceCandidates`](../../Shared/Views/Call/WebRTCClient.swift#L285) | Collects initial ICE candidates | L285 |
|
||||
| [`sendIceCandidates`](../../Shared/Views/Call/WebRTCClient.swift#L305) | Sends gathered ICE candidates | L305 |
|
||||
| [`sendIceCandidates`](../../Shared/Views/Call/WebRTCClient.swift#L409) | Sends gathered ICE candidates | L409 |
|
||||
| [`startReconnecting`](../../Shared/Views/Call/WebRTCClient.swift#L313) | Reports "reconnecting", starts reconnection timeout and, for the party that sent the initial offer, periodic ICE restarts | L313 |
|
||||
| [`restartIce`](../../Shared/Views/Call/WebRTCClient.swift#L339) | Creates ICE restart offer and sends it with candidates in `.ice(iceCandidates:session:)` | L339 |
|
||||
| [`receiveRestartSession`](../../Shared/Views/Call/WebRTCClient.swift#L350) | Applies ICE restart offer (and sends the answer) or answer received in `.ice` command | L350 |
|
||||
| [`enableMedia`](../../Shared/Views/Call/WebRTCClient.swift#L365) | Enable/disable audio or video track | L365 |
|
||||
| [`setupLocalTracks`](../../Shared/Views/Call/WebRTCClient.swift#L423) | Creates audio/video tracks and adds to connection | L423 |
|
||||
| [`startCaptureLocalVideo`](../../Shared/Views/Call/WebRTCClient.swift#L581) | Front/back camera toggle and capture start | L581 |
|
||||
@@ -179,6 +182,10 @@ All call signaling travels through the same encrypted SMP message channels used
|
||||
5. P2P connection established
|
||||
```
|
||||
|
||||
### Reconnection
|
||||
|
||||
When a connected call loses ICE connection (`.disconnected` or `.failed`), `WebRTCClient` reports `"reconnecting"` connection state instead of ending the call. `ActiveCallView` sets `.reconnecting` call state and plays the connecting sound; this state is not sent to the core via `apiCallStatus`, and `connectedAt` is kept. The party that sent the initial offer (callee) restarts ICE every 10 seconds: the restart offer and answer are sent via `apiSendCallExtraInfo` with `rtcSession` (`x.call.extra`). When ICE connects again, the call returns to `.connected`. If it does not reconnect in 30 seconds, `WebRTCClient` reports `"disconnected"` and ends the call.
|
||||
|
||||
### API Commands
|
||||
|
||||
| Command | Direction | Purpose |
|
||||
@@ -187,7 +194,7 @@ All call signaling travels through the same encrypted SMP message channels used
|
||||
| `apiRejectCall(contact:)` | Callee -> Caller | Reject call |
|
||||
| `apiSendCallOffer(contact:, callOffer:)` | Callee -> Caller | Send SDP offer |
|
||||
| `apiSendCallAnswer(contact:, answer:)` | Caller -> Callee | Send SDP answer |
|
||||
| `apiSendCallExtraInfo(contact:, extraInfo:)` | Both | Send ICE candidates |
|
||||
| `apiSendCallExtraInfo(contact:, extraInfo:)` | Both | Send ICE candidates, or ICE restart offer/answer in `rtcSession` |
|
||||
| `apiEndCall(contact:)` | Either | End call |
|
||||
| `apiGetCallInvitations` | -- | Get pending invitations |
|
||||
| `apiCallStatus(contact:, callStatus:)` | -- | Report status change |
|
||||
@@ -317,10 +324,11 @@ struct WebRTCSession {
|
||||
}
|
||||
```
|
||||
|
||||
### [WebRTCExtraInfo](../../SimpleXChat/CallTypes.swift#L36-L42)
|
||||
### [WebRTCExtraInfo](../../SimpleXChat/CallTypes.swift#L36-L45)
|
||||
|
||||
```swift
|
||||
struct WebRTCExtraInfo {
|
||||
var rtcSession: String? // ICE restart offer or answer
|
||||
var rtcIceCandidates: String // Additional ICE candidates
|
||||
}
|
||||
```
|
||||
@@ -330,7 +338,7 @@ struct WebRTCExtraInfo {
|
||||
Stored in `ChatModel.activeCall`:
|
||||
- Contact reference
|
||||
- Call UUID
|
||||
- Call state (enum: `.waitCapabilities`, `.invitationAccepted`, `.offerSent`, `.answerReceived`, `.connected`, etc.)
|
||||
- Call state (enum: `.waitCapabilities`, `.invitationAccepted`, `.offerSent`, `.answerReceived`, `.connected`, `.reconnecting`, etc.)
|
||||
- Media type
|
||||
- WebRTCClient reference
|
||||
|
||||
|
||||
+24
-15
@@ -52,8 +52,7 @@ import dev.icerock.moko.resources.StringResource
|
||||
import dev.icerock.moko.resources.compose.painterResource
|
||||
import dev.icerock.moko.resources.compose.stringResource
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.flow.distinctUntilChanged
|
||||
import kotlinx.coroutines.flow.filterNotNull
|
||||
import kotlinx.coroutines.flow.first
|
||||
import kotlinx.datetime.Clock
|
||||
import kotlinx.serialization.encodeToString
|
||||
import java.io.Closeable
|
||||
@@ -168,13 +167,24 @@ actual fun ActiveCallView() {
|
||||
CallSoundsPlayer.stop()
|
||||
}
|
||||
is WCallResponse.Ice -> withBGApi {
|
||||
chatModel.controller.apiSendCallExtraInfo(callRh, call.contact, r.iceCandidates)
|
||||
chatModel.controller.apiSendCallExtraInfo(callRh, call.contact, r.session, r.iceCandidates)
|
||||
}
|
||||
is WCallResponse.Connection ->
|
||||
try {
|
||||
// the call is restored and NOT sent to the core
|
||||
// Makes sure the core DOES NOT record the duration wrong if a reconnect happens
|
||||
if (r.state.connectionState == "reconnecting") {
|
||||
if (call.callState != CallState.Reconnecting) {
|
||||
CallSoundsPlayer.startConnectingCallSound(scope)
|
||||
}
|
||||
updateActiveCall(call) { it.copy(callState = CallState.Reconnecting) }
|
||||
} else try {
|
||||
val callStatus = json.decodeFromString<WebRTCCallStatus>("\"${r.state.connectionState}\"")
|
||||
if (callStatus == WebRTCCallStatus.Connected) {
|
||||
updateActiveCall(call) { it.copy(callState = CallState.Connected, connectedAt = Clock.System.now()) }
|
||||
if (call.callState == CallState.Reconnecting) {
|
||||
CallSoundsPlayer.stop()
|
||||
}
|
||||
// connectedAt is only set ONCE, this way call duration is NOT reset when the call is restored from reconnecting
|
||||
updateActiveCall(call) { it.copy(callState = CallState.Connected, connectedAt = it.connectedAt ?: Clock.System.now()) }
|
||||
}
|
||||
withBGApi { chatModel.controller.apiCallStatus(callRh, call.contact, callStatus) }
|
||||
} catch (e: Throwable) {
|
||||
@@ -705,18 +715,17 @@ fun WebRTCView(callCommand: SnapshotStateList<WCallCommand>, onResponse: (WVAPIM
|
||||
val wv = webView.value
|
||||
if (wv != null) {
|
||||
LaunchedEffect(Unit) {
|
||||
snapshotFlow { callCommand.firstOrNull() }
|
||||
.distinctUntilChanged()
|
||||
.filterNotNull()
|
||||
.collect {
|
||||
while (callCommand.isNotEmpty()) {
|
||||
val cmd = callCommand.removeFirstOrNull()
|
||||
Log.d(TAG, "WebRTCView LaunchedEffect executing ${cmd?.javaClass?.simpleName}")
|
||||
if (cmd != null) {
|
||||
processCommand(wv, cmd)
|
||||
}
|
||||
while (isActive) {
|
||||
// waiting for the first command would miss a command equal to the previous one
|
||||
snapshotFlow { callCommand.isNotEmpty() }.first { it }
|
||||
while (callCommand.isNotEmpty()) {
|
||||
val cmd = callCommand.removeFirstOrNull()
|
||||
Log.d(TAG, "WebRTCView LaunchedEffect executing ${cmd?.javaClass?.simpleName}")
|
||||
if (cmd != null) {
|
||||
processCommand(wv, cmd)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
val assetLoader = WebViewAssetLoader.Builder()
|
||||
|
||||
+3
-3
@@ -2185,8 +2185,8 @@ object ChatController {
|
||||
return r.result is CR.CmdOk
|
||||
}
|
||||
|
||||
suspend fun apiSendCallExtraInfo(rh: Long?, contact: Contact, rtcIceCandidates: String): Boolean {
|
||||
val extraInfo = WebRTCExtraInfo(rtcIceCandidates)
|
||||
suspend fun apiSendCallExtraInfo(rh: Long?, contact: Contact, rtcSession: String?, rtcIceCandidates: String): Boolean {
|
||||
val extraInfo = WebRTCExtraInfo(rtcSession, rtcIceCandidates)
|
||||
val r = sendCmd(rh, CC.ApiSendCallExtraInfo(contact, extraInfo))
|
||||
return r.result is CR.CmdOk
|
||||
}
|
||||
@@ -3436,7 +3436,7 @@ object ChatController {
|
||||
}
|
||||
is CR.CallExtraInfo -> {
|
||||
withCall(r, r.contact) { _ ->
|
||||
chatModel.callCommand.add(WCallCommand.Ice(iceCandidates = r.extraInfo.rtcIceCandidates))
|
||||
chatModel.callCommand.add(WCallCommand.Ice(iceCandidates = r.extraInfo.rtcIceCandidates, session = r.extraInfo.rtcSession))
|
||||
}
|
||||
}
|
||||
is CR.CallEnded -> {
|
||||
|
||||
+5
-3
@@ -56,6 +56,7 @@ enum class CallState {
|
||||
AnswerReceived,
|
||||
Negotiated,
|
||||
Connected,
|
||||
Reconnecting,
|
||||
Ended;
|
||||
|
||||
val text: String get() = when(this) {
|
||||
@@ -67,6 +68,7 @@ enum class CallState {
|
||||
AnswerReceived -> generalGetString(MR.strings.callstate_received_confirmation)
|
||||
Negotiated -> generalGetString(MR.strings.callstate_connecting)
|
||||
Connected -> generalGetString(MR.strings.callstate_connected)
|
||||
Reconnecting -> generalGetString(MR.strings.callstate_reconnecting)
|
||||
Ended -> generalGetString(MR.strings.callstate_ended)
|
||||
}
|
||||
}
|
||||
@@ -91,7 +93,7 @@ sealed class WCallCommand {
|
||||
@Serializable @SerialName("start") data class Start(val media: CallMediaType, val aesKey: String? = null, val iceServers: List<RTCIceServer>? = null, val relay: Boolean? = null): WCallCommand()
|
||||
@Serializable @SerialName("offer") data class Offer(val offer: String, val iceCandidates: String, val media: CallMediaType, val aesKey: String? = null, val iceServers: List<RTCIceServer>? = null, val relay: Boolean? = null): WCallCommand()
|
||||
@Serializable @SerialName("answer") data class Answer (val answer: String, val iceCandidates: String): WCallCommand()
|
||||
@Serializable @SerialName("ice") data class Ice(val iceCandidates: String): WCallCommand()
|
||||
@Serializable @SerialName("ice") data class Ice(val iceCandidates: String, val session: String? = null): WCallCommand()
|
||||
@Serializable @SerialName("media") data class Media(val source: CallMediaSource, val enable: Boolean): WCallCommand()
|
||||
@Serializable @SerialName("camera") data class Camera(val camera: VideoCamera): WCallCommand()
|
||||
@Serializable @SerialName("description") data class Description(val state: String, val description: String): WCallCommand()
|
||||
@@ -104,7 +106,7 @@ sealed class WCallResponse {
|
||||
@Serializable @SerialName("capabilities") data class Capabilities(val capabilities: CallCapabilities): WCallResponse()
|
||||
@Serializable @SerialName("offer") data class Offer(val offer: String, val iceCandidates: String, val capabilities: CallCapabilities): WCallResponse()
|
||||
@Serializable @SerialName("answer") data class Answer(val answer: String, val iceCandidates: String): WCallResponse()
|
||||
@Serializable @SerialName("ice") data class Ice(val iceCandidates: String): WCallResponse()
|
||||
@Serializable @SerialName("ice") data class Ice(val iceCandidates: String, val session: String? = null): WCallResponse()
|
||||
@Serializable @SerialName("connection") data class Connection(val state: ConnectionState): WCallResponse()
|
||||
@Serializable @SerialName("connected") data class Connected(val connectionInfo: ConnectionInfo): WCallResponse()
|
||||
@Serializable @SerialName("peerMedia") data class PeerMedia(val source: CallMediaSource, val enabled: Boolean): WCallResponse()
|
||||
@@ -116,7 +118,7 @@ sealed class WCallResponse {
|
||||
|
||||
@Serializable data class WebRTCCallOffer(val callType: CallType, val rtcSession: WebRTCSession)
|
||||
@Serializable data class WebRTCSession(val rtcSession: String, val rtcIceCandidates: String)
|
||||
@Serializable data class WebRTCExtraInfo(val rtcIceCandidates: String)
|
||||
@Serializable data class WebRTCExtraInfo(val rtcSession: String? = null, val rtcIceCandidates: String)
|
||||
@Serializable data class CallType(val media: CallMediaType, val capabilities: CallCapabilities)
|
||||
@Serializable data class RcvCallInvitation(
|
||||
val remoteHostId: Long?,
|
||||
|
||||
@@ -1301,6 +1301,7 @@
|
||||
<string name="callstate_received_answer">received answer…</string>
|
||||
<string name="callstate_received_confirmation">received confirmation…</string>
|
||||
<string name="callstate_connecting">connecting…</string>
|
||||
<string name="callstate_reconnecting">reconnecting…</string>
|
||||
<string name="callstate_connected">connected</string>
|
||||
<string name="callstate_ended">ended</string>
|
||||
|
||||
|
||||
@@ -54,6 +54,8 @@ let inactiveCallMediaSources = {
|
||||
let activeCall;
|
||||
let notConnectedCall;
|
||||
let answerTimeout = 30000;
|
||||
let reconnectTimeout = 30000;
|
||||
let iceRestartInterval = 10000;
|
||||
var useWorker = false;
|
||||
var isDesktop = false;
|
||||
var localizedState = "";
|
||||
@@ -75,7 +77,8 @@ const processCommand = (function () {
|
||||
return {
|
||||
peerConnectionConfig: {
|
||||
iceServers: iceServers !== null && iceServers !== void 0 ? iceServers : defaultIceServers,
|
||||
iceCandidatePoolSize: 10,
|
||||
// Chrome gathers no candidates on ICE restart when candidates pool is used
|
||||
iceCandidatePoolSize: 0,
|
||||
encodedInsertableStreams,
|
||||
iceTransportPolicy: relay ? "relay" : "all",
|
||||
// needed for Android WebView >= 69 && <= 72 where default was "plan-b" which is incompatible with transceivers
|
||||
@@ -202,10 +205,14 @@ const processCommand = (function () {
|
||||
layout: (_c = notConnectedCall === null || notConnectedCall === void 0 ? void 0 : notConnectedCall.layout) !== null && _c !== void 0 ? _c : LayoutType.Default,
|
||||
cameraTrackWasSetBefore: localStream.getVideoTracks().length > 0,
|
||||
peerHasOldVersion: false,
|
||||
receiveRestartSession,
|
||||
};
|
||||
localOrPeerMediaSourcesChanged(call);
|
||||
await setupMediaStreams(call);
|
||||
let connectionTimeout = setTimeout(connectionHandler, answerTimeout);
|
||||
let reconnectingTimeout;
|
||||
let iceRestartTimeout;
|
||||
let wasConnected = false;
|
||||
if (pc.connectionState) {
|
||||
pc.addEventListener("connectionstatechange", connectionStateChange);
|
||||
}
|
||||
@@ -216,30 +223,49 @@ const processCommand = (function () {
|
||||
async function connectionStateChange() {
|
||||
// "failed" means the second party did not answer in time (15 sec timeout in Chrome WebView)
|
||||
// See https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/p2p/base/p2p_constants.cc;l=70)
|
||||
if (pc.connectionState !== "failed")
|
||||
if (pc.connectionState !== "failed" || wasConnected)
|
||||
connectionHandler();
|
||||
}
|
||||
async function connectionHandler() {
|
||||
async function connectionHandler(reconnectTimedOut = false) {
|
||||
var _a;
|
||||
let state = (_a = pc.connectionState) !== null && _a !== void 0 ? _a : (pc.iceConnectionState != "completed" && pc.iceConnectionState != "checking"
|
||||
? pc.iceConnectionState
|
||||
: pc.iceConnectionState == "completed"
|
||||
? "connected"
|
||||
: "connecting"); /* webView 69-70 doesn't have connectionState yet */
|
||||
if (wasConnected) {
|
||||
if (reconnectTimedOut) {
|
||||
reconnectingTimeout = undefined;
|
||||
if (activeCall !== call || state == "connected")
|
||||
return;
|
||||
state = "disconnected";
|
||||
}
|
||||
else if (state == "disconnected" || state == "failed") {
|
||||
if (reconnectingTimeout !== undefined)
|
||||
return;
|
||||
reconnectingTimeout = setTimeout(() => connectionHandler(true), reconnectTimeout);
|
||||
if (isIceRestarter())
|
||||
restartIce();
|
||||
state = "reconnecting";
|
||||
}
|
||||
else if (state == "connecting" || state == "new") {
|
||||
return;
|
||||
}
|
||||
}
|
||||
sendMessageToNative({
|
||||
resp: {
|
||||
type: "connection",
|
||||
state: {
|
||||
connectionState: (_a = pc.connectionState) !== null && _a !== void 0 ? _a : (pc.iceConnectionState != "completed" && pc.iceConnectionState != "checking"
|
||||
? pc.iceConnectionState
|
||||
: pc.iceConnectionState == "completed"
|
||||
? "connected"
|
||||
: "connecting") /* webView 69-70 doesn't have connectionState yet */,
|
||||
connectionState: state,
|
||||
iceConnectionState: pc.iceConnectionState,
|
||||
iceGatheringState: pc.iceGatheringState,
|
||||
signalingState: pc.signalingState,
|
||||
},
|
||||
},
|
||||
});
|
||||
if (pc.connectionState == "disconnected" ||
|
||||
pc.connectionState == "failed" ||
|
||||
(!pc.connectionState && (pc.iceConnectionState == "disconnected" || pc.iceConnectionState == "failed"))) {
|
||||
if (state == "disconnected" || state == "failed") {
|
||||
clearConnectionTimeout();
|
||||
clearReconnecting();
|
||||
if (pc.connectionState) {
|
||||
pc.removeEventListener("connectionstatechange", connectionStateChange);
|
||||
}
|
||||
@@ -251,8 +277,10 @@ const processCommand = (function () {
|
||||
}
|
||||
endCall();
|
||||
}
|
||||
else if (pc.connectionState == "connected" || (!pc.connectionState && pc.iceConnectionState == "connected")) {
|
||||
else if (state == "connected") {
|
||||
clearConnectionTimeout();
|
||||
clearReconnecting();
|
||||
wasConnected = true;
|
||||
const stats = (await pc.getStats());
|
||||
for (const stat of stats.values()) {
|
||||
const { type, state } = stat;
|
||||
@@ -278,6 +306,53 @@ const processCommand = (function () {
|
||||
connectionTimeout = undefined;
|
||||
}
|
||||
}
|
||||
function clearReconnecting() {
|
||||
clearTimeout(reconnectingTimeout);
|
||||
reconnectingTimeout = undefined;
|
||||
clearTimeout(iceRestartTimeout);
|
||||
iceRestartTimeout = undefined;
|
||||
}
|
||||
// Only the party that sent the initial offer restarts ICE, so that restart offers never collide
|
||||
function isIceRestarter() {
|
||||
var _a;
|
||||
return ((_a = pc.localDescription) === null || _a === void 0 ? void 0 : _a.type) == "offer";
|
||||
}
|
||||
async function restartIce() {
|
||||
if (activeCall !== call || reconnectingTimeout === undefined)
|
||||
return;
|
||||
iceRestartTimeout = setTimeout(restartIce, iceRestartInterval);
|
||||
// the previous restart offer is not answered yet
|
||||
if (pc.signalingState != "stable")
|
||||
return;
|
||||
try {
|
||||
const iceCandidates = getIceCandidates(pc, config);
|
||||
const offer = await pc.createOffer({ iceRestart: true });
|
||||
await pc.setLocalDescription(offer);
|
||||
sendMessageToNative({ resp: { type: "ice", session: serialize(offer), iceCandidates: await iceCandidates } });
|
||||
}
|
||||
catch (e) {
|
||||
console.log("ICE restart error", e);
|
||||
}
|
||||
}
|
||||
async function receiveRestartSession(session, iceCandidates) {
|
||||
const desc = parse(session);
|
||||
const remoteDesc = new RTCSessionDescription(!webView69Or70() ? desc : adaptSdpToOldWebView(desc));
|
||||
if (desc.type == "offer" && !isIceRestarter()) {
|
||||
await pc.setRemoteDescription(remoteDesc);
|
||||
const localIceCandidates = getIceCandidates(pc, config);
|
||||
const answer = await pc.createAnswer();
|
||||
await pc.setLocalDescription(answer);
|
||||
addIceCandidates(pc, iceCandidates);
|
||||
sendMessageToNative({ resp: { type: "ice", session: serialize(answer), iceCandidates: await localIceCandidates } });
|
||||
}
|
||||
else if (desc.type == "answer" && pc.signalingState == "have-local-offer") {
|
||||
await pc.setRemoteDescription(remoteDesc);
|
||||
addIceCandidates(pc, iceCandidates);
|
||||
}
|
||||
else {
|
||||
throw new Error(`unexpected ICE restart ${desc.type} in ${pc.signalingState} state`);
|
||||
}
|
||||
}
|
||||
}
|
||||
function serialize(x) {
|
||||
return LZString.compressToBase64(JSON.stringify(x));
|
||||
@@ -296,6 +371,8 @@ const processCommand = (function () {
|
||||
console.log("starting outgoing call - capabilities");
|
||||
if (activeCall)
|
||||
endCall();
|
||||
// Stop a preview stream from an earlier pre-connect outgoing call being replaced (activeCall may be null here)
|
||||
stopNotConnectedCall();
|
||||
let localStream = null;
|
||||
try {
|
||||
localStream = await getLocalMediaStream(true, command.media == CallMediaType.Video && (await browserHasCamera()), VideoCamera.User);
|
||||
@@ -332,7 +409,8 @@ const processCommand = (function () {
|
||||
if (activeCall)
|
||||
endCall();
|
||||
// It can be already defined on Android when switching calls (if the previous call was outgoing)
|
||||
notConnectedCall = undefined;
|
||||
// Stop its preview tracks before clearing, otherwise camera/mic stay live
|
||||
stopNotConnectedCall();
|
||||
inactiveCallMediaSources.mic = true;
|
||||
inactiveCallMediaSources.camera = command.media == CallMediaType.Video;
|
||||
inactiveCallMediaSourcesChanged(inactiveCallMediaSources);
|
||||
@@ -436,7 +514,11 @@ const processCommand = (function () {
|
||||
break;
|
||||
case "ice":
|
||||
const remoteIceCandidates = parse(command.iceCandidates);
|
||||
if (pc) {
|
||||
if (activeCall && command.session) {
|
||||
await activeCall.receiveRestartSession(command.session, remoteIceCandidates);
|
||||
resp = { type: "ok" };
|
||||
}
|
||||
else if (pc) {
|
||||
addIceCandidates(pc, remoteIceCandidates);
|
||||
resp = { type: "ok" };
|
||||
}
|
||||
@@ -1131,6 +1213,13 @@ const processCommand = (function () {
|
||||
activeCall.localScreenStream.getTracks().forEach((track) => track.stop());
|
||||
}
|
||||
}
|
||||
// Call on any path that abandons notConnectedCall, otherwise its preview camera/mic tracks stay live.
|
||||
function stopNotConnectedCall() {
|
||||
if (notConnectedCall) {
|
||||
notConnectedCall.localStream.getTracks().forEach((track) => track.stop());
|
||||
notConnectedCall = undefined;
|
||||
}
|
||||
}
|
||||
function resetVideoElements() {
|
||||
const videos = getVideoElements();
|
||||
if (!videos)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Override defaults to enable worker on Chrome and Safari
|
||||
useWorker = typeof window.Worker !== "undefined";
|
||||
isDesktop = true;
|
||||
// Create WebSocket connection.
|
||||
// Create WebSocket connection. location.search carries the per-call ?token=... capability required by the server.
|
||||
const socket = new WebSocket(`ws://${location.host}${location.search}`);
|
||||
socket.addEventListener("open", (_event) => {
|
||||
console.log("Opened socket");
|
||||
@@ -192,4 +192,4 @@ function updateCallInfoView(state, description) {
|
||||
document.getElementById("state").innerText = state;
|
||||
document.getElementById("description").innerText = description;
|
||||
}
|
||||
//# sourceMappingURL=ui.js.map
|
||||
//# sourceMappingURL=ui.js.map
|
||||
+3
-1
@@ -313,7 +313,9 @@ actual object CallSoundsPlayer: CallSoundsPlayerInterface {
|
||||
override fun startConnectingCallSound(scope: CoroutineScope) {
|
||||
// Taken from https://github.com/TelegramOrg/Telegram-Android
|
||||
// https://github.com/TelegramOrg/Telegram-Android/blob/master/LICENSE
|
||||
start("/media/connecting_call.mp3", 3000, scope)
|
||||
// the delay is the length of the track exactly, so it repeats with no delay, which already happens on ios and android
|
||||
// Android (isLooping) and iOS (which sleeps for the track duration)
|
||||
start("/media/connecting_call.mp3", 3536, scope)
|
||||
}
|
||||
|
||||
override fun startInCallSound(scope: CoroutineScope) {
|
||||
|
||||
+27
-16
@@ -8,6 +8,7 @@ import chat.simplex.common.platform.*
|
||||
import chat.simplex.common.views.helpers.*
|
||||
import chat.simplex.res.*
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.flow.*
|
||||
import kotlinx.datetime.Clock
|
||||
import kotlinx.serialization.encodeToString
|
||||
@@ -55,13 +56,24 @@ actual fun ActiveCallView() {
|
||||
CallSoundsPlayer.stop()
|
||||
}
|
||||
is WCallResponse.Ice -> withBGApi {
|
||||
chatModel.controller.apiSendCallExtraInfo(callRh, call.contact, r.iceCandidates)
|
||||
chatModel.controller.apiSendCallExtraInfo(callRh, call.contact, r.session, r.iceCandidates)
|
||||
}
|
||||
is WCallResponse.Connection ->
|
||||
try {
|
||||
// the call is restored and NOT sent to the core
|
||||
// Makes sure the core DOES NOT record the duration wrong if a reconnect happens
|
||||
if (r.state.connectionState == "reconnecting") {
|
||||
if (call.callState != CallState.Reconnecting) {
|
||||
CallSoundsPlayer.startConnectingCallSound(scope)
|
||||
}
|
||||
chatModel.activeCall.value = call.copy(callState = CallState.Reconnecting)
|
||||
} else try {
|
||||
val callStatus = json.decodeFromString<WebRTCCallStatus>("\"${r.state.connectionState}\"")
|
||||
if (callStatus == WebRTCCallStatus.Connected) {
|
||||
chatModel.activeCall.value = call.copy(callState = CallState.Connected, connectedAt = Clock.System.now())
|
||||
if (call.callState == CallState.Reconnecting) {
|
||||
CallSoundsPlayer.stop()
|
||||
}
|
||||
// connectedAt is only set ONCE, this way call duration is NOT reset when the call is restored from reconnecting
|
||||
chatModel.activeCall.value = call.copy(callState = CallState.Connected, connectedAt = call.connectedAt ?: Clock.System.now())
|
||||
}
|
||||
withBGApi { chatModel.controller.apiCallStatus(callRh, call.contact, callStatus) }
|
||||
} catch (e: Throwable) {
|
||||
@@ -194,21 +206,20 @@ fun WebRTCController(callCommand: SnapshotStateList<WCallCommand>, onResponse: (
|
||||
}
|
||||
}
|
||||
LaunchedEffect(Unit) {
|
||||
snapshotFlow { callCommand.firstOrNull() }
|
||||
.distinctUntilChanged()
|
||||
.filterNotNull()
|
||||
.collect {
|
||||
while (connections.isEmpty()) {
|
||||
delay(100)
|
||||
}
|
||||
while (callCommand.isNotEmpty()) {
|
||||
val cmd = callCommand.removeFirstOrNull()
|
||||
Log.d(TAG, "WebRTCController LaunchedEffect executing ${cmd?.javaClass?.simpleName}")
|
||||
if (cmd != null) {
|
||||
processCommand(cmd)
|
||||
}
|
||||
while (isActive) {
|
||||
// waiting for the first command would miss a command equal to the previous one
|
||||
snapshotFlow { callCommand.isNotEmpty() }.first { it }
|
||||
while (connections.isEmpty()) {
|
||||
delay(100)
|
||||
}
|
||||
while (callCommand.isNotEmpty()) {
|
||||
val cmd = callCommand.removeFirstOrNull()
|
||||
Log.d(TAG, "WebRTCController LaunchedEffect executing ${cmd?.javaClass?.simpleName}")
|
||||
if (cmd != null) {
|
||||
processCommand(cmd)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -99,6 +99,7 @@ Both platforms share a common signaling flow through the Haskell core API.
|
||||
```
|
||||
Outgoing: WaitCapabilities -> InvitationSent -> OfferSent -> AnswerReceived -> Negotiated -> Connected -> Ended
|
||||
Incoming: InvitationAccepted -> OfferReceived -> Negotiated -> Connected -> Ended
|
||||
Reconnection: Connected -> Reconnecting -> Connected | Ended
|
||||
```
|
||||
|
||||
| State | Description |
|
||||
@@ -111,8 +112,17 @@ Incoming: InvitationAccepted -> OfferReceived -> Negotiated -> Connected -> Ende
|
||||
| `AnswerReceived` | Caller received SDP answer |
|
||||
| `Negotiated` | ICE negotiation complete |
|
||||
| `Connected` | Media flowing |
|
||||
| `Reconnecting` | Connection lost, waiting for ICE restart |
|
||||
| `Ended` | Call terminated |
|
||||
|
||||
### 3.1 Reconnection
|
||||
|
||||
1. A connected call loses its WebRTC connection, e.g. when the device switches between Wi-Fi and mobile network.
|
||||
2. The WebRTC engine reports `"reconnecting"` connection state; the app shows "reconnecting…" and plays the connecting sound. This state is not reported to the core, and `connectedAt` is kept, so call duration continues.
|
||||
3. The callee's WebRTC engine (the party that sent the initial offer) restarts ICE every 10 seconds, sending the new offer in `WCallResponse.Ice` with `session` via `apiSendCallExtraInfo`. The caller answers the same way.
|
||||
4. When the connection is restored, the call returns to `Connected` and the sound stops.
|
||||
5. If the connection is not restored in 30 seconds, the call ends as disconnected. Clients without ICE restart support ignore the restart offer, so the call ends in this case.
|
||||
|
||||
---
|
||||
|
||||
## 4. Ending a Call
|
||||
@@ -184,7 +194,7 @@ Incoming: InvitationAccepted -> OfferReceived -> Negotiated -> Connected -> Ende
|
||||
| `apiRejectCall(rh, contact)` | Reject incoming call |
|
||||
| `apiSendCallOffer(rh, contact, rtcSession, rtcIceCandidates, media, capabilities)` | Send SDP offer |
|
||||
| `apiSendCallAnswer(rh, contact, rtcSession, rtcIceCandidates)` | Send SDP answer |
|
||||
| `apiSendCallExtraInfo(rh, contact, rtcIceCandidates)` | Send additional ICE candidates |
|
||||
| `apiSendCallExtraInfo(rh, contact, rtcSession, rtcIceCandidates)` | Send additional ICE candidates, or ICE restart offer/answer |
|
||||
| `apiEndCall(rh, contact)` | End active call |
|
||||
| `apiCallStatus(rh, contact, status)` | Report WebRTC connection status |
|
||||
|
||||
|
||||
@@ -71,8 +71,8 @@ Sound: `SoundPlayer.start()` plays ringtone while banner is visible (unless call
|
||||
| 1. Capabilities | `WCallResponse.Capabilities` | Local capabilities reported; `apiSendCallInvitation()` called |
|
||||
| 2. Offer | `WCallResponse.Offer` | SDP offer + ICE candidates sent via `apiSendCallOffer()` |
|
||||
| 3. Answer | `WCallResponse.Answer` | SDP answer + ICE candidates sent via `apiSendCallAnswer()` |
|
||||
| 4. ICE | `WCallResponse.Ice` | Additional ICE candidates exchanged via `apiSendCallExtraInfo()` |
|
||||
| 5. Connection | `WCallResponse.Connection` | WebRTC connection state changes; `CallState.Connected` set on success |
|
||||
| 4. ICE | `WCallResponse.Ice` | Additional ICE candidates, and ICE restart offer/answer in `session`, exchanged via `apiSendCallExtraInfo()` |
|
||||
| 5. Connection | `WCallResponse.Connection` | WebRTC connection state changes; `CallState.Connected` set on success, `CallState.Reconnecting` when a connected call loses connection |
|
||||
| 6. Connected | `WCallResponse.Connected` | Connection info (relay/direct) stored in `call.connectionInfo` |
|
||||
| 7. PeerMedia | `WCallResponse.PeerMedia` | Remote party media source changes (mic, camera, screen) |
|
||||
| 8. Media control | `WCallCommand.Media` | Toggle local media sources (mic, camera, screen audio/video) |
|
||||
@@ -90,14 +90,15 @@ Sound: `SoundPlayer.start()` plays ringtone while banner is visible (unless call
|
||||
| `OfferReceived` | Callee received SDP offer |
|
||||
| `AnswerReceived` | Caller received SDP answer |
|
||||
| `Negotiated` | ICE negotiation complete |
|
||||
| `Connected` | WebRTC media flowing; `connectedAt` timestamp set |
|
||||
| `Connected` | WebRTC media flowing; `connectedAt` timestamp set on the first connection |
|
||||
| `Reconnecting` | Connection lost; "reconnecting…" shown while ICE restarts, call ends after 30 seconds without connection |
|
||||
| `Ended` | Call terminated |
|
||||
|
||||
### Call Sounds
|
||||
|
||||
| Sound | Trigger |
|
||||
|---|---|
|
||||
| Connecting sound | `CallSoundsPlayer.startConnectingCallSound()` after invitation sent |
|
||||
| Connecting sound | `CallSoundsPlayer.startConnectingCallSound()` after invitation sent, and while reconnecting |
|
||||
| In-call sound | `CallSoundsPlayer.startInCallSound()` when delivery receipt received |
|
||||
| Ringtone | `SoundPlayer.start()` for incoming calls |
|
||||
| End vibration | `CallSoundsPlayer.vibrate()` on call end (if was connected) |
|
||||
|
||||
@@ -225,7 +225,7 @@ All functions below are `suspend fun` members of `ChatController` ([SimpleXAPI.k
|
||||
| `apiRejectCall` | `rh: Long?, contact: Contact` | Reject an incoming call | [L1854](../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1854) |
|
||||
| `apiSendCallOffer` | `rh, contact, rtcSession, rtcIceCandidates, media, capabilities` | Send a WebRTC call offer | [L1859](../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1859) |
|
||||
| `apiSendCallAnswer` | `rh: Long?, contact: Contact, rtcSession: String, rtcIceCandidates: String` | Send a WebRTC call answer | [L1866](../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1866) |
|
||||
| `apiSendCallExtraInfo` | `rh: Long?, contact: Contact, rtcIceCandidates: String` | Send additional ICE candidates during a call | [L1872](../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1872) |
|
||||
| `apiSendCallExtraInfo` | `rh: Long?, contact: Contact, rtcSession: String?, rtcIceCandidates: String` | Send additional ICE candidates or ICE restart offer/answer during a call | [L2156](../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L2156) |
|
||||
| `apiEndCall` | `rh: Long?, contact: Contact` | End an active call | [L1878](../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1878) |
|
||||
| `apiCallStatus` | `rh: Long?, contact: Contact, status: WebRTCCallStatus` | Report call status updates to the core | [L1883](../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1883) |
|
||||
|
||||
|
||||
@@ -43,6 +43,7 @@ enum class CallState {
|
||||
AnswerReceived, // SDP answer received from peer
|
||||
Negotiated, // ICE negotiation in progress
|
||||
Connected, // Media flowing
|
||||
Reconnecting, // Connection lost, waiting for ICE to recover
|
||||
Ended; // Call terminated
|
||||
}
|
||||
```
|
||||
@@ -53,6 +54,8 @@ enum class CallState {
|
||||
|
||||
State transitions are driven by `WCallResponse` messages from the WebRTC layer. Each transition typically triggers a corresponding API command (e.g., `apiSendCallInvitation`, `apiSendCallOffer`).
|
||||
|
||||
**Reconnection**: `Connected` -> `Reconnecting` -> `Connected` or `Ended`. When a connected call loses its WebRTC connection, `call.js` reports `"reconnecting"` connection state instead of ending the call. The app sets `CallState.Reconnecting` and plays the connecting sound; this state is not sent to the core with `apiCallStatus`, so the call item duration is not affected, and `connectedAt` is kept. The party that sent the initial offer (callee) restarts ICE every 10 seconds until reconnected: the restart offer and the answer are sent as `WCallResponse.Ice` / `WCallCommand.Ice` with `session`, via `apiSendCallExtraInfo` (`x.call.extra` with `rtcSession`). If the connection is not restored in 30 seconds, `call.js` reports `"disconnected"` and ends the call.
|
||||
|
||||
---
|
||||
|
||||
<a id="ActiveCallView"></a>
|
||||
@@ -140,7 +143,7 @@ Defined in [`SimpleXAPI.kt`](../../common/src/commonMain/kotlin/chat/simplex/com
|
||||
| `apiRejectCall` | [L1854](../../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1854) | Reject an incoming call |
|
||||
| `apiSendCallOffer` | [L1859](../../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1859) | Send SDP offer with ICE candidates and capabilities |
|
||||
| `apiSendCallAnswer` | [L1866](../../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1866) | Send SDP answer with ICE candidates |
|
||||
| `apiSendCallExtraInfo` | [L1872](../../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1872) | Send additional ICE candidates discovered after initial exchange |
|
||||
| `apiSendCallExtraInfo` | [L2156](../../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L2156) | Send additional ICE candidates discovered after initial exchange, or ICE restart offer/answer in `rtcSession` |
|
||||
| `apiEndCall` | [L1878](../../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1878) | Terminate a call |
|
||||
| `apiCallStatus` | [L1883](../../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1883) | Report WebRTC connection status to the backend |
|
||||
|
||||
|
||||
Reference in new issue
Block a user