mirror of
https://github.com/simplex-chat/simplex-chat.git
synced 2026-10-20 21:02:23 +00:00
ui: reconnect call (#7711)
* Update Call.hs * Update Internal.hs * add reconnecting string * Update WebRTC.kt * desktop: show reconnecting state, keep call duration * android keep connection time correct wiith adding reconnecting * Update CallTypes.swift add reconnecting status on IOS * Update WebRTC.swift add reconnecting * add reconnecting and make sure call duration is correct * Update call.ts * core: regenerate call.js from call.ts * Update Call.hs revert call status change minimization * Update Internal.hs rever to minimize changes * Update CallTypes.swift minimize changes * Update WebRTC.kt minimize changes * Update CallView.desktop.kt desktop shows reconnecting without sending to core * Update CallView.android.kt show reconnecting without sending it to the core * Update ActiveCallView.swift show reconnecting to ui without sending to core * Update CallView.desktop.kt add reconnecting sound on reconnect * Update CallView.android.kt add reconnecting sound * Update ActiveCallView.swift play reconnecting sound when reconnecting * Update RecAndPlay.desktop.kt * rebuild * refactor * build * android, desktop: restart ICE to reconnect calls * ios: restart ICE to reconnect calls * webrtc: add call reconnection e2e tests * android, desktop: fix lost call commands --------- Co-authored-by: GuavaGoat <320862384+GuavaGoat@users.noreply.github.com> Co-authored-by: Evgeny Poberezkin <evgeny@poberezkin.com> Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com>
This commit is contained in:
sh
co-authored by
GuavaGoat
Evgeny Poberezkin
Evgeny @ SimpleX Chat
GuavaGoat
Evgeny Poberezkin
Evgeny @ SimpleX Chat
parent
634fa5ecff
commit
2ea1ec0d50
@@ -1850,8 +1850,8 @@ func apiSendCallAnswer(_ contact: Contact, _ rtcSession: String, _ rtcIceCandida
|
||||
try await sendCommandOkResp(.apiSendCallAnswer(contact: contact, answer: answer))
|
||||
}
|
||||
|
||||
func apiSendCallExtraInfo(_ contact: Contact, _ rtcIceCandidates: String) async throws {
|
||||
let extraInfo = WebRTCExtraInfo(rtcIceCandidates: rtcIceCandidates)
|
||||
func apiSendCallExtraInfo(_ contact: Contact, _ rtcSession: String?, _ rtcIceCandidates: String) async throws {
|
||||
let extraInfo = WebRTCExtraInfo(rtcSession: rtcSession, rtcIceCandidates: rtcIceCandidates)
|
||||
try await sendCommandOkResp(.apiSendCallExtraInfo(contact: contact, extraInfo: extraInfo))
|
||||
}
|
||||
|
||||
@@ -2987,7 +2987,7 @@ func processReceivedMsg(_ res: ChatEvent) async {
|
||||
}
|
||||
case let .callExtraInfo(_, contact, extraInfo):
|
||||
await withCall(contact) { _ in
|
||||
await m.callCommand.processCommand(.ice(iceCandidates: extraInfo.rtcIceCandidates))
|
||||
await m.callCommand.processCommand(.ice(iceCandidates: extraInfo.rtcIceCandidates, session: extraInfo.rtcSession))
|
||||
}
|
||||
case let .callEnded(_, contact):
|
||||
if let invitation = await MainActor.run(body: { m.callInvitations.removeValue(forKey: contact.id) }) {
|
||||
|
||||
@@ -161,10 +161,10 @@ struct ActiveCallView: View {
|
||||
CallSoundsPlayer.shared.stop()
|
||||
}
|
||||
}
|
||||
case let .ice(iceCandidates):
|
||||
case let .ice(iceCandidates, session):
|
||||
Task {
|
||||
do {
|
||||
try await apiSendCallExtraInfo(call.contact, iceCandidates)
|
||||
try await apiSendCallExtraInfo(call.contact, session, iceCandidates)
|
||||
} catch {
|
||||
logger.error("apiSendCallExtraInfo \(responseError(error))")
|
||||
}
|
||||
@@ -172,15 +172,24 @@ struct ActiveCallView: View {
|
||||
case let .connection(state):
|
||||
if let callStatus = WebRTCCallStatus.init(rawValue: state.connectionState),
|
||||
case .connected = callStatus {
|
||||
call.direction == .outgoing
|
||||
? CallController.shared.reportOutgoingCall(call: call, connectedAt: nil)
|
||||
: CallController.shared.reportIncomingCall(call: call, connectedAt: nil)
|
||||
// connectedAt is kept on reconnection, so that call duration is not reset
|
||||
if call.connectedAt == nil {
|
||||
call.direction == .outgoing
|
||||
? CallController.shared.reportOutgoingCall(call: call, connectedAt: nil)
|
||||
: CallController.shared.reportIncomingCall(call: call, connectedAt: nil)
|
||||
call.connectedAt = .now
|
||||
}
|
||||
call.callState = .connected
|
||||
call.connectedAt = .now
|
||||
CallSoundsPlayer.shared.stop()
|
||||
if !wasConnected {
|
||||
CallSoundsPlayer.shared.vibrate(long: false)
|
||||
wasConnected = true
|
||||
}
|
||||
} else if state.connectionState == "reconnecting" {
|
||||
if call.callState != .reconnecting {
|
||||
CallSoundsPlayer.shared.startConnectingCallSound()
|
||||
}
|
||||
call.callState = .reconnecting
|
||||
}
|
||||
if state.connectionState == "closed" {
|
||||
closeCallView(client)
|
||||
@@ -190,17 +199,22 @@ struct ActiveCallView: View {
|
||||
m.activeCall = nil
|
||||
m.activeCallViewIsCollapsed = false
|
||||
}
|
||||
Task {
|
||||
do {
|
||||
try await apiCallStatus(call.contact, state.connectionState)
|
||||
} catch {
|
||||
logger.error("apiCallStatus \(responseError(error))")
|
||||
// "reconnecting" is not a core call status
|
||||
if state.connectionState != "reconnecting" {
|
||||
Task {
|
||||
do {
|
||||
try await apiCallStatus(call.contact, state.connectionState)
|
||||
} catch {
|
||||
logger.error("apiCallStatus \(responseError(error))")
|
||||
}
|
||||
}
|
||||
}
|
||||
case let .connected(connectionInfo):
|
||||
call.callState = .connected
|
||||
call.connectionInfo = connectionInfo
|
||||
call.connectedAt = .now
|
||||
if call.connectedAt == nil {
|
||||
call.connectedAt = .now
|
||||
}
|
||||
if !wasConnected {
|
||||
CallSoundsPlayer.shared.vibrate(long: false)
|
||||
wasConnected = true
|
||||
|
||||
@@ -77,6 +77,7 @@ enum CallState {
|
||||
case answerReceived // incoming - webrtc answer received via API
|
||||
case negotiated // outgoing - webrtc offer processed and answer sent, incoming - webrtc answer processed
|
||||
case connected
|
||||
case reconnecting
|
||||
case ended
|
||||
|
||||
var text: LocalizedStringKey {
|
||||
@@ -89,6 +90,7 @@ enum CallState {
|
||||
case .answerReceived: return "received confirmation…"
|
||||
case .negotiated: return "connecting…"
|
||||
case .connected: return "connected"
|
||||
case .reconnecting: return "reconnecting…"
|
||||
case .ended: return "ended"
|
||||
}
|
||||
}
|
||||
@@ -119,7 +121,7 @@ enum WCallCommand: Equatable, Encodable, Decodable {
|
||||
case start(media: CallMediaType, aesKey: String? = nil, iceServers: [RTCIceServer]? = nil, relay: Bool? = nil)
|
||||
case offer(offer: String, iceCandidates: String, media: CallMediaType, aesKey: String? = nil, iceServers: [RTCIceServer]? = nil, relay: Bool? = nil)
|
||||
case answer(answer: String, iceCandidates: String)
|
||||
case ice(iceCandidates: String)
|
||||
case ice(iceCandidates: String, session: String? = nil)
|
||||
case media(source: CallMediaSource, enable: Bool)
|
||||
case end
|
||||
|
||||
@@ -131,6 +133,7 @@ enum WCallCommand: Equatable, Encodable, Decodable {
|
||||
case offer
|
||||
case answer
|
||||
case iceCandidates
|
||||
case session
|
||||
case enable
|
||||
case iceServers
|
||||
case relay
|
||||
@@ -174,9 +177,10 @@ enum WCallCommand: Equatable, Encodable, Decodable {
|
||||
try container.encode("answer", forKey: .type)
|
||||
try container.encode(answer, forKey: .answer)
|
||||
try container.encode(iceCandidates, forKey: .iceCandidates)
|
||||
case let .ice(iceCandidates):
|
||||
case let .ice(iceCandidates, session):
|
||||
try container.encode("ice", forKey: .type)
|
||||
try container.encode(iceCandidates, forKey: .iceCandidates)
|
||||
try container.encodeIfPresent(session, forKey: .session)
|
||||
case let .media(source, enable):
|
||||
try container.encode("media", forKey: .type)
|
||||
try container.encode(source, forKey: .media)
|
||||
@@ -213,7 +217,8 @@ enum WCallCommand: Equatable, Encodable, Decodable {
|
||||
self = .answer(answer: answer, iceCandidates: iceCandidates)
|
||||
case "ice":
|
||||
let iceCandidates = try container.decode(String.self, forKey: CodingKeys.iceCandidates)
|
||||
self = .ice(iceCandidates: iceCandidates)
|
||||
let session = try container.decodeIfPresent(String.self, forKey: CodingKeys.session)
|
||||
self = .ice(iceCandidates: iceCandidates, session: session)
|
||||
case "media":
|
||||
let source = try container.decode(CallMediaSource.self, forKey: CodingKeys.source)
|
||||
let enable = try container.decode(Bool.self, forKey: CodingKeys.enable)
|
||||
@@ -231,7 +236,7 @@ enum WCallResponse: Equatable, Decodable {
|
||||
case capabilities(capabilities: CallCapabilities)
|
||||
case offer(offer: String, iceCandidates: String, capabilities: CallCapabilities)
|
||||
case answer(answer: String, iceCandidates: String)
|
||||
case ice(iceCandidates: String)
|
||||
case ice(iceCandidates: String, session: String? = nil)
|
||||
case connection(state: ConnectionState)
|
||||
case connected(connectionInfo: ConnectionInfo)
|
||||
case peerMedia(source: CallMediaSource, enabled: Bool)
|
||||
@@ -246,6 +251,7 @@ enum WCallResponse: Equatable, Decodable {
|
||||
case offer
|
||||
case answer
|
||||
case iceCandidates
|
||||
case session
|
||||
case state
|
||||
case connectionInfo
|
||||
case message
|
||||
@@ -290,7 +296,8 @@ enum WCallResponse: Equatable, Decodable {
|
||||
self = .answer(answer: answer, iceCandidates: iceCandidates)
|
||||
case "ice":
|
||||
let iceCandidates = try container.decode(String.self, forKey: CodingKeys.iceCandidates)
|
||||
self = .ice(iceCandidates: iceCandidates)
|
||||
let session = try container.decodeIfPresent(String.self, forKey: CodingKeys.session)
|
||||
self = .ice(iceCandidates: iceCandidates, session: session)
|
||||
case "connection":
|
||||
let state = try container.decode(ConnectionState.self, forKey: CodingKeys.state)
|
||||
self = .connection(state: state)
|
||||
@@ -333,9 +340,10 @@ extension WCallResponse: Encodable {
|
||||
try container.encode("answer", forKey: .type)
|
||||
try container.encode(answer, forKey: .answer)
|
||||
try container.encode(iceCandidates, forKey: .iceCandidates)
|
||||
case let .ice(iceCandidates):
|
||||
case let .ice(iceCandidates, session):
|
||||
try container.encode("ice", forKey: .type)
|
||||
try container.encode(iceCandidates, forKey: .iceCandidates)
|
||||
try container.encodeIfPresent(session, forKey: .session)
|
||||
case let .connection(state):
|
||||
try container.encode("connection", forKey: .type)
|
||||
try container.encode(state, forKey: .state)
|
||||
|
||||
@@ -20,6 +20,8 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
|
||||
}()
|
||||
private static let ivTagBytes: Int = 28
|
||||
private static let enableEncryption: Bool = true
|
||||
private static let reconnectTimeout: UInt64 = 30_000_000_000
|
||||
private static let iceRestartInterval: UInt64 = 10_000_000_000
|
||||
private var chat_ctrl = getChatCtrl()
|
||||
|
||||
struct Call {
|
||||
@@ -37,6 +39,9 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
|
||||
var frameEncryptor: RTCFrameEncryptor?
|
||||
var frameDecryptor: RTCFrameDecryptor?
|
||||
var peerHasOldVersion: Bool
|
||||
var wasConnected = false
|
||||
var reconnection: Task<Void, Never>? = nil
|
||||
var iceRestarts: Task<Void, Never>? = nil
|
||||
}
|
||||
|
||||
struct NotConnectedCall {
|
||||
@@ -257,11 +262,15 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
|
||||
resp = .error(message: "answer: remote description is not set")
|
||||
}
|
||||
}
|
||||
case let .ice(iceCandidates):
|
||||
case let .ice(iceCandidates, session):
|
||||
if let pc = pc,
|
||||
let remoteIceCandidates: [RTCIceCandidate] = decodeJSON(decompressFromBase64(input: iceCandidates)) {
|
||||
addIceCandidates(pc, remoteIceCandidates)
|
||||
resp = .ok
|
||||
if let session {
|
||||
resp = await receiveRestartSession(pc, session, remoteIceCandidates)
|
||||
} else {
|
||||
addIceCandidates(pc, remoteIceCandidates)
|
||||
resp = .ok
|
||||
}
|
||||
} else {
|
||||
resp = .error(message: "ice: call not started")
|
||||
}
|
||||
@@ -301,6 +310,101 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
|
||||
}
|
||||
}
|
||||
|
||||
private func startReconnecting(_ connection: RTCPeerConnection) async {
|
||||
activeCall?.reconnection = Task {
|
||||
_ = try? await Task.sleep(nanoseconds: WebRTCClient.reconnectTimeout)
|
||||
if Task.isCancelled { return }
|
||||
await sendConnectionState(connection, "disconnected")
|
||||
endCall()
|
||||
}
|
||||
// Only the party that sent the initial offer restarts ICE, so that restart offers never collide
|
||||
if connection.localDescription?.type == .offer {
|
||||
activeCall?.iceRestarts = Task {
|
||||
while !Task.isCancelled {
|
||||
await restartIce(connection)
|
||||
_ = try? await Task.sleep(nanoseconds: WebRTCClient.iceRestartInterval)
|
||||
}
|
||||
}
|
||||
}
|
||||
await sendConnectionState(connection, "reconnecting")
|
||||
}
|
||||
|
||||
private func stopReconnecting() {
|
||||
activeCall?.reconnection?.cancel()
|
||||
activeCall?.reconnection = nil
|
||||
activeCall?.iceRestarts?.cancel()
|
||||
activeCall?.iceRestarts = nil
|
||||
}
|
||||
|
||||
private func restartIce(_ connection: RTCPeerConnection) async {
|
||||
// the previous restart offer is not answered yet
|
||||
if connection.signalingState != .stable { return }
|
||||
let (offer, error) = await connection.offer(iceRestart: true)
|
||||
if let offer {
|
||||
await sendRestartSession(offer)
|
||||
} else {
|
||||
logger.error("WebRTCClient: ICE restart error: \(error?.localizedDescription ?? "unknown error")")
|
||||
}
|
||||
}
|
||||
|
||||
private func receiveRestartSession(_ pc: RTCPeerConnection, _ session: String, _ remoteIceCandidates: [RTCIceCandidate]) async -> WCallResponse {
|
||||
guard let desc: CustomRTCSessionDescription = decodeJSON(decompressFromBase64(input: session)),
|
||||
let type = desc.type, let sdp = desc.sdp else {
|
||||
return .error(message: "ice: invalid session")
|
||||
}
|
||||
let remoteDesc = RTCSessionDescription(type: type.toWebRTCSdpType(), sdp: sdp)
|
||||
if type == .offer && pc.localDescription?.type != .offer {
|
||||
if (try? await pc.setRemoteDescription(remoteDesc)) == nil {
|
||||
return .error(message: "ice: remote offer is not set")
|
||||
}
|
||||
let (answer, error) = await pc.answer()
|
||||
guard let answer else {
|
||||
return .error(message: "ice: answer error: \(error?.localizedDescription ?? "unknown error")")
|
||||
}
|
||||
addIceCandidates(pc, remoteIceCandidates)
|
||||
await sendRestartSession(answer)
|
||||
} else if type == .answer && pc.signalingState == .haveLocalOffer {
|
||||
if (try? await pc.setRemoteDescription(remoteDesc)) == nil {
|
||||
return .error(message: "ice: remote answer is not set")
|
||||
}
|
||||
addIceCandidates(pc, remoteIceCandidates)
|
||||
} else {
|
||||
return .error(message: "ice: unexpected ICE restart \(type.rawValue)")
|
||||
}
|
||||
return .ok
|
||||
}
|
||||
|
||||
private func sendRestartSession(_ desc: RTCSessionDescription) async {
|
||||
await sendCallResponse(.init(
|
||||
corrId: nil,
|
||||
resp: .ice(
|
||||
iceCandidates: compressToBase64(input: encodeJSON(await getInitialIceCandidates())),
|
||||
session: compressToBase64(input: encodeJSON(CustomRTCSessionDescription(type: desc.type.toSdpType(), sdp: desc.sdp)))
|
||||
),
|
||||
command: nil)
|
||||
)
|
||||
waitForMoreIceCandidates()
|
||||
}
|
||||
|
||||
private func sendConnectionState(_ connection: RTCPeerConnection, _ connectionState: String) async {
|
||||
guard let iceConnectionState = connection.iceConnectionState.toString(),
|
||||
let iceGatheringState = connection.iceGatheringState.toString(),
|
||||
let signalingState = connection.signalingState.toString()
|
||||
else {
|
||||
return
|
||||
}
|
||||
await sendCallResponse(.init(
|
||||
corrId: nil,
|
||||
resp: .connection(state: ConnectionState(
|
||||
connectionState: connectionState,
|
||||
iceConnectionState: iceConnectionState,
|
||||
iceGatheringState: iceGatheringState,
|
||||
signalingState: signalingState)
|
||||
),
|
||||
command: nil)
|
||||
)
|
||||
}
|
||||
|
||||
// Spec: spec/services/calls.md#sendIceCandidates
|
||||
func sendIceCandidates(_ candidates: [RTCIceCandidate]) async {
|
||||
await self.sendCallResponse(.init(
|
||||
@@ -657,6 +761,8 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
|
||||
(notConnectedCall?.localCameraAndTrack?.0 as? RTCCameraVideoCapturer)?.stopCapture()
|
||||
guard let call = activeCall else { return }
|
||||
logger.debug("WebRTCClient: ending the call")
|
||||
call.reconnection?.cancel()
|
||||
call.iceRestarts?.cancel()
|
||||
call.connection.close()
|
||||
call.connection.delegate = nil
|
||||
call.frameEncryptor?.delegate = nil
|
||||
@@ -678,16 +784,18 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
|
||||
|
||||
|
||||
extension WebRTC.RTCPeerConnection {
|
||||
func mediaConstraints() -> RTCMediaConstraints {
|
||||
RTCMediaConstraints(
|
||||
mandatoryConstraints: [kRTCMediaConstraintsOfferToReceiveAudio: kRTCMediaConstraintsValueTrue,
|
||||
kRTCMediaConstraintsOfferToReceiveVideo: kRTCMediaConstraintsValueTrue],
|
||||
optionalConstraints: nil)
|
||||
func mediaConstraints(iceRestart: Bool = false) -> RTCMediaConstraints {
|
||||
var constraints = [kRTCMediaConstraintsOfferToReceiveAudio: kRTCMediaConstraintsValueTrue,
|
||||
kRTCMediaConstraintsOfferToReceiveVideo: kRTCMediaConstraintsValueTrue]
|
||||
if iceRestart {
|
||||
constraints[kRTCMediaConstraintsIceRestart] = kRTCMediaConstraintsValueTrue
|
||||
}
|
||||
return RTCMediaConstraints(mandatoryConstraints: constraints, optionalConstraints: nil)
|
||||
}
|
||||
|
||||
func offer() async -> (RTCSessionDescription?, Error?) {
|
||||
func offer(iceRestart: Bool = false) async -> (RTCSessionDescription?, Error?) {
|
||||
await withCheckedContinuation { cont in
|
||||
offer(for: mediaConstraints()) { (sdp, error) in
|
||||
offer(for: mediaConstraints(iceRestart: iceRestart)) { (sdp, error) in
|
||||
self.processSDP(cont, sdp, error)
|
||||
}
|
||||
}
|
||||
@@ -766,24 +874,20 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
|
||||
func peerConnection(_ connection: RTCPeerConnection, didChange newState: RTCIceConnectionState) {
|
||||
debugPrint("Connection new connection state: \(newState.toString() ?? "" + newState.rawValue.description) \(connection.receivers)")
|
||||
|
||||
guard let connectionStateString = newState.toString(),
|
||||
let iceConnectionStateString = connection.iceConnectionState.toString(),
|
||||
let iceGatheringStateString = connection.iceGatheringState.toString(),
|
||||
let signalingStateString = connection.signalingState.toString()
|
||||
else {
|
||||
return
|
||||
}
|
||||
guard let connectionStateString = newState.toString() else { return }
|
||||
Task {
|
||||
await sendCallResponse(.init(
|
||||
corrId: nil,
|
||||
resp: .connection(state: ConnectionState(
|
||||
connectionState: connectionStateString,
|
||||
iceConnectionState: iceConnectionStateString,
|
||||
iceGatheringState: iceGatheringStateString,
|
||||
signalingState: signalingStateString)
|
||||
),
|
||||
command: nil)
|
||||
)
|
||||
if activeCall?.wasConnected == true {
|
||||
switch newState {
|
||||
case .disconnected, .failed:
|
||||
if activeCall?.reconnection == nil {
|
||||
await startReconnecting(connection)
|
||||
}
|
||||
return
|
||||
case .new, .checking: return
|
||||
default: ()
|
||||
}
|
||||
}
|
||||
await sendConnectionState(connection, connectionStateString)
|
||||
|
||||
switch newState {
|
||||
case .checking:
|
||||
@@ -792,7 +896,10 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
|
||||
}
|
||||
let enableSpeaker: Bool = ChatModel.shared.activeCall?.localMediaSources.hasVideo == true
|
||||
setSpeakerEnabledAndConfigureSession(enableSpeaker)
|
||||
case .connected: sendConnectedEvent(connection)
|
||||
case .connected:
|
||||
stopReconnecting()
|
||||
activeCall?.wasConnected = true
|
||||
sendConnectedEvent(connection)
|
||||
case .disconnected, .failed: endCall()
|
||||
default: ()
|
||||
}
|
||||
|
||||
@@ -34,10 +34,12 @@ public struct WebRTCSession: Codable {
|
||||
|
||||
// Spec: spec/services/calls.md#WebRTCExtraInfo
|
||||
public struct WebRTCExtraInfo: Codable {
|
||||
public init(rtcIceCandidates: String) {
|
||||
public init(rtcSession: String?, rtcIceCandidates: String) {
|
||||
self.rtcSession = rtcSession
|
||||
self.rtcIceCandidates = rtcIceCandidates
|
||||
}
|
||||
|
||||
public var rtcSession: String?
|
||||
public var rtcIceCandidates: String
|
||||
}
|
||||
|
||||
|
||||
@@ -103,6 +103,14 @@ All signaling messages are exchanged via E2E encrypted SMP messages (no central
|
||||
5. `CallViewRenderers` manages WebRTC video rendering surfaces.
|
||||
6. Call status updates are sent via `apiCallStatus(contact:status:)`.
|
||||
|
||||
### 6.1 Reconnection
|
||||
|
||||
1. A connected call loses its connection, e.g. when the device switches between Wi-Fi and mobile network.
|
||||
2. The call shows "reconnecting…" and plays the connecting sound. Call duration continues, as this state is not reported to the core.
|
||||
3. The callee (the party that sent the initial offer) restarts ICE every 10 seconds, sending the new offer via `apiSendCallExtraInfo` with `rtcSession`; the caller answers the same way.
|
||||
4. When the connection is restored, the call returns to connected state.
|
||||
5. If the connection is not restored in 30 seconds, the call ends. Clients without ICE restart support ignore the restart offer, so the call ends in this case.
|
||||
|
||||
### 7. Audio Routing
|
||||
|
||||
1. `CallAudioDeviceManager` handles audio device selection.
|
||||
@@ -141,7 +149,7 @@ All signaling messages are exchanged via E2E encrypted SMP messages (no central
|
||||
| `RcvCallInvitation` | `SimpleXChat/CallTypes.swift` | Incoming call: user, contact, callType, sharedKey, callUUID, callTs |
|
||||
| `WebRTCCallOffer` | `SimpleXChat/CallTypes.swift` | SDP offer with call type and WebRTC session data |
|
||||
| `WebRTCSession` | `SimpleXChat/CallTypes.swift` | `rtcSession` (SDP) and `rtcIceCandidates` (serialized) |
|
||||
| `WebRTCExtraInfo` | `SimpleXChat/CallTypes.swift` | Additional ICE candidates sent after initial offer/answer |
|
||||
| `WebRTCExtraInfo` | `SimpleXChat/CallTypes.swift` | Additional ICE candidates sent after initial offer/answer, and ICE restart offer/answer |
|
||||
| `WebRTCCallStatus` | `SimpleXChat/CallTypes.swift` | Call lifecycle states for status reporting |
|
||||
| `CallMediaSource` | `SimpleXChat/CallTypes.swift` | `.mic`, `.camera`, `.screenAudio`, `.screenVideo`, `.unknown` |
|
||||
| `VideoCamera` | `SimpleXChat/CallTypes.swift` | `.user` (front) or `.environment` (rear) camera |
|
||||
@@ -152,7 +160,7 @@ All signaling messages are exchanged via E2E encrypted SMP messages (no central
|
||||
|-------|-------|----------|
|
||||
| Chat not ready on CallKit answer | App suspended, slow startup | `waitUntilChatStarted` with 30s timeout; `action.fail()` on timeout |
|
||||
| Call invitation not found | Race condition between notification and event processing | `justRefreshCallInvitations()` retry |
|
||||
| WebRTC peer connection failure | NAT traversal, network issues | Call ends with error status |
|
||||
| WebRTC peer connection failure | NAT traversal, network issues | Before connection: call ends with error status; after connection: reconnection with ICE restart, call ends if not reconnected in 30 seconds |
|
||||
| CallKit action fail | Internal state mismatch | `action.fail()` called, call cleaned up |
|
||||
| No camera/mic permission | User denied permissions | Permission request dialog shown |
|
||||
|
||||
|
||||
@@ -179,7 +179,7 @@ The `ChatCommand` enum ([`AppAPITypes.swift` L15](../Shared/Model/AppAPITypes.sw
|
||||
| `apiRejectCall` | `contact` | Reject incoming call | [L158](../Shared/Model/AppAPITypes.swift#L158) |
|
||||
| `apiSendCallOffer` | `contact, callOffer: WebRTCCallOffer` | Send SDP offer | [L159](../Shared/Model/AppAPITypes.swift#L159) |
|
||||
| `apiSendCallAnswer` | `contact, answer: WebRTCSession` | Send SDP answer | [L160](../Shared/Model/AppAPITypes.swift#L160) |
|
||||
| `apiSendCallExtraInfo` | `contact, extraInfo: WebRTCExtraInfo` | Send ICE candidates | [L161](../Shared/Model/AppAPITypes.swift#L161) |
|
||||
| `apiSendCallExtraInfo` | `contact, extraInfo: WebRTCExtraInfo` | Send ICE candidates or ICE restart offer/answer | [L171](../Shared/Model/AppAPITypes.swift#L171) |
|
||||
| `apiEndCall` | `contact` | End active call | [L162](../Shared/Model/AppAPITypes.swift#L162) |
|
||||
| `apiGetCallInvitations` | -- | Get pending call invitations | [L163](../Shared/Model/AppAPITypes.swift#L163) |
|
||||
| `apiCallStatus` | `contact, callStatus` | Report call status change | [L164](../Shared/Model/AppAPITypes.swift#L164) |
|
||||
|
||||
@@ -116,7 +116,10 @@ Manages the WebRTC peer connection, media streams, and data channels.
|
||||
| [`sendCallCommand`](../../Shared/Views/Call/WebRTCClient.swift#L176) | Dispatches WCallCommand (offer/answer/ICE) | L176 |
|
||||
| [`addIceCandidates`](../../Shared/Views/Call/WebRTCClient.swift#L165) | `peerConnection.add(RTCIceCandidate)` | L165 |
|
||||
| [`getInitialIceCandidates`](../../Shared/Views/Call/WebRTCClient.swift#L285) | Collects initial ICE candidates | L285 |
|
||||
| [`sendIceCandidates`](../../Shared/Views/Call/WebRTCClient.swift#L305) | Sends gathered ICE candidates | L305 |
|
||||
| [`sendIceCandidates`](../../Shared/Views/Call/WebRTCClient.swift#L409) | Sends gathered ICE candidates | L409 |
|
||||
| [`startReconnecting`](../../Shared/Views/Call/WebRTCClient.swift#L313) | Reports "reconnecting", starts reconnection timeout and, for the party that sent the initial offer, periodic ICE restarts | L313 |
|
||||
| [`restartIce`](../../Shared/Views/Call/WebRTCClient.swift#L339) | Creates ICE restart offer and sends it with candidates in `.ice(iceCandidates:session:)` | L339 |
|
||||
| [`receiveRestartSession`](../../Shared/Views/Call/WebRTCClient.swift#L350) | Applies ICE restart offer (and sends the answer) or answer received in `.ice` command | L350 |
|
||||
| [`enableMedia`](../../Shared/Views/Call/WebRTCClient.swift#L365) | Enable/disable audio or video track | L365 |
|
||||
| [`setupLocalTracks`](../../Shared/Views/Call/WebRTCClient.swift#L423) | Creates audio/video tracks and adds to connection | L423 |
|
||||
| [`startCaptureLocalVideo`](../../Shared/Views/Call/WebRTCClient.swift#L581) | Front/back camera toggle and capture start | L581 |
|
||||
@@ -179,6 +182,10 @@ All call signaling travels through the same encrypted SMP message channels used
|
||||
5. P2P connection established
|
||||
```
|
||||
|
||||
### Reconnection
|
||||
|
||||
When a connected call loses ICE connection (`.disconnected` or `.failed`), `WebRTCClient` reports `"reconnecting"` connection state instead of ending the call. `ActiveCallView` sets `.reconnecting` call state and plays the connecting sound; this state is not sent to the core via `apiCallStatus`, and `connectedAt` is kept. The party that sent the initial offer (callee) restarts ICE every 10 seconds: the restart offer and answer are sent via `apiSendCallExtraInfo` with `rtcSession` (`x.call.extra`). When ICE connects again, the call returns to `.connected`. If it does not reconnect in 30 seconds, `WebRTCClient` reports `"disconnected"` and ends the call.
|
||||
|
||||
### API Commands
|
||||
|
||||
| Command | Direction | Purpose |
|
||||
@@ -187,7 +194,7 @@ All call signaling travels through the same encrypted SMP message channels used
|
||||
| `apiRejectCall(contact:)` | Callee -> Caller | Reject call |
|
||||
| `apiSendCallOffer(contact:, callOffer:)` | Callee -> Caller | Send SDP offer |
|
||||
| `apiSendCallAnswer(contact:, answer:)` | Caller -> Callee | Send SDP answer |
|
||||
| `apiSendCallExtraInfo(contact:, extraInfo:)` | Both | Send ICE candidates |
|
||||
| `apiSendCallExtraInfo(contact:, extraInfo:)` | Both | Send ICE candidates, or ICE restart offer/answer in `rtcSession` |
|
||||
| `apiEndCall(contact:)` | Either | End call |
|
||||
| `apiGetCallInvitations` | -- | Get pending invitations |
|
||||
| `apiCallStatus(contact:, callStatus:)` | -- | Report status change |
|
||||
@@ -317,10 +324,11 @@ struct WebRTCSession {
|
||||
}
|
||||
```
|
||||
|
||||
### [WebRTCExtraInfo](../../SimpleXChat/CallTypes.swift#L36-L42)
|
||||
### [WebRTCExtraInfo](../../SimpleXChat/CallTypes.swift#L36-L45)
|
||||
|
||||
```swift
|
||||
struct WebRTCExtraInfo {
|
||||
var rtcSession: String? // ICE restart offer or answer
|
||||
var rtcIceCandidates: String // Additional ICE candidates
|
||||
}
|
||||
```
|
||||
@@ -330,7 +338,7 @@ struct WebRTCExtraInfo {
|
||||
Stored in `ChatModel.activeCall`:
|
||||
- Contact reference
|
||||
- Call UUID
|
||||
- Call state (enum: `.waitCapabilities`, `.invitationAccepted`, `.offerSent`, `.answerReceived`, `.connected`, etc.)
|
||||
- Call state (enum: `.waitCapabilities`, `.invitationAccepted`, `.offerSent`, `.answerReceived`, `.connected`, `.reconnecting`, etc.)
|
||||
- Media type
|
||||
- WebRTCClient reference
|
||||
|
||||
|
||||
+24
-15
@@ -52,8 +52,7 @@ import dev.icerock.moko.resources.StringResource
|
||||
import dev.icerock.moko.resources.compose.painterResource
|
||||
import dev.icerock.moko.resources.compose.stringResource
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.flow.distinctUntilChanged
|
||||
import kotlinx.coroutines.flow.filterNotNull
|
||||
import kotlinx.coroutines.flow.first
|
||||
import kotlinx.datetime.Clock
|
||||
import kotlinx.serialization.encodeToString
|
||||
import java.io.Closeable
|
||||
@@ -168,13 +167,24 @@ actual fun ActiveCallView() {
|
||||
CallSoundsPlayer.stop()
|
||||
}
|
||||
is WCallResponse.Ice -> withBGApi {
|
||||
chatModel.controller.apiSendCallExtraInfo(callRh, call.contact, r.iceCandidates)
|
||||
chatModel.controller.apiSendCallExtraInfo(callRh, call.contact, r.session, r.iceCandidates)
|
||||
}
|
||||
is WCallResponse.Connection ->
|
||||
try {
|
||||
// the call is restored and NOT sent to the core
|
||||
// Makes sure the core DOES NOT record the duration wrong if a reconnect happens
|
||||
if (r.state.connectionState == "reconnecting") {
|
||||
if (call.callState != CallState.Reconnecting) {
|
||||
CallSoundsPlayer.startConnectingCallSound(scope)
|
||||
}
|
||||
updateActiveCall(call) { it.copy(callState = CallState.Reconnecting) }
|
||||
} else try {
|
||||
val callStatus = json.decodeFromString<WebRTCCallStatus>("\"${r.state.connectionState}\"")
|
||||
if (callStatus == WebRTCCallStatus.Connected) {
|
||||
updateActiveCall(call) { it.copy(callState = CallState.Connected, connectedAt = Clock.System.now()) }
|
||||
if (call.callState == CallState.Reconnecting) {
|
||||
CallSoundsPlayer.stop()
|
||||
}
|
||||
// connectedAt is only set ONCE, this way call duration is NOT reset when the call is restored from reconnecting
|
||||
updateActiveCall(call) { it.copy(callState = CallState.Connected, connectedAt = it.connectedAt ?: Clock.System.now()) }
|
||||
}
|
||||
withBGApi { chatModel.controller.apiCallStatus(callRh, call.contact, callStatus) }
|
||||
} catch (e: Throwable) {
|
||||
@@ -705,18 +715,17 @@ fun WebRTCView(callCommand: SnapshotStateList<WCallCommand>, onResponse: (WVAPIM
|
||||
val wv = webView.value
|
||||
if (wv != null) {
|
||||
LaunchedEffect(Unit) {
|
||||
snapshotFlow { callCommand.firstOrNull() }
|
||||
.distinctUntilChanged()
|
||||
.filterNotNull()
|
||||
.collect {
|
||||
while (callCommand.isNotEmpty()) {
|
||||
val cmd = callCommand.removeFirstOrNull()
|
||||
Log.d(TAG, "WebRTCView LaunchedEffect executing ${cmd?.javaClass?.simpleName}")
|
||||
if (cmd != null) {
|
||||
processCommand(wv, cmd)
|
||||
}
|
||||
while (isActive) {
|
||||
// waiting for the first command would miss a command equal to the previous one
|
||||
snapshotFlow { callCommand.isNotEmpty() }.first { it }
|
||||
while (callCommand.isNotEmpty()) {
|
||||
val cmd = callCommand.removeFirstOrNull()
|
||||
Log.d(TAG, "WebRTCView LaunchedEffect executing ${cmd?.javaClass?.simpleName}")
|
||||
if (cmd != null) {
|
||||
processCommand(wv, cmd)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
val assetLoader = WebViewAssetLoader.Builder()
|
||||
|
||||
+3
-3
@@ -2185,8 +2185,8 @@ object ChatController {
|
||||
return r.result is CR.CmdOk
|
||||
}
|
||||
|
||||
suspend fun apiSendCallExtraInfo(rh: Long?, contact: Contact, rtcIceCandidates: String): Boolean {
|
||||
val extraInfo = WebRTCExtraInfo(rtcIceCandidates)
|
||||
suspend fun apiSendCallExtraInfo(rh: Long?, contact: Contact, rtcSession: String?, rtcIceCandidates: String): Boolean {
|
||||
val extraInfo = WebRTCExtraInfo(rtcSession, rtcIceCandidates)
|
||||
val r = sendCmd(rh, CC.ApiSendCallExtraInfo(contact, extraInfo))
|
||||
return r.result is CR.CmdOk
|
||||
}
|
||||
@@ -3436,7 +3436,7 @@ object ChatController {
|
||||
}
|
||||
is CR.CallExtraInfo -> {
|
||||
withCall(r, r.contact) { _ ->
|
||||
chatModel.callCommand.add(WCallCommand.Ice(iceCandidates = r.extraInfo.rtcIceCandidates))
|
||||
chatModel.callCommand.add(WCallCommand.Ice(iceCandidates = r.extraInfo.rtcIceCandidates, session = r.extraInfo.rtcSession))
|
||||
}
|
||||
}
|
||||
is CR.CallEnded -> {
|
||||
|
||||
+5
-3
@@ -56,6 +56,7 @@ enum class CallState {
|
||||
AnswerReceived,
|
||||
Negotiated,
|
||||
Connected,
|
||||
Reconnecting,
|
||||
Ended;
|
||||
|
||||
val text: String get() = when(this) {
|
||||
@@ -67,6 +68,7 @@ enum class CallState {
|
||||
AnswerReceived -> generalGetString(MR.strings.callstate_received_confirmation)
|
||||
Negotiated -> generalGetString(MR.strings.callstate_connecting)
|
||||
Connected -> generalGetString(MR.strings.callstate_connected)
|
||||
Reconnecting -> generalGetString(MR.strings.callstate_reconnecting)
|
||||
Ended -> generalGetString(MR.strings.callstate_ended)
|
||||
}
|
||||
}
|
||||
@@ -91,7 +93,7 @@ sealed class WCallCommand {
|
||||
@Serializable @SerialName("start") data class Start(val media: CallMediaType, val aesKey: String? = null, val iceServers: List<RTCIceServer>? = null, val relay: Boolean? = null): WCallCommand()
|
||||
@Serializable @SerialName("offer") data class Offer(val offer: String, val iceCandidates: String, val media: CallMediaType, val aesKey: String? = null, val iceServers: List<RTCIceServer>? = null, val relay: Boolean? = null): WCallCommand()
|
||||
@Serializable @SerialName("answer") data class Answer (val answer: String, val iceCandidates: String): WCallCommand()
|
||||
@Serializable @SerialName("ice") data class Ice(val iceCandidates: String): WCallCommand()
|
||||
@Serializable @SerialName("ice") data class Ice(val iceCandidates: String, val session: String? = null): WCallCommand()
|
||||
@Serializable @SerialName("media") data class Media(val source: CallMediaSource, val enable: Boolean): WCallCommand()
|
||||
@Serializable @SerialName("camera") data class Camera(val camera: VideoCamera): WCallCommand()
|
||||
@Serializable @SerialName("description") data class Description(val state: String, val description: String): WCallCommand()
|
||||
@@ -104,7 +106,7 @@ sealed class WCallResponse {
|
||||
@Serializable @SerialName("capabilities") data class Capabilities(val capabilities: CallCapabilities): WCallResponse()
|
||||
@Serializable @SerialName("offer") data class Offer(val offer: String, val iceCandidates: String, val capabilities: CallCapabilities): WCallResponse()
|
||||
@Serializable @SerialName("answer") data class Answer(val answer: String, val iceCandidates: String): WCallResponse()
|
||||
@Serializable @SerialName("ice") data class Ice(val iceCandidates: String): WCallResponse()
|
||||
@Serializable @SerialName("ice") data class Ice(val iceCandidates: String, val session: String? = null): WCallResponse()
|
||||
@Serializable @SerialName("connection") data class Connection(val state: ConnectionState): WCallResponse()
|
||||
@Serializable @SerialName("connected") data class Connected(val connectionInfo: ConnectionInfo): WCallResponse()
|
||||
@Serializable @SerialName("peerMedia") data class PeerMedia(val source: CallMediaSource, val enabled: Boolean): WCallResponse()
|
||||
@@ -116,7 +118,7 @@ sealed class WCallResponse {
|
||||
|
||||
@Serializable data class WebRTCCallOffer(val callType: CallType, val rtcSession: WebRTCSession)
|
||||
@Serializable data class WebRTCSession(val rtcSession: String, val rtcIceCandidates: String)
|
||||
@Serializable data class WebRTCExtraInfo(val rtcIceCandidates: String)
|
||||
@Serializable data class WebRTCExtraInfo(val rtcSession: String? = null, val rtcIceCandidates: String)
|
||||
@Serializable data class CallType(val media: CallMediaType, val capabilities: CallCapabilities)
|
||||
@Serializable data class RcvCallInvitation(
|
||||
val remoteHostId: Long?,
|
||||
|
||||
@@ -1301,6 +1301,7 @@
|
||||
<string name="callstate_received_answer">received answer…</string>
|
||||
<string name="callstate_received_confirmation">received confirmation…</string>
|
||||
<string name="callstate_connecting">connecting…</string>
|
||||
<string name="callstate_reconnecting">reconnecting…</string>
|
||||
<string name="callstate_connected">connected</string>
|
||||
<string name="callstate_ended">ended</string>
|
||||
|
||||
|
||||
@@ -54,6 +54,8 @@ let inactiveCallMediaSources = {
|
||||
let activeCall;
|
||||
let notConnectedCall;
|
||||
let answerTimeout = 30000;
|
||||
let reconnectTimeout = 30000;
|
||||
let iceRestartInterval = 10000;
|
||||
var useWorker = false;
|
||||
var isDesktop = false;
|
||||
var localizedState = "";
|
||||
@@ -75,7 +77,8 @@ const processCommand = (function () {
|
||||
return {
|
||||
peerConnectionConfig: {
|
||||
iceServers: iceServers !== null && iceServers !== void 0 ? iceServers : defaultIceServers,
|
||||
iceCandidatePoolSize: 10,
|
||||
// Chrome gathers no candidates on ICE restart when candidates pool is used
|
||||
iceCandidatePoolSize: 0,
|
||||
encodedInsertableStreams,
|
||||
iceTransportPolicy: relay ? "relay" : "all",
|
||||
// needed for Android WebView >= 69 && <= 72 where default was "plan-b" which is incompatible with transceivers
|
||||
@@ -202,10 +205,14 @@ const processCommand = (function () {
|
||||
layout: (_c = notConnectedCall === null || notConnectedCall === void 0 ? void 0 : notConnectedCall.layout) !== null && _c !== void 0 ? _c : LayoutType.Default,
|
||||
cameraTrackWasSetBefore: localStream.getVideoTracks().length > 0,
|
||||
peerHasOldVersion: false,
|
||||
receiveRestartSession,
|
||||
};
|
||||
localOrPeerMediaSourcesChanged(call);
|
||||
await setupMediaStreams(call);
|
||||
let connectionTimeout = setTimeout(connectionHandler, answerTimeout);
|
||||
let reconnectingTimeout;
|
||||
let iceRestartTimeout;
|
||||
let wasConnected = false;
|
||||
if (pc.connectionState) {
|
||||
pc.addEventListener("connectionstatechange", connectionStateChange);
|
||||
}
|
||||
@@ -216,30 +223,49 @@ const processCommand = (function () {
|
||||
async function connectionStateChange() {
|
||||
// "failed" means the second party did not answer in time (15 sec timeout in Chrome WebView)
|
||||
// See https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/p2p/base/p2p_constants.cc;l=70)
|
||||
if (pc.connectionState !== "failed")
|
||||
if (pc.connectionState !== "failed" || wasConnected)
|
||||
connectionHandler();
|
||||
}
|
||||
async function connectionHandler() {
|
||||
async function connectionHandler(reconnectTimedOut = false) {
|
||||
var _a;
|
||||
let state = (_a = pc.connectionState) !== null && _a !== void 0 ? _a : (pc.iceConnectionState != "completed" && pc.iceConnectionState != "checking"
|
||||
? pc.iceConnectionState
|
||||
: pc.iceConnectionState == "completed"
|
||||
? "connected"
|
||||
: "connecting"); /* webView 69-70 doesn't have connectionState yet */
|
||||
if (wasConnected) {
|
||||
if (reconnectTimedOut) {
|
||||
reconnectingTimeout = undefined;
|
||||
if (activeCall !== call || state == "connected")
|
||||
return;
|
||||
state = "disconnected";
|
||||
}
|
||||
else if (state == "disconnected" || state == "failed") {
|
||||
if (reconnectingTimeout !== undefined)
|
||||
return;
|
||||
reconnectingTimeout = setTimeout(() => connectionHandler(true), reconnectTimeout);
|
||||
if (isIceRestarter())
|
||||
restartIce();
|
||||
state = "reconnecting";
|
||||
}
|
||||
else if (state == "connecting" || state == "new") {
|
||||
return;
|
||||
}
|
||||
}
|
||||
sendMessageToNative({
|
||||
resp: {
|
||||
type: "connection",
|
||||
state: {
|
||||
connectionState: (_a = pc.connectionState) !== null && _a !== void 0 ? _a : (pc.iceConnectionState != "completed" && pc.iceConnectionState != "checking"
|
||||
? pc.iceConnectionState
|
||||
: pc.iceConnectionState == "completed"
|
||||
? "connected"
|
||||
: "connecting") /* webView 69-70 doesn't have connectionState yet */,
|
||||
connectionState: state,
|
||||
iceConnectionState: pc.iceConnectionState,
|
||||
iceGatheringState: pc.iceGatheringState,
|
||||
signalingState: pc.signalingState,
|
||||
},
|
||||
},
|
||||
});
|
||||
if (pc.connectionState == "disconnected" ||
|
||||
pc.connectionState == "failed" ||
|
||||
(!pc.connectionState && (pc.iceConnectionState == "disconnected" || pc.iceConnectionState == "failed"))) {
|
||||
if (state == "disconnected" || state == "failed") {
|
||||
clearConnectionTimeout();
|
||||
clearReconnecting();
|
||||
if (pc.connectionState) {
|
||||
pc.removeEventListener("connectionstatechange", connectionStateChange);
|
||||
}
|
||||
@@ -251,8 +277,10 @@ const processCommand = (function () {
|
||||
}
|
||||
endCall();
|
||||
}
|
||||
else if (pc.connectionState == "connected" || (!pc.connectionState && pc.iceConnectionState == "connected")) {
|
||||
else if (state == "connected") {
|
||||
clearConnectionTimeout();
|
||||
clearReconnecting();
|
||||
wasConnected = true;
|
||||
const stats = (await pc.getStats());
|
||||
for (const stat of stats.values()) {
|
||||
const { type, state } = stat;
|
||||
@@ -278,6 +306,53 @@ const processCommand = (function () {
|
||||
connectionTimeout = undefined;
|
||||
}
|
||||
}
|
||||
function clearReconnecting() {
|
||||
clearTimeout(reconnectingTimeout);
|
||||
reconnectingTimeout = undefined;
|
||||
clearTimeout(iceRestartTimeout);
|
||||
iceRestartTimeout = undefined;
|
||||
}
|
||||
// Only the party that sent the initial offer restarts ICE, so that restart offers never collide
|
||||
function isIceRestarter() {
|
||||
var _a;
|
||||
return ((_a = pc.localDescription) === null || _a === void 0 ? void 0 : _a.type) == "offer";
|
||||
}
|
||||
async function restartIce() {
|
||||
if (activeCall !== call || reconnectingTimeout === undefined)
|
||||
return;
|
||||
iceRestartTimeout = setTimeout(restartIce, iceRestartInterval);
|
||||
// the previous restart offer is not answered yet
|
||||
if (pc.signalingState != "stable")
|
||||
return;
|
||||
try {
|
||||
const iceCandidates = getIceCandidates(pc, config);
|
||||
const offer = await pc.createOffer({ iceRestart: true });
|
||||
await pc.setLocalDescription(offer);
|
||||
sendMessageToNative({ resp: { type: "ice", session: serialize(offer), iceCandidates: await iceCandidates } });
|
||||
}
|
||||
catch (e) {
|
||||
console.log("ICE restart error", e);
|
||||
}
|
||||
}
|
||||
async function receiveRestartSession(session, iceCandidates) {
|
||||
const desc = parse(session);
|
||||
const remoteDesc = new RTCSessionDescription(!webView69Or70() ? desc : adaptSdpToOldWebView(desc));
|
||||
if (desc.type == "offer" && !isIceRestarter()) {
|
||||
await pc.setRemoteDescription(remoteDesc);
|
||||
const localIceCandidates = getIceCandidates(pc, config);
|
||||
const answer = await pc.createAnswer();
|
||||
await pc.setLocalDescription(answer);
|
||||
addIceCandidates(pc, iceCandidates);
|
||||
sendMessageToNative({ resp: { type: "ice", session: serialize(answer), iceCandidates: await localIceCandidates } });
|
||||
}
|
||||
else if (desc.type == "answer" && pc.signalingState == "have-local-offer") {
|
||||
await pc.setRemoteDescription(remoteDesc);
|
||||
addIceCandidates(pc, iceCandidates);
|
||||
}
|
||||
else {
|
||||
throw new Error(`unexpected ICE restart ${desc.type} in ${pc.signalingState} state`);
|
||||
}
|
||||
}
|
||||
}
|
||||
function serialize(x) {
|
||||
return LZString.compressToBase64(JSON.stringify(x));
|
||||
@@ -296,6 +371,8 @@ const processCommand = (function () {
|
||||
console.log("starting outgoing call - capabilities");
|
||||
if (activeCall)
|
||||
endCall();
|
||||
// Stop a preview stream from an earlier pre-connect outgoing call being replaced (activeCall may be null here)
|
||||
stopNotConnectedCall();
|
||||
let localStream = null;
|
||||
try {
|
||||
localStream = await getLocalMediaStream(true, command.media == CallMediaType.Video && (await browserHasCamera()), VideoCamera.User);
|
||||
@@ -332,7 +409,8 @@ const processCommand = (function () {
|
||||
if (activeCall)
|
||||
endCall();
|
||||
// It can be already defined on Android when switching calls (if the previous call was outgoing)
|
||||
notConnectedCall = undefined;
|
||||
// Stop its preview tracks before clearing, otherwise camera/mic stay live
|
||||
stopNotConnectedCall();
|
||||
inactiveCallMediaSources.mic = true;
|
||||
inactiveCallMediaSources.camera = command.media == CallMediaType.Video;
|
||||
inactiveCallMediaSourcesChanged(inactiveCallMediaSources);
|
||||
@@ -436,7 +514,11 @@ const processCommand = (function () {
|
||||
break;
|
||||
case "ice":
|
||||
const remoteIceCandidates = parse(command.iceCandidates);
|
||||
if (pc) {
|
||||
if (activeCall && command.session) {
|
||||
await activeCall.receiveRestartSession(command.session, remoteIceCandidates);
|
||||
resp = { type: "ok" };
|
||||
}
|
||||
else if (pc) {
|
||||
addIceCandidates(pc, remoteIceCandidates);
|
||||
resp = { type: "ok" };
|
||||
}
|
||||
@@ -1131,6 +1213,13 @@ const processCommand = (function () {
|
||||
activeCall.localScreenStream.getTracks().forEach((track) => track.stop());
|
||||
}
|
||||
}
|
||||
// Call on any path that abandons notConnectedCall, otherwise its preview camera/mic tracks stay live.
|
||||
function stopNotConnectedCall() {
|
||||
if (notConnectedCall) {
|
||||
notConnectedCall.localStream.getTracks().forEach((track) => track.stop());
|
||||
notConnectedCall = undefined;
|
||||
}
|
||||
}
|
||||
function resetVideoElements() {
|
||||
const videos = getVideoElements();
|
||||
if (!videos)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Override defaults to enable worker on Chrome and Safari
|
||||
useWorker = typeof window.Worker !== "undefined";
|
||||
isDesktop = true;
|
||||
// Create WebSocket connection.
|
||||
// Create WebSocket connection. location.search carries the per-call ?token=... capability required by the server.
|
||||
const socket = new WebSocket(`ws://${location.host}${location.search}`);
|
||||
socket.addEventListener("open", (_event) => {
|
||||
console.log("Opened socket");
|
||||
@@ -192,4 +192,4 @@ function updateCallInfoView(state, description) {
|
||||
document.getElementById("state").innerText = state;
|
||||
document.getElementById("description").innerText = description;
|
||||
}
|
||||
//# sourceMappingURL=ui.js.map
|
||||
//# sourceMappingURL=ui.js.map
|
||||
+3
-1
@@ -313,7 +313,9 @@ actual object CallSoundsPlayer: CallSoundsPlayerInterface {
|
||||
override fun startConnectingCallSound(scope: CoroutineScope) {
|
||||
// Taken from https://github.com/TelegramOrg/Telegram-Android
|
||||
// https://github.com/TelegramOrg/Telegram-Android/blob/master/LICENSE
|
||||
start("/media/connecting_call.mp3", 3000, scope)
|
||||
// the delay is the length of the track exactly, so it repeats with no delay, which already happens on ios and android
|
||||
// Android (isLooping) and iOS (which sleeps for the track duration)
|
||||
start("/media/connecting_call.mp3", 3536, scope)
|
||||
}
|
||||
|
||||
override fun startInCallSound(scope: CoroutineScope) {
|
||||
|
||||
+27
-16
@@ -8,6 +8,7 @@ import chat.simplex.common.platform.*
|
||||
import chat.simplex.common.views.helpers.*
|
||||
import chat.simplex.res.*
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.flow.*
|
||||
import kotlinx.datetime.Clock
|
||||
import kotlinx.serialization.encodeToString
|
||||
@@ -55,13 +56,24 @@ actual fun ActiveCallView() {
|
||||
CallSoundsPlayer.stop()
|
||||
}
|
||||
is WCallResponse.Ice -> withBGApi {
|
||||
chatModel.controller.apiSendCallExtraInfo(callRh, call.contact, r.iceCandidates)
|
||||
chatModel.controller.apiSendCallExtraInfo(callRh, call.contact, r.session, r.iceCandidates)
|
||||
}
|
||||
is WCallResponse.Connection ->
|
||||
try {
|
||||
// the call is restored and NOT sent to the core
|
||||
// Makes sure the core DOES NOT record the duration wrong if a reconnect happens
|
||||
if (r.state.connectionState == "reconnecting") {
|
||||
if (call.callState != CallState.Reconnecting) {
|
||||
CallSoundsPlayer.startConnectingCallSound(scope)
|
||||
}
|
||||
chatModel.activeCall.value = call.copy(callState = CallState.Reconnecting)
|
||||
} else try {
|
||||
val callStatus = json.decodeFromString<WebRTCCallStatus>("\"${r.state.connectionState}\"")
|
||||
if (callStatus == WebRTCCallStatus.Connected) {
|
||||
chatModel.activeCall.value = call.copy(callState = CallState.Connected, connectedAt = Clock.System.now())
|
||||
if (call.callState == CallState.Reconnecting) {
|
||||
CallSoundsPlayer.stop()
|
||||
}
|
||||
// connectedAt is only set ONCE, this way call duration is NOT reset when the call is restored from reconnecting
|
||||
chatModel.activeCall.value = call.copy(callState = CallState.Connected, connectedAt = call.connectedAt ?: Clock.System.now())
|
||||
}
|
||||
withBGApi { chatModel.controller.apiCallStatus(callRh, call.contact, callStatus) }
|
||||
} catch (e: Throwable) {
|
||||
@@ -194,21 +206,20 @@ fun WebRTCController(callCommand: SnapshotStateList<WCallCommand>, onResponse: (
|
||||
}
|
||||
}
|
||||
LaunchedEffect(Unit) {
|
||||
snapshotFlow { callCommand.firstOrNull() }
|
||||
.distinctUntilChanged()
|
||||
.filterNotNull()
|
||||
.collect {
|
||||
while (connections.isEmpty()) {
|
||||
delay(100)
|
||||
}
|
||||
while (callCommand.isNotEmpty()) {
|
||||
val cmd = callCommand.removeFirstOrNull()
|
||||
Log.d(TAG, "WebRTCController LaunchedEffect executing ${cmd?.javaClass?.simpleName}")
|
||||
if (cmd != null) {
|
||||
processCommand(cmd)
|
||||
}
|
||||
while (isActive) {
|
||||
// waiting for the first command would miss a command equal to the previous one
|
||||
snapshotFlow { callCommand.isNotEmpty() }.first { it }
|
||||
while (connections.isEmpty()) {
|
||||
delay(100)
|
||||
}
|
||||
while (callCommand.isNotEmpty()) {
|
||||
val cmd = callCommand.removeFirstOrNull()
|
||||
Log.d(TAG, "WebRTCController LaunchedEffect executing ${cmd?.javaClass?.simpleName}")
|
||||
if (cmd != null) {
|
||||
processCommand(cmd)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -99,6 +99,7 @@ Both platforms share a common signaling flow through the Haskell core API.
|
||||
```
|
||||
Outgoing: WaitCapabilities -> InvitationSent -> OfferSent -> AnswerReceived -> Negotiated -> Connected -> Ended
|
||||
Incoming: InvitationAccepted -> OfferReceived -> Negotiated -> Connected -> Ended
|
||||
Reconnection: Connected -> Reconnecting -> Connected | Ended
|
||||
```
|
||||
|
||||
| State | Description |
|
||||
@@ -111,8 +112,17 @@ Incoming: InvitationAccepted -> OfferReceived -> Negotiated -> Connected -> Ende
|
||||
| `AnswerReceived` | Caller received SDP answer |
|
||||
| `Negotiated` | ICE negotiation complete |
|
||||
| `Connected` | Media flowing |
|
||||
| `Reconnecting` | Connection lost, waiting for ICE restart |
|
||||
| `Ended` | Call terminated |
|
||||
|
||||
### 3.1 Reconnection
|
||||
|
||||
1. A connected call loses its WebRTC connection, e.g. when the device switches between Wi-Fi and mobile network.
|
||||
2. The WebRTC engine reports `"reconnecting"` connection state; the app shows "reconnecting…" and plays the connecting sound. This state is not reported to the core, and `connectedAt` is kept, so call duration continues.
|
||||
3. The callee's WebRTC engine (the party that sent the initial offer) restarts ICE every 10 seconds, sending the new offer in `WCallResponse.Ice` with `session` via `apiSendCallExtraInfo`. The caller answers the same way.
|
||||
4. When the connection is restored, the call returns to `Connected` and the sound stops.
|
||||
5. If the connection is not restored in 30 seconds, the call ends as disconnected. Clients without ICE restart support ignore the restart offer, so the call ends in this case.
|
||||
|
||||
---
|
||||
|
||||
## 4. Ending a Call
|
||||
@@ -184,7 +194,7 @@ Incoming: InvitationAccepted -> OfferReceived -> Negotiated -> Connected -> Ende
|
||||
| `apiRejectCall(rh, contact)` | Reject incoming call |
|
||||
| `apiSendCallOffer(rh, contact, rtcSession, rtcIceCandidates, media, capabilities)` | Send SDP offer |
|
||||
| `apiSendCallAnswer(rh, contact, rtcSession, rtcIceCandidates)` | Send SDP answer |
|
||||
| `apiSendCallExtraInfo(rh, contact, rtcIceCandidates)` | Send additional ICE candidates |
|
||||
| `apiSendCallExtraInfo(rh, contact, rtcSession, rtcIceCandidates)` | Send additional ICE candidates, or ICE restart offer/answer |
|
||||
| `apiEndCall(rh, contact)` | End active call |
|
||||
| `apiCallStatus(rh, contact, status)` | Report WebRTC connection status |
|
||||
|
||||
|
||||
@@ -71,8 +71,8 @@ Sound: `SoundPlayer.start()` plays ringtone while banner is visible (unless call
|
||||
| 1. Capabilities | `WCallResponse.Capabilities` | Local capabilities reported; `apiSendCallInvitation()` called |
|
||||
| 2. Offer | `WCallResponse.Offer` | SDP offer + ICE candidates sent via `apiSendCallOffer()` |
|
||||
| 3. Answer | `WCallResponse.Answer` | SDP answer + ICE candidates sent via `apiSendCallAnswer()` |
|
||||
| 4. ICE | `WCallResponse.Ice` | Additional ICE candidates exchanged via `apiSendCallExtraInfo()` |
|
||||
| 5. Connection | `WCallResponse.Connection` | WebRTC connection state changes; `CallState.Connected` set on success |
|
||||
| 4. ICE | `WCallResponse.Ice` | Additional ICE candidates, and ICE restart offer/answer in `session`, exchanged via `apiSendCallExtraInfo()` |
|
||||
| 5. Connection | `WCallResponse.Connection` | WebRTC connection state changes; `CallState.Connected` set on success, `CallState.Reconnecting` when a connected call loses connection |
|
||||
| 6. Connected | `WCallResponse.Connected` | Connection info (relay/direct) stored in `call.connectionInfo` |
|
||||
| 7. PeerMedia | `WCallResponse.PeerMedia` | Remote party media source changes (mic, camera, screen) |
|
||||
| 8. Media control | `WCallCommand.Media` | Toggle local media sources (mic, camera, screen audio/video) |
|
||||
@@ -90,14 +90,15 @@ Sound: `SoundPlayer.start()` plays ringtone while banner is visible (unless call
|
||||
| `OfferReceived` | Callee received SDP offer |
|
||||
| `AnswerReceived` | Caller received SDP answer |
|
||||
| `Negotiated` | ICE negotiation complete |
|
||||
| `Connected` | WebRTC media flowing; `connectedAt` timestamp set |
|
||||
| `Connected` | WebRTC media flowing; `connectedAt` timestamp set on the first connection |
|
||||
| `Reconnecting` | Connection lost; "reconnecting…" shown while ICE restarts, call ends after 30 seconds without connection |
|
||||
| `Ended` | Call terminated |
|
||||
|
||||
### Call Sounds
|
||||
|
||||
| Sound | Trigger |
|
||||
|---|---|
|
||||
| Connecting sound | `CallSoundsPlayer.startConnectingCallSound()` after invitation sent |
|
||||
| Connecting sound | `CallSoundsPlayer.startConnectingCallSound()` after invitation sent, and while reconnecting |
|
||||
| In-call sound | `CallSoundsPlayer.startInCallSound()` when delivery receipt received |
|
||||
| Ringtone | `SoundPlayer.start()` for incoming calls |
|
||||
| End vibration | `CallSoundsPlayer.vibrate()` on call end (if was connected) |
|
||||
|
||||
@@ -225,7 +225,7 @@ All functions below are `suspend fun` members of `ChatController` ([SimpleXAPI.k
|
||||
| `apiRejectCall` | `rh: Long?, contact: Contact` | Reject an incoming call | [L1854](../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1854) |
|
||||
| `apiSendCallOffer` | `rh, contact, rtcSession, rtcIceCandidates, media, capabilities` | Send a WebRTC call offer | [L1859](../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1859) |
|
||||
| `apiSendCallAnswer` | `rh: Long?, contact: Contact, rtcSession: String, rtcIceCandidates: String` | Send a WebRTC call answer | [L1866](../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1866) |
|
||||
| `apiSendCallExtraInfo` | `rh: Long?, contact: Contact, rtcIceCandidates: String` | Send additional ICE candidates during a call | [L1872](../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1872) |
|
||||
| `apiSendCallExtraInfo` | `rh: Long?, contact: Contact, rtcSession: String?, rtcIceCandidates: String` | Send additional ICE candidates or ICE restart offer/answer during a call | [L2156](../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L2156) |
|
||||
| `apiEndCall` | `rh: Long?, contact: Contact` | End an active call | [L1878](../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1878) |
|
||||
| `apiCallStatus` | `rh: Long?, contact: Contact, status: WebRTCCallStatus` | Report call status updates to the core | [L1883](../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1883) |
|
||||
|
||||
|
||||
@@ -43,6 +43,7 @@ enum class CallState {
|
||||
AnswerReceived, // SDP answer received from peer
|
||||
Negotiated, // ICE negotiation in progress
|
||||
Connected, // Media flowing
|
||||
Reconnecting, // Connection lost, waiting for ICE to recover
|
||||
Ended; // Call terminated
|
||||
}
|
||||
```
|
||||
@@ -53,6 +54,8 @@ enum class CallState {
|
||||
|
||||
State transitions are driven by `WCallResponse` messages from the WebRTC layer. Each transition typically triggers a corresponding API command (e.g., `apiSendCallInvitation`, `apiSendCallOffer`).
|
||||
|
||||
**Reconnection**: `Connected` -> `Reconnecting` -> `Connected` or `Ended`. When a connected call loses its WebRTC connection, `call.js` reports `"reconnecting"` connection state instead of ending the call. The app sets `CallState.Reconnecting` and plays the connecting sound; this state is not sent to the core with `apiCallStatus`, so the call item duration is not affected, and `connectedAt` is kept. The party that sent the initial offer (callee) restarts ICE every 10 seconds until reconnected: the restart offer and the answer are sent as `WCallResponse.Ice` / `WCallCommand.Ice` with `session`, via `apiSendCallExtraInfo` (`x.call.extra` with `rtcSession`). If the connection is not restored in 30 seconds, `call.js` reports `"disconnected"` and ends the call.
|
||||
|
||||
---
|
||||
|
||||
<a id="ActiveCallView"></a>
|
||||
@@ -140,7 +143,7 @@ Defined in [`SimpleXAPI.kt`](../../common/src/commonMain/kotlin/chat/simplex/com
|
||||
| `apiRejectCall` | [L1854](../../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1854) | Reject an incoming call |
|
||||
| `apiSendCallOffer` | [L1859](../../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1859) | Send SDP offer with ICE candidates and capabilities |
|
||||
| `apiSendCallAnswer` | [L1866](../../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1866) | Send SDP answer with ICE candidates |
|
||||
| `apiSendCallExtraInfo` | [L1872](../../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1872) | Send additional ICE candidates discovered after initial exchange |
|
||||
| `apiSendCallExtraInfo` | [L2156](../../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L2156) | Send additional ICE candidates discovered after initial exchange, or ICE restart offer/answer in `rtcSession` |
|
||||
| `apiEndCall` | [L1878](../../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1878) | Terminate a call |
|
||||
| `apiCallStatus` | [L1883](../../common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt#L1883) | Report WebRTC connection status to the backend |
|
||||
|
||||
|
||||
@@ -4,3 +4,20 @@
|
||||
npm i
|
||||
npm run build
|
||||
```
|
||||
|
||||
## Tests
|
||||
|
||||
End-to-end tests run two headless Chromium peers with `call.js` and simulate network outages by relaying media via UDP sockets controlled by the test:
|
||||
|
||||
```
|
||||
npx playwright-core install chromium
|
||||
npm test
|
||||
```
|
||||
|
||||
Set `CHROME` to use another Chromium executable, and `VERBOSE=1` to log signaling and connection states.
|
||||
|
||||
Full-stack tests in `test/fullstack.test.js` also pass call signaling via two chat cores and a local SMP server. They run when `SIMPLEX_CHAT` and `SMP_SERVER` are set to `simplex-chat` and `smp-server` executables:
|
||||
|
||||
```
|
||||
SIMPLEX_CHAT=path/to/simplex-chat SMP_SERVER=path/to/smp-server npm test
|
||||
```
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
"dist"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1",
|
||||
"test": "tsc && node --test test/*.test.js",
|
||||
"build": "prettier --write --ignore-unknown . && tsc && ./copy"
|
||||
},
|
||||
"repository": {
|
||||
@@ -31,6 +31,7 @@
|
||||
"husky": "^7.0.4",
|
||||
"isomorphic-webcrypto": "^2.3.8",
|
||||
"lint-staged": "^12.4.1",
|
||||
"playwright-core": "^1.64.0",
|
||||
"prettier": "^2.6.2",
|
||||
"typescript": "^4.6.4"
|
||||
},
|
||||
|
||||
@@ -142,6 +142,7 @@ interface WCallAnswer extends IWCallCommand, IWCallResponse {
|
||||
interface WCallIceCandidates extends IWCallCommand, IWCallResponse {
|
||||
type: "ice"
|
||||
iceCandidates: string // JSON strings for RTCIceCandidateInit[]
|
||||
session?: string // JSON string for RTCSessionDescriptionInit of ICE restart offer or answer
|
||||
}
|
||||
|
||||
interface WCEnableMedia extends IWCallCommand {
|
||||
@@ -281,6 +282,7 @@ interface Call {
|
||||
// controls whether transceiver's track was replaced already or set initially (in video call)
|
||||
cameraTrackWasSetBefore: boolean
|
||||
peerHasOldVersion: boolean
|
||||
receiveRestartSession: (session: string, iceCandidates: RTCIceCandidateInit[]) => Promise<void>
|
||||
}
|
||||
|
||||
interface NotConnectedCall {
|
||||
@@ -306,6 +308,8 @@ let inactiveCallMediaSources: CallMediaSources = {
|
||||
let activeCall: Call | undefined
|
||||
let notConnectedCall: NotConnectedCall | undefined
|
||||
let answerTimeout = 30_000
|
||||
let reconnectTimeout = 30_000
|
||||
let iceRestartInterval = 10_000
|
||||
var useWorker = false
|
||||
var isDesktop = false
|
||||
var localizedState = ""
|
||||
@@ -357,7 +361,8 @@ const processCommand = (function () {
|
||||
return {
|
||||
peerConnectionConfig: {
|
||||
iceServers: iceServers ?? defaultIceServers,
|
||||
iceCandidatePoolSize: 10,
|
||||
// Chrome gathers no candidates on ICE restart when candidates pool is used
|
||||
iceCandidatePoolSize: 0,
|
||||
encodedInsertableStreams,
|
||||
iceTransportPolicy: relay ? "relay" : "all",
|
||||
// needed for Android WebView >= 69 && <= 72 where default was "plan-b" which is incompatible with transceivers
|
||||
@@ -486,10 +491,14 @@ const processCommand = (function () {
|
||||
layout: notConnectedCall?.layout ?? LayoutType.Default,
|
||||
cameraTrackWasSetBefore: localStream.getVideoTracks().length > 0,
|
||||
peerHasOldVersion: false,
|
||||
receiveRestartSession,
|
||||
}
|
||||
localOrPeerMediaSourcesChanged(call)
|
||||
await setupMediaStreams(call)
|
||||
let connectionTimeout: number | undefined = setTimeout(connectionHandler, answerTimeout)
|
||||
let reconnectingTimeout: number | undefined
|
||||
let iceRestartTimeout: number | undefined
|
||||
let wasConnected = false
|
||||
if (pc.connectionState) {
|
||||
pc.addEventListener("connectionstatechange", connectionStateChange)
|
||||
} else {
|
||||
@@ -500,33 +509,45 @@ const processCommand = (function () {
|
||||
async function connectionStateChange() {
|
||||
// "failed" means the second party did not answer in time (15 sec timeout in Chrome WebView)
|
||||
// See https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/p2p/base/p2p_constants.cc;l=70)
|
||||
if (pc.connectionState !== "failed") connectionHandler()
|
||||
if (pc.connectionState !== "failed" || wasConnected) connectionHandler()
|
||||
}
|
||||
|
||||
async function connectionHandler() {
|
||||
async function connectionHandler(reconnectTimedOut = false) {
|
||||
let state: string =
|
||||
pc.connectionState ??
|
||||
(pc.iceConnectionState != "completed" && pc.iceConnectionState != "checking"
|
||||
? pc.iceConnectionState
|
||||
: pc.iceConnectionState == "completed"
|
||||
? "connected"
|
||||
: "connecting") /* webView 69-70 doesn't have connectionState yet */
|
||||
if (wasConnected) {
|
||||
if (reconnectTimedOut) {
|
||||
reconnectingTimeout = undefined
|
||||
if (activeCall !== call || state == "connected") return
|
||||
state = "disconnected"
|
||||
} else if (state == "disconnected" || state == "failed") {
|
||||
if (reconnectingTimeout !== undefined) return
|
||||
reconnectingTimeout = setTimeout(() => connectionHandler(true), reconnectTimeout)
|
||||
if (isIceRestarter()) restartIce()
|
||||
state = "reconnecting"
|
||||
} else if (state == "connecting" || state == "new") {
|
||||
return
|
||||
}
|
||||
}
|
||||
sendMessageToNative({
|
||||
resp: {
|
||||
type: "connection",
|
||||
state: {
|
||||
connectionState:
|
||||
pc.connectionState ??
|
||||
(pc.iceConnectionState != "completed" && pc.iceConnectionState != "checking"
|
||||
? pc.iceConnectionState
|
||||
: pc.iceConnectionState == "completed"
|
||||
? "connected"
|
||||
: "connecting") /* webView 69-70 doesn't have connectionState yet */,
|
||||
connectionState: state,
|
||||
iceConnectionState: pc.iceConnectionState,
|
||||
iceGatheringState: pc.iceGatheringState,
|
||||
signalingState: pc.signalingState,
|
||||
},
|
||||
},
|
||||
})
|
||||
if (
|
||||
pc.connectionState == "disconnected" ||
|
||||
pc.connectionState == "failed" ||
|
||||
(!pc.connectionState && (pc.iceConnectionState == "disconnected" || pc.iceConnectionState == "failed"))
|
||||
) {
|
||||
if (state == "disconnected" || state == "failed") {
|
||||
clearConnectionTimeout()
|
||||
clearReconnecting()
|
||||
if (pc.connectionState) {
|
||||
pc.removeEventListener("connectionstatechange", connectionStateChange)
|
||||
} else {
|
||||
@@ -536,8 +557,10 @@ const processCommand = (function () {
|
||||
setTimeout(() => sendMessageToNative({resp: {type: "ended"}}), 0)
|
||||
}
|
||||
endCall()
|
||||
} else if (pc.connectionState == "connected" || (!pc.connectionState && pc.iceConnectionState == "connected")) {
|
||||
} else if (state == "connected") {
|
||||
clearConnectionTimeout()
|
||||
clearReconnecting()
|
||||
wasConnected = true
|
||||
const stats = (await pc.getStats()) as Map<string, any>
|
||||
for (const stat of stats.values()) {
|
||||
const {type, state} = stat
|
||||
@@ -564,6 +587,51 @@ const processCommand = (function () {
|
||||
connectionTimeout = undefined
|
||||
}
|
||||
}
|
||||
|
||||
function clearReconnecting() {
|
||||
clearTimeout(reconnectingTimeout)
|
||||
reconnectingTimeout = undefined
|
||||
clearTimeout(iceRestartTimeout)
|
||||
iceRestartTimeout = undefined
|
||||
}
|
||||
|
||||
// Only the party that sent the initial offer restarts ICE, so that restart offers never collide
|
||||
function isIceRestarter(): boolean {
|
||||
return pc.localDescription?.type == "offer"
|
||||
}
|
||||
|
||||
async function restartIce() {
|
||||
if (activeCall !== call || reconnectingTimeout === undefined) return
|
||||
iceRestartTimeout = setTimeout(restartIce, iceRestartInterval)
|
||||
// the previous restart offer is not answered yet
|
||||
if (pc.signalingState != "stable") return
|
||||
try {
|
||||
const iceCandidates = getIceCandidates(pc, config)
|
||||
const offer = await pc.createOffer({iceRestart: true})
|
||||
await pc.setLocalDescription(offer)
|
||||
sendMessageToNative({resp: {type: "ice", session: serialize(offer), iceCandidates: await iceCandidates}})
|
||||
} catch (e) {
|
||||
console.log("ICE restart error", e)
|
||||
}
|
||||
}
|
||||
|
||||
async function receiveRestartSession(session: string, iceCandidates: RTCIceCandidateInit[]) {
|
||||
const desc: RTCSessionDescriptionInit = parse(session)
|
||||
const remoteDesc = new RTCSessionDescription(!webView69Or70() ? desc : adaptSdpToOldWebView(desc))
|
||||
if (desc.type == "offer" && !isIceRestarter()) {
|
||||
await pc.setRemoteDescription(remoteDesc)
|
||||
const localIceCandidates = getIceCandidates(pc, config)
|
||||
const answer = await pc.createAnswer()
|
||||
await pc.setLocalDescription(answer)
|
||||
addIceCandidates(pc, iceCandidates)
|
||||
sendMessageToNative({resp: {type: "ice", session: serialize(answer), iceCandidates: await localIceCandidates}})
|
||||
} else if (desc.type == "answer" && pc.signalingState == "have-local-offer") {
|
||||
await pc.setRemoteDescription(remoteDesc)
|
||||
addIceCandidates(pc, iceCandidates)
|
||||
} else {
|
||||
throw new Error(`unexpected ICE restart ${desc.type} in ${pc.signalingState} state`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function serialize<T>(x: T): string {
|
||||
@@ -729,7 +797,10 @@ const processCommand = (function () {
|
||||
break
|
||||
case "ice":
|
||||
const remoteIceCandidates: RTCIceCandidateInit[] = parse(command.iceCandidates)
|
||||
if (pc) {
|
||||
if (activeCall && command.session) {
|
||||
await activeCall.receiveRestartSession(command.session, remoteIceCandidates)
|
||||
resp = {type: "ok"}
|
||||
} else if (pc) {
|
||||
addIceCandidates(pc, remoteIceCandidates)
|
||||
resp = {type: "ok"}
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
// Runs a local SMP server and chat cores with WebSocket API, so that call signaling goes via the chat protocol.
|
||||
const {spawn, execFileSync} = require("child_process")
|
||||
const fs = require("fs")
|
||||
const net = require("net")
|
||||
const path = require("path")
|
||||
const {sleep, log} = require("./harness")
|
||||
|
||||
const START_TIMEOUT = 30000
|
||||
|
||||
function freePort() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = net.createServer()
|
||||
server.on("error", reject)
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
const {port} = server.address()
|
||||
server.close(() => resolve(port))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async function waitUntil(what, check) {
|
||||
const deadline = Date.now() + START_TIMEOUT
|
||||
while (!(await check())) {
|
||||
if (Date.now() > deadline) throw new Error(`${what} did not start in ${START_TIMEOUT}ms`)
|
||||
await sleep(200)
|
||||
}
|
||||
}
|
||||
|
||||
function canConnect(port) {
|
||||
return new Promise((resolve) => {
|
||||
const socket = net.connect(port, "127.0.0.1", () => {
|
||||
socket.destroy()
|
||||
resolve(true)
|
||||
})
|
||||
socket.on("error", () => resolve(false))
|
||||
})
|
||||
}
|
||||
|
||||
function spawnLogged(name, bin, args, env) {
|
||||
const proc = spawn(bin, args, {env, stdio: ["ignore", "pipe", "pipe"]})
|
||||
const output = (data) => log(`${name}:`, data.toString().trimEnd())
|
||||
proc.stdout.on("data", output)
|
||||
proc.stderr.on("data", output)
|
||||
return proc
|
||||
}
|
||||
|
||||
async function startSmpServer(bin, dir) {
|
||||
const env = {...process.env, SMP_SERVER_CFG_PATH: path.join(dir, "cfg"), SMP_SERVER_LOG_PATH: path.join(dir, "log")}
|
||||
const init = ["init", "--ip", "127.0.0.1", "--no-password", "--disable-web", "--disable-store-log", "-y"]
|
||||
const fingerprint = execFileSync(bin, init, {env, stdio: ["ignore", "pipe", "pipe"]})
|
||||
.toString()
|
||||
.match(/Fingerprint: (\S+)/)[1]
|
||||
const port = await freePort()
|
||||
const iniFile = path.join(dir, "cfg", "smp-server.ini")
|
||||
const ini = fs
|
||||
.readFileSync(iniFile, "utf8")
|
||||
.replace(/^port = .*$/m, `port = ${port}`)
|
||||
.replace(/^host = .*$/m, "host = localhost")
|
||||
fs.writeFileSync(iniFile, ini)
|
||||
const proc = spawnLogged("smp-server", bin, ["start"], env)
|
||||
await waitUntil("SMP server", () => canConnect(port))
|
||||
return {address: `smp://${fingerprint}@localhost:${port}`, stop: () => proc.kill()}
|
||||
}
|
||||
|
||||
class ChatClient {
|
||||
constructor(name, proc, ws) {
|
||||
this.name = name
|
||||
this.proc = proc
|
||||
this.ws = ws
|
||||
this.corrId = 0
|
||||
this.pending = new Map()
|
||||
this.events = []
|
||||
this.waiters = []
|
||||
this.onEvent = async () => {}
|
||||
ws.addEventListener("message", (e) => this.receive(JSON.parse(e.data)))
|
||||
}
|
||||
|
||||
static async start(bin, dir, name, smpServer) {
|
||||
const port = await freePort()
|
||||
const args = ["-d", path.join(dir, name), "-s", smpServer, "-p", String(port), "--user-display-name", name]
|
||||
const proc = spawnLogged(`chat ${name}`, bin, args, process.env)
|
||||
await waitUntil(`chat ${name}`, () => canConnect(port))
|
||||
const ws = new WebSocket(`ws://127.0.0.1:${port}`)
|
||||
await new Promise((resolve, reject) => {
|
||||
ws.addEventListener("open", resolve, {once: true})
|
||||
ws.addEventListener("error", reject, {once: true})
|
||||
})
|
||||
return new ChatClient(name, proc, ws)
|
||||
}
|
||||
|
||||
receive({corrId, resp}) {
|
||||
if (corrId) {
|
||||
this.pending.get(corrId)?.(resp)
|
||||
this.pending.delete(corrId)
|
||||
return
|
||||
}
|
||||
log(`${this.name} event:`, resp.type)
|
||||
this.events.push(resp)
|
||||
this.onEvent(resp).catch((e) => log(`${this.name} event error`, e))
|
||||
this.waiters = this.waiters.filter((w) => !(w.pred(resp) && (w.resolve(resp), true)))
|
||||
}
|
||||
|
||||
async cmd(cmd) {
|
||||
const corrId = String(++this.corrId)
|
||||
log(`${this.name} cmd:`, cmd.slice(0, 100))
|
||||
const resp = await new Promise((resolve) => {
|
||||
this.pending.set(corrId, resolve)
|
||||
this.ws.send(JSON.stringify({corrId, cmd}))
|
||||
})
|
||||
if (resp.type === "chatCmdError") throw new Error(`${this.name} ${cmd.split(" ", 2).join(" ")}: ${JSON.stringify(resp)}`)
|
||||
return resp
|
||||
}
|
||||
|
||||
waitEvent(pred, timeout, what) {
|
||||
const found = this.events.find(pred)
|
||||
if (found) return Promise.resolve(found)
|
||||
return new Promise((resolve, reject) => {
|
||||
const waiter = {
|
||||
pred,
|
||||
resolve: (resp) => {
|
||||
clearTimeout(timer)
|
||||
resolve(resp)
|
||||
},
|
||||
}
|
||||
this.waiters.push(waiter)
|
||||
const timer = setTimeout(() => {
|
||||
this.waiters = this.waiters.filter((w) => w !== waiter)
|
||||
reject(new Error(`${this.name}: no ${what} event in ${timeout}ms`))
|
||||
}, timeout)
|
||||
})
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.ws.close()
|
||||
this.proc.kill()
|
||||
}
|
||||
}
|
||||
|
||||
async function connectContacts(alice, bob) {
|
||||
const inv = await alice.cmd("/c")
|
||||
await bob.cmd(`/c ${inv.connLinkInvitation.connFullLink}`)
|
||||
const connected = (c) => c.waitEvent((e) => e.type === "contactConnected", START_TIMEOUT, "contactConnected")
|
||||
const [a, b] = await Promise.all([connected(alice), connected(bob)])
|
||||
return [a.contact.contactId, b.contact.contactId]
|
||||
}
|
||||
|
||||
module.exports = {startSmpServer, ChatClient, connectContacts}
|
||||
@@ -0,0 +1,188 @@
|
||||
// Calls between two chat cores connected via a local SMP server, with the test playing the native app:
|
||||
// it passes messages between call.js and the core API as the Android and desktop apps do.
|
||||
const {test} = require("node:test")
|
||||
const assert = require("node:assert/strict")
|
||||
const fs = require("fs")
|
||||
const os = require("os")
|
||||
const path = require("path")
|
||||
const {openPeers, sleep} = require("./harness")
|
||||
const {startSmpServer, ChatClient, connectContacts} = require("./chat")
|
||||
|
||||
const {SIMPLEX_CHAT, SMP_SERVER} = process.env
|
||||
const MEDIA = "audio"
|
||||
const DISCONNECT_DETECTION = 10000
|
||||
const RECONNECT = 20000
|
||||
const SIGNALING = 15000
|
||||
const CORE_CALL_STATUSES = ["connecting", "connected", "disconnected", "failed"]
|
||||
|
||||
async function withChatCall({useWorker = false}, run) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "simplex-call-test-"))
|
||||
const stops = []
|
||||
try {
|
||||
const smp = await startSmpServer(SMP_SERVER, path.join(dir, "smp"))
|
||||
stops.push(smp.stop)
|
||||
const alice = await ChatClient.start(SIMPLEX_CHAT, dir, "alice", smp.address)
|
||||
stops.push(() => alice.stop())
|
||||
const bob = await ChatClient.start(SIMPLEX_CHAT, dir, "bob", smp.address)
|
||||
stops.push(() => bob.stop())
|
||||
const [aliceContact, bobContact] = await connectContacts(alice, bob)
|
||||
const peers = await openPeers({useWorker})
|
||||
stops.push(peers.close)
|
||||
// alice calls bob: a is alice's call.js, b is bob's call.js
|
||||
const call = {...peers, alice, bob, aliceContact, bobContact}
|
||||
connectApp(call.a, alice, aliceContact, peers)
|
||||
connectApp(call.b, bob, bobContact, peers)
|
||||
await startCall(call)
|
||||
await run(call)
|
||||
} finally {
|
||||
for (const stop of stops.reverse()) await stop()
|
||||
fs.rmSync(dir, {recursive: true, force: true})
|
||||
}
|
||||
}
|
||||
|
||||
function connectApp(peer, chat, contactId, {network, signaling}) {
|
||||
peer.onResponse = async (resp) => {
|
||||
switch (resp.type) {
|
||||
case "ice": {
|
||||
const {session, iceCandidates} = await network.routeSignal(peer.name, resp.session, resp.iceCandidates)
|
||||
const extraInfo = JSON.stringify({rtcSession: session, rtcIceCandidates: iceCandidates})
|
||||
signaling.send(() => chat.cmd(`/_call extra @${contactId} ${extraInfo}`))
|
||||
break
|
||||
}
|
||||
case "connection":
|
||||
if (CORE_CALL_STATUSES.includes(resp.state.connectionState)) {
|
||||
await chat.cmd(`/_call status @${contactId} ${resp.state.connectionState}`)
|
||||
}
|
||||
break
|
||||
case "ended":
|
||||
await chat.cmd(`/_call end @${contactId}`).catch(() => {})
|
||||
break
|
||||
}
|
||||
}
|
||||
chat.onEvent = async (event) => {
|
||||
switch (event.type) {
|
||||
case "callExtraInfo":
|
||||
await peer.command({type: "ice", iceCandidates: event.extraInfo.rtcIceCandidates, session: event.extraInfo.rtcSession})
|
||||
break
|
||||
case "callEnded":
|
||||
await peer.command({type: "end"})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function startCall({a, b, network, alice, bob, aliceContact, bobContact}) {
|
||||
const capabilities = await a.command({type: "capabilities", media: MEDIA})
|
||||
assert.equal(capabilities.resp.type, "capabilities")
|
||||
const callType = {media: MEDIA, capabilities: capabilities.resp.capabilities}
|
||||
await alice.cmd(`/_call invite @${aliceContact} ${JSON.stringify(callType)}`)
|
||||
|
||||
const {callInvitation} = await bob.waitEvent((e) => e.type === "callInvitation", SIGNALING, "callInvitation")
|
||||
const offer = await b.command({type: "start", media: MEDIA, aesKey: callInvitation.sharedKey, iceServers: []})
|
||||
assert.equal(offer.resp.type, "offer", JSON.stringify(offer.resp))
|
||||
const offerSignal = await network.routeSignal("B", offer.resp.offer, offer.resp.iceCandidates)
|
||||
const callOffer = {
|
||||
callType: {media: MEDIA, capabilities: offer.resp.capabilities},
|
||||
rtcSession: {rtcSession: offerSignal.session, rtcIceCandidates: offerSignal.iceCandidates},
|
||||
}
|
||||
await bob.cmd(`/_call offer @${bobContact} ${JSON.stringify(callOffer)}`)
|
||||
|
||||
const offered = await alice.waitEvent((e) => e.type === "callOffer", SIGNALING, "callOffer")
|
||||
const answer = await a.command({
|
||||
type: "offer",
|
||||
offer: offered.offer.rtcSession,
|
||||
iceCandidates: offered.offer.rtcIceCandidates,
|
||||
media: MEDIA,
|
||||
aesKey: offered.sharedKey,
|
||||
iceServers: [],
|
||||
})
|
||||
assert.equal(answer.resp.type, "answer", JSON.stringify(answer.resp))
|
||||
const answerSignal = await network.routeSignal("A", answer.resp.answer, answer.resp.iceCandidates)
|
||||
const callAnswer = {rtcSession: answerSignal.session, rtcIceCandidates: answerSignal.iceCandidates}
|
||||
await alice.cmd(`/_call answer @${aliceContact} ${JSON.stringify(callAnswer)}`)
|
||||
|
||||
const answered = await bob.waitEvent((e) => e.type === "callAnswer", SIGNALING, "callAnswer")
|
||||
const ok = await b.command({type: "answer", answer: answered.answer.rtcSession, iceCandidates: answered.answer.rtcIceCandidates})
|
||||
assert.equal(ok.resp.type, "ok", JSON.stringify(ok.resp))
|
||||
await Promise.all([a.waitForState("connected", RECONNECT), b.waitForState("connected", RECONNECT)])
|
||||
}
|
||||
|
||||
async function assertAudioDecoded(a, b) {
|
||||
const before = [await a.receivedAudioEnergy(), await b.receivedAudioEnergy()]
|
||||
await sleep(2000)
|
||||
const after = [await a.receivedAudioEnergy(), await b.receivedAudioEnergy()]
|
||||
assert.ok(after[0] > before[0], `A decoded no audio: ${before[0]} -> ${after[0]}`)
|
||||
assert.ok(after[1] > before[1], `B decoded no audio: ${before[1]} -> ${after[1]}`)
|
||||
}
|
||||
|
||||
async function callItem(chat, contactId) {
|
||||
const r = await chat.cmd(`/_get chat @${contactId} count=1`)
|
||||
return r.chat.chatItems[0].content
|
||||
}
|
||||
|
||||
function restartSessionsReceived(chat) {
|
||||
return chat.events.filter((e) => e.type === "callExtraInfo" && e.extraInfo.rtcSession).length
|
||||
}
|
||||
|
||||
const skip = !(SIMPLEX_CHAT && SMP_SERVER) && "set SIMPLEX_CHAT and SMP_SERVER to chat CLI and SMP server executables"
|
||||
|
||||
test("calls via chat core", {skip}, async (t) => {
|
||||
for (const [app, useWorker] of [
|
||||
["android", false],
|
||||
["desktop", true],
|
||||
]) {
|
||||
await t.test(`${app}: encrypted call reconnects with ICE restart after network path change`, () =>
|
||||
withChatCall({useWorker}, async ({a, b, network, alice, bob, aliceContact, bobContact}) => {
|
||||
assert.ok(a.messages.find((m) => m.resp.type === "capabilities").resp.capabilities.encryption, "call is not encrypted")
|
||||
assert.equal(await a.page.evaluate(() => !!activeCall.worker), useWorker, "encryption worker")
|
||||
await assertAudioDecoded(a, b)
|
||||
assert.deepEqual(await callItem(alice, aliceContact), {type: "sndCall", status: "progress", duration: 0})
|
||||
const connectedAt = Date.now()
|
||||
|
||||
const from = [a.messages.length, b.messages.length]
|
||||
network.down = true
|
||||
network.changePath()
|
||||
await Promise.all([
|
||||
a.waitForState("reconnecting", DISCONNECT_DETECTION, from[0]),
|
||||
b.waitForState("reconnecting", DISCONNECT_DETECTION, from[1]),
|
||||
])
|
||||
await sleep(3000)
|
||||
network.down = false
|
||||
await Promise.all([a.waitForState("connected", RECONNECT, from[0]), b.waitForState("connected", RECONNECT, from[1])])
|
||||
await assertAudioDecoded(a, b)
|
||||
assert.ok(restartSessionsReceived(alice) > 0, "alice received no ICE restart offer")
|
||||
assert.ok(restartSessionsReceived(bob) > 0, "bob received no ICE restart answer")
|
||||
assert.equal((await callItem(alice, aliceContact)).status, "progress")
|
||||
assert.equal((await callItem(bob, bobContact)).status, "progress")
|
||||
|
||||
await alice.cmd(`/_call end @${aliceContact}`)
|
||||
await a.command({type: "end"})
|
||||
await bob.waitEvent((e) => e.type === "callEnded", SIGNALING, "callEnded")
|
||||
const minDuration = Math.floor((Date.now() - connectedAt) / 1000) - 1
|
||||
const aliceItem = await callItem(alice, aliceContact)
|
||||
assert.equal(aliceItem.status, "ended")
|
||||
assert.ok(aliceItem.duration >= minDuration, `call duration ${aliceItem.duration} < ${minDuration}`)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
await t.test("call ends in both chats when reconnection times out", () =>
|
||||
withChatCall({}, async ({a, b, network, alice, bob, aliceContact, bobContact}) => {
|
||||
await Promise.all([a.setTimeouts({reconnectTimeout: 5000}), b.setTimeouts({reconnectTimeout: 5000})])
|
||||
network.down = true
|
||||
// the first party to time out ends the call, and the other party ends it on "callEnded" event
|
||||
const endedBy = (peer, other) => peer.waitForEnded(DISCONNECT_DETECTION + 10000).then(() => other)
|
||||
const other = await Promise.any([endedBy(a, bob), endedBy(b, alice)])
|
||||
await other.waitEvent((e) => e.type === "callEnded", SIGNALING, "callEnded")
|
||||
network.down = false
|
||||
for (const [chat, contactId] of [
|
||||
[alice, aliceContact],
|
||||
[bob, bobContact],
|
||||
]) {
|
||||
const item = await callItem(chat, contactId)
|
||||
assert.equal(item.status, "ended", `${chat.name} call item: ${JSON.stringify(item)}`)
|
||||
assert.ok(item.duration >= 5, `${chat.name} call duration ${item.duration}`)
|
||||
}
|
||||
})
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,317 @@
|
||||
// Runs two headless Chromium peers with call.js; the test plays the native app and the chat core.
|
||||
// Media can only flow through UDP relays controlled by the test, so outages and path changes are simulated
|
||||
// by dropping packets in the relays.
|
||||
const http = require("http")
|
||||
const fs = require("fs")
|
||||
const path = require("path")
|
||||
const dgram = require("dgram")
|
||||
const {chromium} = require("playwright-core")
|
||||
const LZString = require("lz-string")
|
||||
|
||||
const ROOT = path.join(__dirname, "..")
|
||||
const PAGE_FILES = {
|
||||
"/android/call.html": path.join(ROOT, "src/android/call.html"),
|
||||
"/android/style.css": path.join(ROOT, "src/android/style.css"),
|
||||
"/call.js": path.join(ROOT, "dist/call.js"),
|
||||
"/lz-string.min.js": path.join(ROOT, "node_modules/lz-string/libs/lz-string.min.js"),
|
||||
}
|
||||
const CONTENT_TYPES = {".html": "text/html", ".js": "text/javascript", ".css": "text/css"}
|
||||
|
||||
const startTime = Date.now()
|
||||
function log(...args) {
|
||||
if (process.env.VERBOSE) console.log(((Date.now() - startTime) / 1000).toFixed(1).padStart(6), ...args)
|
||||
}
|
||||
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
|
||||
const decode = (s) => JSON.parse(LZString.decompressFromBase64(s))
|
||||
const encode = (x) => LZString.compressToBase64(JSON.stringify(x))
|
||||
|
||||
function servePage() {
|
||||
const server = http.createServer((req, res) => {
|
||||
const file = PAGE_FILES[new URL(req.url, "http://localhost").pathname]
|
||||
if (!file) return res.writeHead(404).end()
|
||||
res.writeHead(200, {"Content-Type": CONTENT_TYPES[path.extname(file)]}).end(fs.readFileSync(file))
|
||||
})
|
||||
return new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(server)))
|
||||
}
|
||||
|
||||
function bindUdp(ip) {
|
||||
return new Promise((resolve) => {
|
||||
const socket = dgram.createSocket("udp4")
|
||||
socket.bind(0, ip, () => resolve(socket))
|
||||
})
|
||||
}
|
||||
|
||||
// Relays one ICE generation: each peer is told that the other peer is at the relay socket.
|
||||
class Relay {
|
||||
constructor(network, ip) {
|
||||
this.network = network
|
||||
this.ip = ip
|
||||
this.ufrag = {A: undefined, B: undefined}
|
||||
this.real = {A: undefined, B: undefined}
|
||||
this.dead = false
|
||||
this.forwarded = 0
|
||||
}
|
||||
|
||||
async open() {
|
||||
this.sockets = {A: await bindUdp(this.ip), B: await bindUdp(this.ip)}
|
||||
this.sockets.A.on("message", (msg, from) => this.forward(msg, from, "B", "A"))
|
||||
this.sockets.B.on("message", (msg, from) => this.forward(msg, from, "A", "B"))
|
||||
}
|
||||
|
||||
// socket A is advertised to B as A's address, so packets on it come from B and go to A via socket B
|
||||
forward(msg, from, sender, receiver) {
|
||||
if (this.dead || this.network.down) return
|
||||
this.real[sender] = from
|
||||
const to = this.real[receiver]
|
||||
if (!to) return
|
||||
this.forwarded++
|
||||
this.sockets[sender].send(msg, to.port, to.address)
|
||||
}
|
||||
|
||||
port(side) {
|
||||
return this.sockets[side].address().port
|
||||
}
|
||||
|
||||
close() {
|
||||
this.sockets.A.close()
|
||||
this.sockets.B.close()
|
||||
}
|
||||
}
|
||||
|
||||
class Network {
|
||||
constructor() {
|
||||
this.down = false
|
||||
this.relays = []
|
||||
}
|
||||
|
||||
// Returns the rewritten candidate, or undefined if the candidate cannot be relayed.
|
||||
async route(side, candidate) {
|
||||
const [, component, protocol, , ip, port, , type] = candidate.candidate.split(" ")
|
||||
if (protocol.toLowerCase() !== "udp" || type !== "host" || ip.includes(":") || component !== "1") return
|
||||
const ufrag = candidate.usernameFragment
|
||||
if (this.relays.some((r) => r.ufrag[side] === ufrag)) return
|
||||
let relay = this.relays.find((r) => !r.dead && r.ufrag[side] === undefined)
|
||||
if (!relay) {
|
||||
relay = new Relay(this, ip)
|
||||
await relay.open()
|
||||
this.relays.push(relay)
|
||||
}
|
||||
relay.ufrag[side] = ufrag
|
||||
relay.real[side] = {address: ip, port: +port}
|
||||
log(`relay ${this.relays.indexOf(relay)}: ${side} ${ip}:${port} -> ${relay.ip}:${relay.port(side)}`)
|
||||
const parts = candidate.candidate.split(" ")
|
||||
parts[4] = relay.ip
|
||||
parts[5] = String(relay.port(side))
|
||||
return {...candidate, candidate: parts.join(" ")}
|
||||
}
|
||||
|
||||
// Takes and returns compressed session description and candidates, as in signaling messages
|
||||
async routeSignal(side, session, iceCandidates) {
|
||||
// candidates in SDP would connect the peers directly, bypassing the relays
|
||||
if (session && decode(session).sdp.includes("a=candidate:")) throw new Error(`${side}: SDP has ICE candidates`)
|
||||
const routed = []
|
||||
for (const c of decode(iceCandidates)) {
|
||||
const r = c?.candidate && (await this.route(side, c))
|
||||
if (r) routed.push(r)
|
||||
}
|
||||
return {session, iceCandidates: encode(routed)}
|
||||
}
|
||||
|
||||
// Addresses of all current ICE generations become unreachable, as when a device switches networks.
|
||||
changePath() {
|
||||
for (const relay of this.relays) relay.dead = true
|
||||
}
|
||||
|
||||
close() {
|
||||
for (const relay of this.relays) relay.close()
|
||||
}
|
||||
}
|
||||
|
||||
class Peer {
|
||||
constructor(name, page, {ignoresIceRestart = false} = {}) {
|
||||
this.name = name
|
||||
this.page = page
|
||||
this.ignoresIceRestart = ignoresIceRestart
|
||||
this.messages = []
|
||||
this.waiters = []
|
||||
this.onResponse = async () => {}
|
||||
}
|
||||
|
||||
async open(url) {
|
||||
await this.page.exposeFunction("sendToTest", (json) => this.receive(JSON.parse(json)))
|
||||
this.page.on("console", (m) => log(`${this.name} console:`, m.text().slice(0, 200)))
|
||||
this.page.on("pageerror", (e) => log(`${this.name} page error:`, e.message))
|
||||
await this.page.goto(url)
|
||||
await this.page.evaluate(() => {
|
||||
sendMessageToNative = (msg) => window.sendToTest(JSON.stringify(msg))
|
||||
})
|
||||
}
|
||||
|
||||
receive(msg) {
|
||||
const resp = msg.resp
|
||||
log(`${this.name} <-`, resp.type, resp.state?.connectionState ?? "", resp.session ? "with session" : "")
|
||||
this.messages.push(msg)
|
||||
if (!msg.command) this.onResponse(resp).catch((e) => log(`${this.name} response error`, e))
|
||||
this.waiters = this.waiters.filter((w) => !(w.pred(msg) && (w.resolve(msg), true)))
|
||||
}
|
||||
|
||||
connectionStates() {
|
||||
return this.messages.filter((m) => m.resp.type === "connection").map((m) => m.resp.state.connectionState)
|
||||
}
|
||||
|
||||
sessionsSent() {
|
||||
return this.messages.filter((m) => m.resp.type === "ice" && !m.command && m.resp.session).length
|
||||
}
|
||||
|
||||
ended() {
|
||||
return this.messages.some((m) => m.resp.type === "ended")
|
||||
}
|
||||
|
||||
waitFor(pred, timeout, what) {
|
||||
const found = this.messages.find(pred)
|
||||
if (found) return Promise.resolve(found)
|
||||
return new Promise((resolve, reject) => {
|
||||
const waiter = {
|
||||
pred,
|
||||
resolve: (msg) => {
|
||||
clearTimeout(timer)
|
||||
resolve(msg)
|
||||
},
|
||||
}
|
||||
this.waiters.push(waiter)
|
||||
const timer = setTimeout(() => {
|
||||
this.waiters = this.waiters.filter((w) => w !== waiter)
|
||||
reject(new Error(`${this.name}: no ${what} in ${timeout}ms, connection states: ${this.connectionStates()}`))
|
||||
}, timeout)
|
||||
})
|
||||
}
|
||||
|
||||
waitForState(state, timeout, after = 0) {
|
||||
const pred = (m) => m.resp.type === "connection" && m.resp.state.connectionState === state && this.messages.indexOf(m) >= after
|
||||
return this.waitFor(pred, timeout, `"${state}" state`)
|
||||
}
|
||||
|
||||
waitForEnded(timeout) {
|
||||
return this.waitFor((m) => m.resp.type === "ended", timeout, "ended")
|
||||
}
|
||||
|
||||
command(command) {
|
||||
log(`${this.name} ->`, command.type, command.session ? "with session" : "")
|
||||
return this.page.evaluate((c) => processCommand({corrId: 0, command: c}), command)
|
||||
}
|
||||
|
||||
async setTimeouts(timeouts) {
|
||||
await this.page.evaluate((t) => {
|
||||
if (t.reconnectTimeout) reconnectTimeout = t.reconnectTimeout
|
||||
if (t.iceRestartInterval) iceRestartInterval = t.iceRestartInterval
|
||||
}, timeouts)
|
||||
}
|
||||
|
||||
async receivedBytes() {
|
||||
return (await this.inboundAudio()).bytesReceived
|
||||
}
|
||||
|
||||
// energy grows only when received audio is decoded, i.e. it is decrypted with the right key
|
||||
async receivedAudioEnergy() {
|
||||
return (await this.inboundAudio()).totalAudioEnergy
|
||||
}
|
||||
|
||||
inboundAudio() {
|
||||
return this.page.evaluate(async () => {
|
||||
let audio
|
||||
;(await activeCall.connection.getStats()).forEach((s) => {
|
||||
if (s.type === "inbound-rtp" && s.kind === "audio" && (!audio || s.bytesReceived > audio.bytesReceived)) audio = s
|
||||
})
|
||||
return {bytesReceived: audio?.bytesReceived ?? 0, totalAudioEnergy: audio?.totalAudioEnergy ?? 0}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Signaling stops while the network is down and resumes in order, as messages queued in SMP servers.
|
||||
class Signaling {
|
||||
constructor(network) {
|
||||
this.network = network
|
||||
this.queue = Promise.resolve()
|
||||
this.closed = false
|
||||
}
|
||||
|
||||
send(deliver) {
|
||||
this.queue = this.queue.then(async () => {
|
||||
while (this.network.down && !this.closed) await sleep(100)
|
||||
if (!this.closed) await deliver().catch((e) => log("signaling error", e.message))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// useWorker encrypts media in a worker with RTCRtpScriptTransform, as the desktop app does
|
||||
async function openPeers({answerer = {}, useWorker = false} = {}) {
|
||||
const server = await servePage()
|
||||
const url = `http://localhost:${server.address().port}/android/call.html`
|
||||
const browser = await chromium.launch({
|
||||
executablePath: process.env.CHROME,
|
||||
args: [
|
||||
"--use-fake-ui-for-media-stream",
|
||||
"--use-fake-device-for-media-stream",
|
||||
"--disable-features=WebRtcHideLocalIpsWithMdns",
|
||||
"--autoplay-policy=no-user-gesture-required",
|
||||
],
|
||||
})
|
||||
const newPeer = async (name, opts) => {
|
||||
const context = await browser.newContext({permissions: ["camera", "microphone"]})
|
||||
const peer = new Peer(name, await context.newPage(), opts)
|
||||
await peer.open(url)
|
||||
if (useWorker) await peer.page.evaluate(() => (useWorker = true))
|
||||
return peer
|
||||
}
|
||||
// A sends the call invitation and B accepts it, so B sends the WebRTC offer, as "start" command in the apps
|
||||
const a = await newPeer("A", answerer)
|
||||
const b = await newPeer("B")
|
||||
const network = new Network()
|
||||
const signaling = new Signaling(network)
|
||||
const close = async () => {
|
||||
signaling.closed = true
|
||||
await browser.close()
|
||||
network.close()
|
||||
server.close()
|
||||
}
|
||||
return {a, b, network, signaling, close}
|
||||
}
|
||||
|
||||
// The test relays signaling between the peers directly, as the apps and the chat core would do.
|
||||
async function startCall({media = "audio", answerer = {}} = {}) {
|
||||
const peers = await openPeers({answerer})
|
||||
const {a, b, network, signaling, close} = peers
|
||||
const relayIce = (from, to) => async (resp) => {
|
||||
if (resp.type !== "ice") return
|
||||
const {session, iceCandidates} = await network.routeSignal(from.name, resp.session, resp.iceCandidates)
|
||||
signaling.send(() => to.command({type: "ice", iceCandidates, session: to.ignoresIceRestart ? undefined : session}))
|
||||
}
|
||||
a.onResponse = relayIce(a, b)
|
||||
b.onResponse = relayIce(b, a)
|
||||
|
||||
try {
|
||||
await a.command({type: "capabilities", media})
|
||||
const offer = await b.command({type: "start", media, iceServers: []})
|
||||
if (offer.resp.type !== "offer") throw new Error(`B start: ${JSON.stringify(offer.resp)}`)
|
||||
const offerSignal = await network.routeSignal("B", offer.resp.offer, offer.resp.iceCandidates)
|
||||
const answer = await a.command({
|
||||
type: "offer",
|
||||
offer: offerSignal.session,
|
||||
iceCandidates: offerSignal.iceCandidates,
|
||||
media,
|
||||
iceServers: [],
|
||||
})
|
||||
if (answer.resp.type !== "answer") throw new Error(`A offer: ${JSON.stringify(answer.resp)}`)
|
||||
const answerSignal = await network.routeSignal("A", answer.resp.answer, answer.resp.iceCandidates)
|
||||
const ok = await b.command({type: "answer", answer: answerSignal.session, iceCandidates: answerSignal.iceCandidates})
|
||||
if (ok.resp.type !== "ok") throw new Error(`B answer: ${JSON.stringify(ok.resp)}`)
|
||||
await Promise.all([a.waitForState("connected", 20000), b.waitForState("connected", 20000)])
|
||||
} catch (e) {
|
||||
await close()
|
||||
throw e
|
||||
}
|
||||
return peers
|
||||
}
|
||||
|
||||
module.exports = {openPeers, startCall, sleep, log}
|
||||
@@ -0,0 +1,95 @@
|
||||
const {test} = require("node:test")
|
||||
const assert = require("node:assert/strict")
|
||||
const {startCall, sleep} = require("./harness")
|
||||
|
||||
// Chrome reports "disconnected" about 6 seconds after packets stop
|
||||
const DISCONNECT_DETECTION = 10000
|
||||
const RECONNECT = 20000
|
||||
|
||||
async function assertMediaFlows(a, b) {
|
||||
const before = [await a.receivedBytes(), await b.receivedBytes()]
|
||||
await sleep(2000)
|
||||
const after = [await a.receivedBytes(), await b.receivedBytes()]
|
||||
assert.ok(after[0] > before[0], `A received no media: ${before[0]} -> ${after[0]}`)
|
||||
assert.ok(after[1] > before[1], `B received no media: ${before[1]} -> ${after[1]}`)
|
||||
}
|
||||
|
||||
async function assertReconnects(a, b, outage) {
|
||||
const from = [a.messages.length, b.messages.length]
|
||||
await Promise.all([
|
||||
a.waitForState("reconnecting", DISCONNECT_DETECTION, from[0]),
|
||||
b.waitForState("reconnecting", DISCONNECT_DETECTION, from[1]),
|
||||
])
|
||||
await outage()
|
||||
await Promise.all([a.waitForState("connected", RECONNECT, from[0]), b.waitForState("connected", RECONNECT, from[1])])
|
||||
await assertMediaFlows(a, b)
|
||||
assert.ok(!a.ended() && !b.ended(), "call ended")
|
||||
assert.ok(!a.connectionStates().includes("disconnected"), `A states: ${a.connectionStates()}`)
|
||||
assert.ok(!b.connectionStates().includes("disconnected"), `B states: ${b.connectionStates()}`)
|
||||
}
|
||||
|
||||
async function withCall(opts, run) {
|
||||
const call = await startCall(opts)
|
||||
try {
|
||||
await run(call)
|
||||
} finally {
|
||||
await call.close()
|
||||
}
|
||||
}
|
||||
|
||||
test("call reconnection", {concurrency: true}, async (t) => {
|
||||
await t.test("recovers after a brief outage", () =>
|
||||
withCall({}, async ({a, b, network}) => {
|
||||
await assertMediaFlows(a, b)
|
||||
network.down = true
|
||||
await assertReconnects(a, b, async () => {
|
||||
network.down = false
|
||||
})
|
||||
})
|
||||
)
|
||||
|
||||
await t.test("recovers after an outage longer than ICE consent timeout", () =>
|
||||
withCall({}, async ({a, b, network}) => {
|
||||
network.down = true
|
||||
await assertReconnects(a, b, async () => {
|
||||
await sleep(15000)
|
||||
network.down = false
|
||||
})
|
||||
assert.ok(b.sessionsSent() > 0, "offerer did not restart ICE")
|
||||
assert.ok(a.sessionsSent() > 0, "answerer did not answer ICE restart")
|
||||
})
|
||||
)
|
||||
|
||||
await t.test("recovers when the network path changes", () =>
|
||||
withCall({}, async ({a, b, network}) => {
|
||||
network.down = true
|
||||
network.changePath()
|
||||
await assertReconnects(a, b, async () => {
|
||||
network.down = false
|
||||
})
|
||||
const relay = network.relays[network.relays.length - 1]
|
||||
assert.ok(!relay.dead && relay.forwarded > 0, "media does not use the new path")
|
||||
})
|
||||
)
|
||||
|
||||
await t.test("ends the call when reconnection times out", () =>
|
||||
withCall({}, async ({a, b, network}) => {
|
||||
await Promise.all([a.setTimeouts({reconnectTimeout: 5000}), b.setTimeouts({reconnectTimeout: 5000})])
|
||||
network.down = true
|
||||
await Promise.all([a.waitForState("reconnecting", DISCONNECT_DETECTION), b.waitForState("reconnecting", DISCONNECT_DETECTION)])
|
||||
await Promise.all([a.waitForEnded(10000), b.waitForEnded(10000)])
|
||||
assert.deepEqual(a.connectionStates().slice(-2), ["reconnecting", "disconnected"])
|
||||
assert.deepEqual(b.connectionStates().slice(-2), ["reconnecting", "disconnected"])
|
||||
})
|
||||
)
|
||||
|
||||
await t.test("ends the call when the peer does not support ICE restart", () =>
|
||||
withCall({answerer: {ignoresIceRestart: true}}, async ({a, b, network}) => {
|
||||
await Promise.all([a.setTimeouts({reconnectTimeout: 10000}), b.setTimeouts({reconnectTimeout: 10000})])
|
||||
network.changePath()
|
||||
await Promise.all([a.waitForEnded(DISCONNECT_DETECTION + 15000), b.waitForEnded(DISCONNECT_DETECTION + 15000)])
|
||||
assert.ok(b.sessionsSent() > 0, "offerer did not restart ICE")
|
||||
assert.equal(a.sessionsSent(), 0)
|
||||
})
|
||||
)
|
||||
})
|
||||
Reference in New Issue
Block a user