ios: CallKit integrated with app lock and screen protect (#2007)

* ios: CallKit integrated with app lock and screen protect

* better lock mechanics

* background color

* logs

* refactor, revert auth changes

* additional state variable to allow connecting call

* fix lock screen, public logs

* show callkit option without dev tools

---------

Co-authored-by: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com>

apps/ios/Shared/ContentView.swift

@@ -13,8 +13,10 @@ struct ContentView: View {
     @EnvironmentObject var chatModel: ChatModel
     @ObservedObject var alertManager = AlertManager.shared
     @ObservedObject var callController = CallController.shared
+    @Environment(\.colorScheme) var colorScheme
     @Binding var doAuthenticate: Bool
     @Binding var userAuthorized: Bool?
+    @Binding var canConnectCall: Bool
     @AppStorage(DEFAULT_SHOW_LA_NOTICE) private var prefShowLANotice = false
     @AppStorage(DEFAULT_LA_NOTICE_SHOWN) private var prefLANoticeShown = false
     @AppStorage(DEFAULT_PERFORM_LA) private var prefPerformLA = false
@@ -24,22 +26,29 @@ struct ContentView: View {
 
     var body: some View {
         ZStack {
+            if chatModel.showCallView, let call = chatModel.activeCall {
+                ActiveCallView(call: call, userAuthorized: $userAuthorized, canConnectCall: $canConnectCall)
+            }
             if prefPerformLA && userAuthorized != true {
+                Rectangle().fill(colorScheme == .dark ? .black : .white)
+                    .frame(maxWidth: .infinity, maxHeight: .infinity)
+                    .onTapGesture(perform: {})
                 Button(action: runAuthenticate) { Label("Unlock", systemImage: "lock") }
             } else if let status = chatModel.chatDbStatus, status != .ok {
                 DatabaseErrorView(status: status)
             } else if !chatModel.v3DBMigration.startChat {
                 MigrateToAppGroupView()
-            } else if let step = chatModel.onboardingStage  {
+            } else if let step = chatModel.onboardingStage, (!chatModel.showCallView || chatModel.activeCall == nil)  {
                 if case .onboardingComplete = step,
                    chatModel.currentUser != nil {
-                    mainView().privacySensitive(protectScreen)
+                    mainView()
                 } else {
                     OnboardingView(onboarding: step)
                 }
             }
         }
         .onAppear {
+            logger.debug("ContentView: canConnectCall \(canConnectCall), doAuthenticate \(doAuthenticate)")
             if doAuthenticate { runAuthenticate() }
         }
         .onChange(of: doAuthenticate) { _ in if doAuthenticate { runAuthenticate() } }
@@ -48,7 +57,7 @@ struct ContentView: View {
 
     private func mainView() -> some View {
         ZStack(alignment: .top) {
-            ChatListView()
+            ChatListView().privacySensitive(protectScreen)
             .onAppear {
                 NtfManager.shared.requestAuthorization(
                     onDeny: {
@@ -75,31 +84,31 @@ struct ContentView: View {
             .sheet(isPresented: $showWhatsNew) {
                 WhatsNewView()
             }
-            if chatModel.showCallView, let call = chatModel.activeCall {
-                ActiveCallView(call: call)
-            }
             IncomingCallView()
         }
-        .onContinueUserActivity("INStartCallIntent", perform: processUserActivity)
-        .onContinueUserActivity("INStartAudioCallIntent", perform: processUserActivity)
-        .onContinueUserActivity("INStartVideoCallIntent", perform: processUserActivity)
+//        .onContinueUserActivity("INStartCallIntent", perform: processUserActivity)
+//        .onContinueUserActivity("INStartAudioCallIntent", perform: processUserActivity)
+//        .onContinueUserActivity("INStartVideoCallIntent", perform: processUserActivity)
     }
 
-    private func processUserActivity(_ activity: NSUserActivity) {
-        let callToContact = { (contactId: ChatId?, mediaType: CallMediaType) in
-            if let chatInfo = chatModel.chats.first(where: { $0.id == contactId })?.chatInfo,
-                case let .direct(contact) = chatInfo {
-                CallController.shared.startCall(contact, mediaType)
-            }
-        }
-        if let intent = activity.interaction?.intent as? INStartCallIntent {
-            callToContact(intent.contacts?.first?.personHandle?.value, .audio)
-        } else if let intent = activity.interaction?.intent as? INStartAudioCallIntent {
-            callToContact(intent.contacts?.first?.personHandle?.value, .audio)
-        } else if let intent = activity.interaction?.intent as? INStartVideoCallIntent {
-            callToContact(intent.contacts?.first?.personHandle?.value, .video)
-        }
-    }
+//    private func processUserActivity(_ activity: NSUserActivity) {
+//        let intent = activity.interaction?.intent
+//        if let contacts = (intent as? INStartCallIntent)?.contacts {
+//            callToContact(contacts, .audio)
+//        } else if let contacts = (intent as? INStartAudioCallIntent)?.contacts {
+//            callToContact(contacts, .audio)
+//        } else if let contacts = (intent as? INStartVideoCallIntent)?.contacts {
+//            callToContact(contacts, .video)
+//        }
+//    }
+//
+//    private func callToContact(_ contacts: [INPerson], _ mediaType: CallMediaType) {
+//        if let contactId = contacts.first?.personHandle?.value,
+//           let chatInfo = chatModel.chats.first(where: { $0.id == contactId })?.chatInfo,
+//           case let .direct(contact) = chatInfo {
+//            CallController.shared.startCall(contact, mediaType)
+//        }
+//    }
 
     private func runAuthenticate() {
         if !prefPerformLA {
@@ -118,10 +127,12 @@ struct ContentView: View {
             switch (laResult) {
             case .success:
                 userAuthorized = true
+                canConnectCall = true
             case .failed:
                 break
             case .unavailable:
                 userAuthorized = true
+                canConnectCall = true
                 prefPerformLA = false
                 AlertManager.shared.showAlert(laUnavailableTurningOffAlert())
             }

apps/ios/Shared/SimpleXApp.swift

@@ -20,6 +20,7 @@ struct SimpleXApp: App {
     @AppStorage(DEFAULT_PERFORM_LA) private var prefPerformLA = false
     @State private var userAuthorized: Bool?
     @State private var doAuthenticate = false
+    @State private var canConnectCall = false
     @State private var enteredBackground: TimeInterval? = nil
 
     init() {
@@ -34,7 +35,7 @@ struct SimpleXApp: App {
 
     var body: some Scene {
         return WindowGroup {
-            ContentView(doAuthenticate: $doAuthenticate, userAuthorized: $userAuthorized)
+            ContentView(doAuthenticate: $doAuthenticate, userAuthorized: $userAuthorized, canConnectCall: $canConnectCall)
                 .environmentObject(chatModel)
                 .onOpenURL { url in
                     logger.debug("ContentView.onOpenURL: \(url)")
@@ -60,6 +61,7 @@ struct SimpleXApp: App {
                             enteredBackground = ProcessInfo.processInfo.systemUptime
                         }
                         doAuthenticate = false
+                        canConnectCall = false
                         NtfManager.shared.setNtfBadgeCount(chatModel.totalUnreadCountForAllUsers())
                     case .active:
                         CallController.shared.onEndCall = nil
@@ -67,9 +69,12 @@ struct SimpleXApp: App {
                         startChatAndActivate()
                         if appState.inactive && chatModel.chatRunning == true {
                             updateChats()
-                            updateCallInvitations()
+                            if !chatModel.showCallView && !CallController.shared.hasActiveCalls() {
+                                updateCallInvitations()
+                            }
                         }
                         doAuthenticate = authenticationExpired()
+                        canConnectCall = !(doAuthenticate && prefPerformLA)
                     default:
                         break
                     }

apps/ios/Shared/Views/Call/ActiveCallView.swift

@@ -12,7 +12,10 @@ import SimpleXChat
 
 struct ActiveCallView: View {
     @EnvironmentObject var m: ChatModel
+    @Environment(\.scenePhase) var scenePhase
     @ObservedObject var call: Call
+    @Binding var userAuthorized: Bool?
+    @Binding var canConnectCall: Bool
     @State private var client: WebRTCClient? = nil
     @State private var activeCall: WebRTCClient.Call? = nil
     @State private var localRendererAspectRatio: CGFloat? = nil
@@ -36,12 +39,19 @@ struct ActiveCallView: View {
             }
         }
         .onAppear {
-            if client == nil {
-                client = WebRTCClient($activeCall, { msg in await MainActor.run { processRtcMessage(msg: msg) } }, $localRendererAspectRatio)
-                sendCommandToClient()
-            }
+            logger.debug("ActiveCallView: appear client is nil \(client == nil), userAuthorized \(userAuthorized.debugDescription, privacy: .public), scenePhase \(String(describing: scenePhase), privacy: .public)")
+            createWebRTCClient()
+        }
+        .onChange(of: userAuthorized) { _ in
+            logger.debug("ActiveCallView: userAuthorized changed to \(userAuthorized.debugDescription, privacy: .public)")
+            createWebRTCClient()
+        }
+        .onChange(of: canConnectCall) { _ in
+            logger.debug("ActiveCallView: canConnectCall changed to \(canConnectCall, privacy: .public)")
+            createWebRTCClient()
         }
         .onDisappear {
+            logger.debug("ActiveCallView: disappear")
             client?.endCall()
         }
         .onChange(of: m.callCommand) { _ in sendCommandToClient()}
@@ -49,6 +59,13 @@ struct ActiveCallView: View {
         .preferredColorScheme(.dark)
     }
 
+    private func createWebRTCClient() {
+        if client == nil && ((userAuthorized == true && canConnectCall) || scenePhase == .background) {
+            client = WebRTCClient($activeCall, { msg in await MainActor.run { processRtcMessage(msg: msg) } }, $localRendererAspectRatio)
+            sendCommandToClient()
+        }
+    }
+
     private func sendCommandToClient() {
         if call == m.activeCall,
            m.activeCall != nil,

apps/ios/Shared/Views/Call/CallController.swift

@@ -21,9 +21,9 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
 
     private let provider = CXProvider(configuration: {
         let configuration = CXProviderConfiguration()
-        configuration.supportsVideo = false
+        configuration.supportsVideo = true
         configuration.supportedHandleTypes = [.generic]
-        configuration.includesCallsInRecents = UserDefaults.standard.bool(forKey: DEFAULT_CALL_KIT_CALLS_IN_RECENTS)
+        configuration.includesCallsInRecents = false // UserDefaults.standard.bool(forKey: DEFAULT_CALL_KIT_CALLS_IN_RECENTS)
         configuration.maximumCallGroups = 1
         configuration.maximumCallsPerCallGroup = 1
         configuration.iconTemplateImageData = UIImage(named: "icon-transparent")?.pngData()
@@ -98,6 +98,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
 
     func provider(_ provider: CXProvider, didActivate audioSession: AVAudioSession) {
         print("received", #function)
+        logger.debug("CallController: activating audioSession and audio in WebRTCClient")
         RTCAudioSession.sharedInstance().audioSessionDidActivate(audioSession)
         RTCAudioSession.sharedInstance().isAudioEnabled = true
         do {
@@ -113,10 +114,12 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
 
     func provider(_ provider: CXProvider, didDeactivate audioSession: AVAudioSession) {
         print("received", #function)
+        logger.debug("CallController: deactivating audioSession and audio in WebRTCClient")
         RTCAudioSession.sharedInstance().audioSessionDidDeactivate(audioSession)
         RTCAudioSession.sharedInstance().isAudioEnabled = false
         do {
             try audioSession.setActive(false)
+            logger.debug("audioSession deactivated")
         } catch {
             print(error)
             logger.error("failed deactivating audio session")
@@ -125,6 +128,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
         // see `.onChange(of: scenePhase)` in SimpleXApp
         DispatchQueue.main.asyncAfter(deadline: .now() + 3) { [weak self] in
             if ChatModel.shared.activeCall == nil {
+                logger.debug("CallController: calling callback onEndCall which is \(self?.onEndCall == nil ? "nil" : "non-nil", privacy: .public)")
                 self?.onEndCall?()
             }
         }
@@ -136,14 +140,17 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
     }
 
     func pushRegistry(_ registry: PKPushRegistry, didReceiveIncomingPushWith payload: PKPushPayload, for type: PKPushType, completion: @escaping () -> Void) {
+        logger.debug("CallController: did receive push with type \(type.rawValue, privacy: .public)")
         if type == .voIP {
             if (!ChatModel.shared.chatInitialized) {
+                logger.debug("CallController: initializing chat and returning")
                 initChatAndMigrate()
                 startChatAndActivate()
                 CallController.shared.onEndCall = { terminateChat() }
                 // CallKit will be called from different place, see SimpleXAPI.startChat()
                 return
             } else {
+                logger.debug("CallController: starting chat (already initialized)")
                 startChatAndActivate()
                 CallController.shared.onEndCall = {
                     suspendChat()
@@ -162,6 +169,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
                 callUpdate.remoteHandle = CXHandle(type: .generic, value: contactId)
                 callUpdate.localizedCallerName = displayName
                 callUpdate.hasVideo = media == CallMediaType.video.rawValue
+                logger.debug("CallController: reporting incoming call directly to CallKit")
                 CallController.shared.provider.reportNewIncomingCall(with: uuid, update: callUpdate, completion: { error in
                     if error != nil {
                         ChatModel.shared.callInvitations.removeValue(forKey: contactId)
@@ -174,7 +182,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
     }
 
     func reportNewIncomingCall(invitation: RcvCallInvitation, completion: @escaping (Error?) -> Void) {
-        logger.debug("CallController.reportNewIncomingCall, UUID=\(String(describing: invitation.callkitUUID))")
+        logger.debug("CallController.reportNewIncomingCall, UUID=\(String(describing: invitation.callkitUUID), privacy: .public)")
         if CallController.useCallKit(), let uuid = invitation.callkitUUID {
             let update = CXCallUpdate()
             update.remoteHandle = CXHandle(type: .generic, value: invitation.contact.id)
@@ -190,6 +198,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
     }
 
     func reportIncomingCall(call: Call, connectedAt dateConnected: Date?) {
+        logger.debug("CallController: reporting incoming call connected")
         if CallController.useCallKit() {
             // Fulfilling this action only after connect, otherwise there are no audio and mic on lockscreen
             fulfillOnConnect?.fulfill()
@@ -198,12 +207,14 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
     }
 
     func reportOutgoingCall(call: Call, connectedAt dateConnected: Date?) {
+        logger.debug("CallController: reporting outgoing call connected")
         if CallController.useCallKit(), let uuid = call.callkitUUID {
             provider.reportOutgoingCall(with: uuid, connectedAt: dateConnected)
         }
     }
 
     func reportCallRemoteEnded(invitation: RcvCallInvitation) {
+        logger.debug("CallController: reporting remote ended")
         if CallController.useCallKit(), let uuid = invitation.callkitUUID {
             provider.reportCall(with: uuid, endedAt: nil, reason: .remoteEnded)
         } else if invitation.contact.id == activeCallInvitation?.contact.id {
@@ -212,6 +223,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
     }
 
     func reportCallRemoteEnded(call: Call) {
+        logger.debug("CallController: reporting remote ended")
         if CallController.useCallKit(), let uuid = call.callkitUUID {
             provider.reportCall(with: uuid, endedAt: nil, reason: .remoteEnded)
         }
@@ -241,6 +253,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
     }
 
     func answerCall(invitation: RcvCallInvitation) {
+        logger.debug("CallController: answering a call")
         if CallController.useCallKit(), let callUUID = invitation.callkitUUID {
             requestTransaction(with: CXAnswerCallAction(call: callUUID))
         } else {
@@ -252,6 +265,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
     }
 
     func endCall(callUUID: UUID) {
+        logger.debug("CallController: ending the call")
         if CallController.useCallKit() {
             requestTransaction(with: CXEndCallAction(call: callUUID))
         } else {
@@ -266,6 +280,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
     }
 
     func endCall(invitation: RcvCallInvitation) {
+        logger.debug("CallController: ending the call")
         callManager.endCall(invitation: invitation) {
             if invitation.contact.id == self.activeCallInvitation?.contact.id {
                 DispatchQueue.main.async {
@@ -276,6 +291,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
     }
 
     func endCall(call: Call, completed: @escaping () -> Void) {
+        logger.debug("CallController: ending the call")
         callManager.endCall(call: call, completed: completed)
     }
 
@@ -292,10 +308,14 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
         provider.configuration = conf
     }
 
+    func hasActiveCalls() -> Bool {
+        controller.callObserver.calls.count > 0
+    }
+
     private func requestTransaction(with action: CXAction, onSuccess: @escaping () -> Void = {}) {
         controller.request(CXTransaction(action: action)) { error in
             if let error = error {
-                logger.error("CallController.requestTransaction error requesting transaction: \(error.localizedDescription)")
+                logger.error("CallController.requestTransaction error requesting transaction: \(error.localizedDescription, privacy: .public)")
             } else {
                 logger.debug("CallController.requestTransaction requested transaction successfully")
                 onSuccess()

apps/ios/Shared/Views/Call/WebRTCClient.swift

@@ -49,6 +49,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
         self.localRendererAspectRatio = localRendererAspectRatio
         rtcAudioSession.useManualAudio = CallController.useCallKit()
         rtcAudioSession.isAudioEnabled = !CallController.useCallKit()
+        logger.debug("WebRTCClient: rtcAudioSession has manual audio \(self.rtcAudioSession.useManualAudio) and audio enabled \(self.rtcAudioSession.isAudioEnabled)}")
         super.init()
     }
 
@@ -241,6 +242,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
     }
 
     func enableMedia(_ media: CallMediaType, _ enable: Bool) {
+        logger.debug("WebRTCClient: enabling media \(media.rawValue) \(enable)")
         media == .video ? setVideoEnabled(enable) : setAudioEnabled(enable)
     }
 
@@ -363,6 +365,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
 
     func endCall() {
         guard let call = activeCall.wrappedValue else { return }
+        logger.debug("WebRTCClient: ending the call")
         activeCall.wrappedValue = nil
         call.connection.close()
         call.connection.delegate = nil
@@ -534,6 +537,7 @@ extension WebRTCClient {
     }
 
     func setSpeakerEnabledAndConfigureSession( _ enabled: Bool) {
+        logger.debug("WebRTCClient: configuring session with speaker enabled \(enabled)")
         audioQueue.async { [weak self] in
             guard let self = self else { return }
             self.rtcAudioSession.lockForConfiguration()
@@ -545,6 +549,7 @@ extension WebRTCClient {
                 try self.rtcAudioSession.setMode(AVAudioSession.Mode.voiceChat.rawValue)
                 try self.rtcAudioSession.overrideOutputAudioPort(enabled ? .speaker : .none)
                 try self.rtcAudioSession.setActive(true)
+                logger.debug("WebRTCClient: configuring session with speaker enabled \(enabled) success")
             } catch let error {
                 logger.debug("Error configuring AVAudioSession: \(error)")
             }
@@ -552,6 +557,7 @@ extension WebRTCClient {
     }
 
     func audioSessionToDefaults() {
+        logger.debug("WebRTCClient: audioSession to defaults")
         audioQueue.async { [weak self] in
             guard let self = self else { return }
             self.rtcAudioSession.lockForConfiguration()
@@ -563,6 +569,7 @@ extension WebRTCClient {
                 try self.rtcAudioSession.setMode(AVAudioSession.Mode.default.rawValue)
                 try self.rtcAudioSession.overrideOutputAudioPort(.none)
                 try self.rtcAudioSession.setActive(false)
+                logger.debug("WebRTCClient: audioSession to defaults success")
             } catch let error {
                 logger.debug("Error configuring AVAudioSession with defaults: \(error)")
             }

apps/ios/Shared/Views/UserSettings/CallSettings.swift

@@ -22,16 +22,16 @@ struct CallSettings: View {
                 Section {
                     Toggle("Connect via relay", isOn: $webrtcPolicyRelay)
 
-                    if !CallController.isInChina && developerTools {
+                    if !CallController.isInChina {
                         Toggle("Use CallKit", isOn: $callKitEnabled)
 
-                        if allowChangingCallsHistory {
-                            Toggle("Show calls in phone history", isOn: $callKitCallsInRecents)
-                            .disabled(!callKitEnabled)
-                            .onChange(of: callKitCallsInRecents) { value in
-                                CallController.shared.showInRecents(value)
-                            }
-                        }
+//                        if allowChangingCallsHistory {
+//                            Toggle("Show calls in phone history", isOn: $callKitCallsInRecents)
+//                            .disabled(!callKitEnabled)
+//                            .onChange(of: callKitCallsInRecents) { value in
+//                                CallController.shared.showInRecents(value)
+//                            }
+//                        }
                     }
 
                     NavigationLink {