From 193e17f7afd7540e0d0b3cbaa1f9b9ffc2b7f86f Mon Sep 17 00:00:00 2001 From: Stanislav Dmitrenko <7953703+avently@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:08:22 +0700 Subject: [PATCH 1/4] android, desktop: thread-safe terminal items and floating terminal improvements (#4992) * android, desktop: thread-safe terminal items and floating terminal improvements * optimization --- .../chat/simplex/common/model/ChatModel.kt | 13 +++++ .../chat/simplex/common/views/TerminalView.kt | 54 ++++++++++++++++--- .../simplex/common/views/helpers/ModalView.kt | 5 +- .../views/usersettings/DeveloperView.kt | 2 +- .../kotlin/chat/simplex/common/DesktopApp.kt | 8 ++- 5 files changed, 71 insertions(+), 11 deletions(-) diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/ChatModel.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/ChatModel.kt index 7234209577..682a472060 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/ChatModel.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/ChatModel.kt @@ -71,6 +71,9 @@ object ChatModel { val groupMembers = mutableStateListOf() val groupMembersIndexes = mutableStateMapOf() + // false: default placement, true: floating window. + // Used for deciding to add terminal items on main thread or not. Floating means appPrefs.terminalAlwaysVisible + var terminalsVisible = setOf() val terminalItems = mutableStateOf>(listOf()) val userAddress = mutableStateOf(null) val chatItemTTL = mutableStateOf(ChatItemTTL.None) @@ -772,6 +775,16 @@ object ChatModel { fun addTerminalItem(item: TerminalItem) { val maxItems = if (appPreferences.developerTools.get()) 500 else 200 + if (terminalsVisible.isNotEmpty()) { + withApi { + addTerminalItem(item, maxItems) + } + } else { + addTerminalItem(item, maxItems) + } + } + + private fun addTerminalItem(item: TerminalItem, maxItems: Int) { if (terminalItems.value.size >= maxItems) { terminalItems.value = terminalItems.value.subList(1, terminalItems.value.size) } diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/TerminalView.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/TerminalView.kt index e44a174b53..d89803f8e4 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/TerminalView.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/TerminalView.kt @@ -20,11 +20,12 @@ import chat.simplex.common.views.chat.* import chat.simplex.common.views.helpers.* import chat.simplex.common.model.ChatModel import chat.simplex.common.platform.* -import chat.simplex.res.MR -import dev.icerock.moko.resources.compose.stringResource +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.launch @Composable -fun TerminalView(chatModel: ChatModel, close: () -> Unit) { +fun TerminalView(floating: Boolean = false, close: () -> Unit) { val composeState = remember { mutableStateOf(ComposeState(useLinkPreviews = false)) } val close = { close() @@ -37,6 +38,7 @@ fun TerminalView(chatModel: ChatModel, close: () -> Unit) { }) TerminalLayout( composeState, + floating, sendCommand = { sendCommand(chatModel, composeState) }, close ) @@ -65,6 +67,7 @@ private fun sendCommand(chatModel: ChatModel, composeState: MutableState, + floating: Boolean, sendCommand: () -> Unit, close: () -> Unit ) { @@ -118,19 +121,40 @@ fun TerminalLayout( color = MaterialTheme.colors.background, contentColor = LocalContentColor.current ) { - TerminalLog() + TerminalLog(floating) } } } } @Composable -fun TerminalLog() { +fun TerminalLog(floating: Boolean) { val reversedTerminalItems by remember { derivedStateOf { chatModel.terminalItems.value.asReversed() } } val clipboard = LocalClipboardManager.current - LazyColumnWithScrollBar(reverseLayout = true) { + val listState = LocalAppBarHandler.current?.listState ?: rememberLazyListState() + LaunchedEffect(Unit) { + var autoScrollToBottom = true + launch { + snapshotFlow { listState.layoutInfo.totalItemsCount } + .filter { autoScrollToBottom } + .collect { + try { + listState.scrollToItem(0) + } catch (e: Exception) { + Log.e(TAG, e.stackTraceToString()) + } + } + } + launch { + snapshotFlow { listState.firstVisibleItemIndex } + .collect { + autoScrollToBottom = listState.firstVisibleItemIndex == 0 + } + } + } + LazyColumnWithScrollBar(reverseLayout = true, state = listState) { items(reversedTerminalItems, key = { item -> item.id to item.createdAtNanos }) { item -> val rhId = item.remoteHostId val rhIdStr = if (rhId == null) "" else "$rhId " @@ -142,7 +166,12 @@ fun TerminalLog() { modifier = Modifier .fillMaxWidth() .clickable { - ModalManager.start.showModal(endButtons = { ShareButton { clipboard.shareText(item.details) } }) { + val modalPlace = if (floating) { + ModalManager.floatingTerminal + } else { + ModalManager.start + } + modalPlace.showModal(endButtons = { ShareButton { clipboard.shareText(item.details) } }) { SelectionContainer(modifier = Modifier.verticalScroll(rememberScrollState())) { val details = item.details .let { @@ -156,6 +185,16 @@ fun TerminalLog() { ) } } + DisposableEffect(Unit) { + val terminals = chatModel.terminalsVisible.toMutableSet() + terminals += floating + chatModel.terminalsVisible = terminals + onDispose { + val terminals = chatModel.terminalsVisible.toMutableSet() + terminals -= floating + chatModel.terminalsVisible = terminals + } + } } @Preview/*( @@ -169,6 +208,7 @@ fun PreviewTerminalLayout() { TerminalLayout( composeState = remember { mutableStateOf(ComposeState(useLinkPreviews = false)) }, sendCommand = {}, + floating = false, close = {} ) } diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/helpers/ModalView.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/helpers/ModalView.kt index 8da73ab3ca..4c35e72701 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/helpers/ModalView.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/helpers/ModalView.kt @@ -2,10 +2,8 @@ package chat.simplex.common.views.helpers import androidx.compose.animation.* import androidx.compose.animation.core.* -import androidx.compose.foundation.ScrollState import androidx.compose.foundation.background import androidx.compose.foundation.layout.* -import androidx.compose.foundation.lazy.LazyListState import androidx.compose.material.* import androidx.compose.runtime.* import androidx.compose.ui.Modifier @@ -209,11 +207,14 @@ class ModalManager(private val placement: ModalPlacement? = null) { val end = if (appPlatform.isAndroid) shared else ModalManager(ModalPlacement.END) val fullscreen = if (appPlatform.isAndroid) shared else ModalManager(ModalPlacement.FULLSCREEN) + val floatingTerminal = if (appPlatform.isAndroid) shared else ModalManager(ModalPlacement.START) + fun closeAllModalsEverywhere() { start.closeModals() center.closeModals() end.closeModals() fullscreen.closeModals() + floatingTerminal.closeModals() } @OptIn(ExperimentalAnimationApi::class) diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/usersettings/DeveloperView.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/usersettings/DeveloperView.kt index 9dfdf23085..2123d98f41 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/usersettings/DeveloperView.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/usersettings/DeveloperView.kt @@ -35,7 +35,7 @@ fun DeveloperView( val unchangedHints = mutableStateOf(unchangedHintPreferences()) SectionView { InstallTerminalAppItem(uriHandler) - ChatConsoleItem { withAuth(generalGetString(MR.strings.auth_open_chat_console), generalGetString(MR.strings.auth_log_in_using_credential), showCustomModal { it, close -> TerminalView(it, close) }) } + ChatConsoleItem { withAuth(generalGetString(MR.strings.auth_open_chat_console), generalGetString(MR.strings.auth_log_in_using_credential), showCustomModal { it, close -> TerminalView(false, close) }) } ResetHintsItem(unchangedHints) SettingsPreferenceItem(painterResource(MR.images.ic_code), stringResource(MR.strings.show_developer_options), developerTools) SectionTextFooter( diff --git a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/DesktopApp.kt b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/DesktopApp.kt index fc0e97b417..1fb739946c 100644 --- a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/DesktopApp.kt +++ b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/DesktopApp.kt @@ -199,7 +199,13 @@ private fun ApplicationScope.AppWindow(closedByError: MutableState) { 768.dp) Window(state = cWindowState, onCloseRequest = { hiddenUntilRestart = true }, title = stringResource(MR.strings.chat_console)) { SimpleXTheme { - TerminalView(ChatModel) { hiddenUntilRestart = true } + TerminalView(true) { hiddenUntilRestart = true } + ModalManager.floatingTerminal.showInView() + DisposableEffect(Unit) { + onDispose { + ModalManager.floatingTerminal.closeModals() + } + } } } } From c5261a416f64c1e936564c6f03d0866a2e28f0c1 Mon Sep 17 00:00:00 2001 From: Stanislav Dmitrenko <7953703+avently@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:49:13 +0700 Subject: [PATCH 2/4] ios: calls switching from audio to video and back (#4964) * ios: switch calls * working audio/video calls without screen recording * ui * follow up * audio devices & permissions * padding * backward compatibility * cleanup & fix * buttons foreground color and converting call to video call from CallKit --------- Co-authored-by: Evgeny Poberezkin --- apps/ios/Shared/ContentView.swift | 15 +- apps/ios/Shared/Model/SimpleXAPI.swift | 1 - .../Shared/Views/Call/ActiveCallView.swift | 270 +++++---- .../Shared/Views/Call/CallController.swift | 8 +- apps/ios/Shared/Views/Call/CallManager.swift | 10 +- .../Shared/Views/Call/CallViewRenderers.swift | 210 +++++-- apps/ios/Shared/Views/Call/WebRTC.swift | 56 +- apps/ios/Shared/Views/Call/WebRTCClient.swift | 518 ++++++++++++++---- apps/ios/SimpleXChat/CallTypes.swift | 8 + 9 files changed, 806 insertions(+), 290 deletions(-) diff --git a/apps/ios/Shared/ContentView.swift b/apps/ios/Shared/ContentView.swift index 58a552c1a0..6d600f33ff 100644 --- a/apps/ios/Shared/ContentView.swift +++ b/apps/ios/Shared/ContentView.swift @@ -297,9 +297,18 @@ struct ContentView: View { if let contactId = contacts?.first?.personHandle?.value, let chat = chatModel.getChat(contactId), case let .direct(contact) = chat.chatInfo { - logger.debug("callToRecentContact: schedule call") - DispatchQueue.main.asyncAfter(deadline: .now() + 1) { - CallController.shared.startCall(contact, mediaType) + let activeCall = chatModel.activeCall + // This line works when a user clicks on a video button in CallKit UI while in call. + // The app tries to make another call to the same contact and overwite activeCall instance making its state broken + if let activeCall, contactId == activeCall.contact.id, mediaType == .video, !activeCall.hasVideo { + Task { + await chatModel.callCommand.processCommand(.media(source: .camera, enable: true)) + } + } else if activeCall == nil { + logger.debug("callToRecentContact: schedule call") + DispatchQueue.main.asyncAfter(deadline: .now() + 1) { + CallController.shared.startCall(contact, mediaType) + } } } } diff --git a/apps/ios/Shared/Model/SimpleXAPI.swift b/apps/ios/Shared/Model/SimpleXAPI.swift index ea48d5d7b3..74cee396c7 100644 --- a/apps/ios/Shared/Model/SimpleXAPI.swift +++ b/apps/ios/Shared/Model/SimpleXAPI.swift @@ -2084,7 +2084,6 @@ func processReceivedMsg(_ res: ChatResponse) async { await withCall(contact) { call in await MainActor.run { call.callState = .offerReceived - call.peerMedia = callType.media call.sharedKey = sharedKey } let useRelay = UserDefaults.standard.bool(forKey: DEFAULT_WEBRTC_POLICY_RELAY) diff --git a/apps/ios/Shared/Views/Call/ActiveCallView.swift b/apps/ios/Shared/Views/Call/ActiveCallView.swift index 049ee95a09..ec0873a634 100644 --- a/apps/ios/Shared/Views/Call/ActiveCallView.swift +++ b/apps/ios/Shared/Views/Call/ActiveCallView.swift @@ -17,8 +17,8 @@ struct ActiveCallView: View { @ObservedObject var call: Call @Environment(\.scenePhase) var scenePhase @State private var client: WebRTCClient? = nil - @State private var activeCall: WebRTCClient.Call? = nil @State private var localRendererAspectRatio: CGFloat? = nil + @State var remoteContentMode: UIView.ContentMode = .scaleAspectFill @Binding var canConnectCall: Bool @State var prevColorScheme: ColorScheme = .dark @State var pipShown = false @@ -27,24 +27,39 @@ struct ActiveCallView: View { var body: some View { ZStack(alignment: .topLeading) { ZStack(alignment: .bottom) { - if let client = client, [call.peerMedia, call.localMedia].contains(.video), activeCall != nil { + if let client = client, call.hasVideo { GeometryReader { g in let width = g.size.width * 0.3 ZStack(alignment: .topTrailing) { - CallViewRemote(client: client, activeCall: $activeCall, activeCallViewIsCollapsed: $m.activeCallViewIsCollapsed, pipShown: $pipShown) - CallViewLocal(client: client, activeCall: $activeCall, localRendererAspectRatio: $localRendererAspectRatio, pipShown: $pipShown) - .cornerRadius(10) - .frame(width: width, height: width / (localRendererAspectRatio ?? 1)) - .padding([.top, .trailing], 17) ZStack(alignment: .center) { // For some reason, when the view in GeometryReader and ZStack is visible, it steals clicks on a back button, so showing something on top like this with background color helps (.clear color doesn't work) } .frame(maxWidth: .infinity, maxHeight: .infinity) .background(Color.primary.opacity(0.000001)) + + CallViewRemote(client: client, call: call, activeCallViewIsCollapsed: $m.activeCallViewIsCollapsed, contentMode: $remoteContentMode, pipShown: $pipShown) + .onTapGesture { + remoteContentMode = remoteContentMode == .scaleAspectFill ? .scaleAspectFit : .scaleAspectFill + } + + Group { + let localVideoTrack = client.activeCall?.localVideoTrack ?? client.notConnectedCall?.localCameraAndTrack?.1 + if localVideoTrack != nil { + CallViewLocal(client: client, localRendererAspectRatio: $localRendererAspectRatio, pipShown: $pipShown) + .onDisappear { + localRendererAspectRatio = nil + } + } else { + Rectangle().fill(.black) + } + } + .cornerRadius(10) + .frame(width: width, height: localRendererAspectRatio == nil ? (g.size.width < g.size.height ? width * 1.33 : width / 1.33) : width / (localRendererAspectRatio ?? 1)) + .padding([.top, .trailing], 17) } } } - if let call = m.activeCall, let client = client, (!pipShown || !call.supportsVideo) { + if let call = m.activeCall, let client = client, (!pipShown || !call.hasVideo) { ActiveCallOverlay(call: call, client: client) } } @@ -54,6 +69,9 @@ struct ActiveCallView: View { .onAppear { logger.debug("ActiveCallView: appear client is nil \(client == nil), scenePhase \(String(describing: scenePhase)), canConnectCall \(canConnectCall)") AppDelegate.keepScreenOn(true) + Task { + await askRequiredPermissions() + } createWebRTCClient() dismissAllSheets() hideKeyboard() @@ -84,7 +102,7 @@ struct ActiveCallView: View { private func createWebRTCClient() { if client == nil && canConnectCall { - client = WebRTCClient($activeCall, { msg in await MainActor.run { processRtcMessage(msg: msg) } }, $localRendererAspectRatio) + client = WebRTCClient({ msg in await MainActor.run { processRtcMessage(msg: msg) } }, $localRendererAspectRatio) Task { await m.callCommand.setClient(client) } @@ -99,7 +117,7 @@ struct ActiveCallView: View { logger.debug("ActiveCallView: response \(msg.resp.respType)") switch msg.resp { case let .capabilities(capabilities): - let callType = CallType(media: call.localMedia, capabilities: capabilities) + let callType = CallType(media: call.initialCallType, capabilities: capabilities) Task { do { try await apiSendCallInvitation(call.contact, callType) @@ -110,7 +128,7 @@ struct ActiveCallView: View { call.callState = .invitationSent call.localCapabilities = capabilities } - if call.supportsVideo && !AVAudioSession.sharedInstance().hasExternalAudioDevice() { + if call.hasVideo && !AVAudioSession.sharedInstance().hasExternalAudioDevice() { try? AVAudioSession.sharedInstance().setCategory(.playback, options: [.allowBluetooth, .allowAirPlay, .allowBluetoothA2DP]) } CallSoundsPlayer.shared.startConnectingCallSound() @@ -120,7 +138,7 @@ struct ActiveCallView: View { Task { do { try await apiSendCallOffer(call.contact, offer, iceCandidates, - media: call.localMedia, capabilities: capabilities) + media: call.initialCallType, capabilities: capabilities) } catch { logger.error("apiSendCallOffer \(responseError(error))") } @@ -164,6 +182,9 @@ struct ActiveCallView: View { } if state.connectionState == "closed" { closeCallView(client) + if let callUUID = m.activeCall?.callUUID { + CallController.shared.endCall(callUUID: callUUID) + } m.activeCall = nil m.activeCallViewIsCollapsed = false } @@ -182,6 +203,14 @@ struct ActiveCallView: View { CallSoundsPlayer.shared.vibrate(long: false) wasConnected = true } + case let .peerMedia(source, enabled): + switch source { + case .mic: call.peerMediaSources.mic = enabled + case .camera: call.peerMediaSources.camera = enabled + case .screenAudio: call.peerMediaSources.screenAudio = enabled + case .screenVideo: call.peerMediaSources.screenVideo = enabled + case .unknown: () + } case .ended: closeCallView(client) call.callState = .ended @@ -226,6 +255,26 @@ struct ActiveCallView: View { } } + private func askRequiredPermissions() async { + let mic = await WebRTCClient.isAuthorized(for: .audio) + await MainActor.run { + call.localMediaSources.mic = mic + } + let cameraAuthorized = AVCaptureDevice.authorizationStatus(for: .video) == .authorized + var camera = call.initialCallType == .audio || cameraAuthorized + if call.initialCallType == .video && !cameraAuthorized { + camera = await WebRTCClient.isAuthorized(for: .video) + await MainActor.run { + if camera, let client { + client.setCameraEnabled(true) + } + } + } + if !mic || !camera { + WebRTCClient.showUnauthorizedAlert(for: !mic ? .audio : .video) + } + } + private func closeCallView(_ client: WebRTCClient) { if m.activeCall != nil { m.showCallView = false @@ -241,44 +290,16 @@ struct ActiveCallOverlay: View { var body: some View { VStack { - switch call.localMedia { - case .video: + switch call.hasVideo { + case true: videoCallInfoView(call) .foregroundColor(.white) .opacity(0.8) - .padding() - - Spacer() - - HStack { - toggleAudioButton() - Spacer() - if deviceManager.availableInputs.allSatisfy({ $0.portType == .builtInMic }) { - toggleSpeakerButton() - .frame(width: 40, height: 40) - } else if call.hasMedia { - AudioDevicePicker() - .scaleEffect(2) - .frame(maxWidth: 40, maxHeight: 40) - } else { - Color.clear.frame(width: 40, height: 40) - } - Spacer() - endCallButton() - Spacer() - if call.videoEnabled { - flipCameraButton() - } else { - Color.clear.frame(width: 40, height: 40) - } - Spacer() - toggleVideoButton() - } - .padding(.horizontal, 20) - .padding(.bottom, 16) - .frame(maxWidth: .infinity, alignment: .center) - - case .audio: + .padding(.horizontal) + // Fixed vertical padding required for preserving position of buttons row when changing audio-to-video and back in landscape orientation. + // Otherwise, bigger padding is added by SwiftUI when switching call types + .padding(.vertical, 10) + case false: ZStack(alignment: .topLeading) { Button { chatModel.activeCallViewIsCollapsed = true @@ -293,35 +314,32 @@ struct ActiveCallOverlay: View { } .foregroundColor(.white) .opacity(0.8) - .padding() + .padding(.horizontal) + .padding(.vertical, 10) .frame(maxHeight: .infinity) } - - Spacer() - - ZStack(alignment: .bottom) { - toggleAudioButton() - .frame(maxWidth: .infinity, alignment: .leading) - endCallButton() - // Check if the only input is microphone. And in this case show toggle button, - // If there are more inputs, it probably means something like bluetooth headphones are available - // and in this case show iOS button for choosing different output. - // There is no way to get available outputs, only inputs - if deviceManager.availableInputs.allSatisfy({ $0.portType == .builtInMic }) { - toggleSpeakerButton() - .frame(maxWidth: .infinity, alignment: .trailing) - } else if call.hasMedia { - AudioDevicePicker() - .scaleEffect(2) - .frame(maxWidth: 50, maxHeight: 40) - .frame(maxWidth: .infinity, alignment: .trailing) - } else { - Color.clear.frame(width: 50, height: 40) - } - } - .padding(.bottom, 60) - .padding(.horizontal, 48) } + + Spacer() + + HStack { + toggleMicButton() + Spacer() + audioDeviceButton() + Spacer() + endCallButton() + Spacer() + if call.localMediaSources.camera { + flipCameraButton() + } else { + Color.clear.frame(width: 60, height: 60) + } + Spacer() + toggleCameraButton() + } + .padding(.horizontal, 20) + .padding(.bottom, 16) + .frame(maxWidth: 440, alignment: .center) } .frame(maxWidth: .infinity) .onAppear { @@ -383,36 +401,56 @@ struct ActiveCallOverlay: View { private func endCallButton() -> some View { let cc = CallController.shared - return callButton("phone.down.fill", width: 60, height: 60) { + return callButton("phone.down.fill", padding: 10) { if let uuid = call.callUUID { cc.endCall(callUUID: uuid) } else { cc.endCall(call: call) {} } } - .foregroundColor(.red) + .background(.red) + .clipShape(.circle) } - private func toggleAudioButton() -> some View { - controlButton(call, call.audioEnabled ? "mic.fill" : "mic.slash") { + private func toggleMicButton() -> some View { + controlButton(call, call.localMediaSources.mic ? "mic.fill" : "mic.slash", padding: 14) { Task { - client.setAudioEnabled(!call.audioEnabled) - DispatchQueue.main.async { - call.audioEnabled = !call.audioEnabled - } + if await WebRTCClient.isAuthorized(for: .audio) { + client.setAudioEnabled(!call.localMediaSources.mic) + } else { WebRTCClient.showUnauthorizedAlert(for: .audio) } + } + } + } + + func audioDeviceButton() -> some View { + // Check if the only input is microphone. And in this case show toggle button, + // If there are more inputs, it probably means something like bluetooth headphones are available + // and in this case show iOS button for choosing different output. + // There is no way to get available outputs, only inputs + Group { + if deviceManager.availableInputs.allSatisfy({ $0.portType == .builtInMic }) { + toggleSpeakerButton() + } else { + audioDevicePickerButton() + } + } + .onChange(of: call.hasVideo) { hasVideo in + let current = AVAudioSession.sharedInstance().currentRoute.outputs.first?.portType + let speakerEnabled = current == .builtInSpeaker + let receiverEnabled = current == .builtInReceiver + // react automatically only when speaker or receiver were selected, otherwise keep an external device selected + if hasVideo != speakerEnabled && (speakerEnabled || receiverEnabled) { + client.setSpeakerEnabledAndConfigureSession(!speakerEnabled, skipExternalDevice: true) + call.speakerEnabled = !speakerEnabled } } } private func toggleSpeakerButton() -> some View { - controlButton(call, call.speakerEnabled ? "speaker.wave.2.fill" : "speaker.wave.1.fill") { - Task { - let speakerEnabled = AVAudioSession.sharedInstance().currentRoute.outputs.first?.portType == .builtInSpeaker - client.setSpeakerEnabledAndConfigureSession(!speakerEnabled) - DispatchQueue.main.async { - call.speakerEnabled = !speakerEnabled - } - } + controlButton(call, !call.peerMediaSources.mic ? "speaker.slash" : call.speakerEnabled ? "speaker.wave.2.fill" : "speaker.wave.1.fill", padding: !call.peerMediaSources.mic ? 16 : call.speakerEnabled ? 15 : 17) { + let speakerEnabled = AVAudioSession.sharedInstance().currentRoute.outputs.first?.portType == .builtInSpeaker + client.setSpeakerEnabledAndConfigureSession(!speakerEnabled) + call.speakerEnabled = !speakerEnabled } .onAppear { deviceManager.call = call @@ -420,53 +458,67 @@ struct ActiveCallOverlay: View { } } - private func toggleVideoButton() -> some View { - controlButton(call, call.videoEnabled ? "video.fill" : "video.slash") { + private func toggleCameraButton() -> some View { + controlButton(call, call.localMediaSources.camera ? "video.fill" : "video.slash", padding: call.localMediaSources.camera ? 16 : 14) { Task { - client.setVideoEnabled(!call.videoEnabled) - DispatchQueue.main.async { - call.videoEnabled = !call.videoEnabled - } + if await WebRTCClient.isAuthorized(for: .video) { + client.setCameraEnabled(!call.localMediaSources.camera) + } else { WebRTCClient.showUnauthorizedAlert(for: .video) } } } + .disabled(call.initialCallType == .audio && client.activeCall?.peerHasOldVersion == true) } @ViewBuilder private func flipCameraButton() -> some View { - controlButton(call, "arrow.triangle.2.circlepath") { + controlButton(call, "arrow.triangle.2.circlepath", padding: 12) { Task { - client.flipCamera() + if await WebRTCClient.isAuthorized(for: .video) { + client.flipCamera() + } } } } - @ViewBuilder private func controlButton(_ call: Call, _ imageName: String, _ perform: @escaping () -> Void) -> some View { - if call.hasMedia { - callButton(imageName, width: 50, height: 38, perform) - .foregroundColor(.white) - .opacity(0.85) - } else { - Color.clear.frame(width: 50, height: 38) - } + @ViewBuilder private func controlButton(_ call: Call, _ imageName: String, padding: CGFloat, _ perform: @escaping () -> Void) -> some View { + callButton(imageName, padding: padding, perform) + .background(call.peerMediaSources.hasVideo ? Color.black.opacity(0.2) : Color.white.opacity(0.2)) + .clipShape(.circle) } - private func callButton(_ imageName: String, width: CGFloat, height: CGFloat, _ perform: @escaping () -> Void) -> some View { + @ViewBuilder private func audioDevicePickerButton() -> some View { + AudioDevicePicker() + .opacity(0.8) + .scaleEffect(2) + .padding(10) + .frame(width: 60, height: 60) + .background(call.peerMediaSources.hasVideo ? Color.black.opacity(0.2) : Color.white.opacity(0.2)) + .clipShape(.circle) + } + + private func callButton(_ imageName: String, padding: CGFloat, _ perform: @escaping () -> Void) -> some View { Button { perform() } label: { Image(systemName: imageName) .resizable() .scaledToFit() - .frame(maxWidth: width, maxHeight: height) + .padding(padding) + .frame(width: 60, height: 60) } + .foregroundColor(whiteColorWithAlpha) + } + + private var whiteColorWithAlpha: Color { + get { Color(red: 204 / 255, green: 204 / 255, blue: 204 / 255) } } } struct ActiveCallOverlay_Previews: PreviewProvider { static var previews: some View { Group{ - ActiveCallOverlay(call: Call(direction: .incoming, contact: Contact.sampleData, callUUID: UUID().uuidString.lowercased(), callState: .offerSent, localMedia: .video), client: WebRTCClient(Binding.constant(nil), { _ in }, Binding.constant(nil))) + ActiveCallOverlay(call: Call(direction: .incoming, contact: Contact.sampleData, callUUID: UUID().uuidString.lowercased(), callState: .offerSent, initialCallType: .video), client: WebRTCClient({ _ in }, Binding.constant(nil))) .background(.black) - ActiveCallOverlay(call: Call(direction: .incoming, contact: Contact.sampleData, callUUID: UUID().uuidString.lowercased(), callState: .offerSent, localMedia: .audio), client: WebRTCClient(Binding.constant(nil), { _ in }, Binding.constant(nil))) + ActiveCallOverlay(call: Call(direction: .incoming, contact: Contact.sampleData, callUUID: UUID().uuidString.lowercased(), callState: .offerSent, initialCallType: .audio), client: WebRTCClient({ _ in }, Binding.constant(nil))) .background(.black) } } diff --git a/apps/ios/Shared/Views/Call/CallController.swift b/apps/ios/Shared/Views/Call/CallController.swift index 36887d6184..bf0f1045a4 100644 --- a/apps/ios/Shared/Views/Call/CallController.swift +++ b/apps/ios/Shared/Views/Call/CallController.swift @@ -104,7 +104,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse } func provider(_ provider: CXProvider, perform action: CXSetMutedCallAction) { - if callManager.enableMedia(media: .audio, enable: !action.isMuted, callUUID: action.callUUID.uuidString.lowercased()) { + if callManager.enableMedia(source: .mic, enable: !action.isMuted, callUUID: action.callUUID.uuidString.lowercased()) { action.fulfill() } else { action.fail() @@ -121,8 +121,8 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse RTCAudioSession.sharedInstance().audioSessionDidActivate(audioSession) RTCAudioSession.sharedInstance().isAudioEnabled = true do { - let supportsVideo = ChatModel.shared.activeCall?.supportsVideo == true - if supportsVideo { + let hasVideo = ChatModel.shared.activeCall?.hasVideo == true + if hasVideo { try audioSession.setCategory(.playAndRecord, mode: .videoChat, options: [.defaultToSpeaker, .mixWithOthers, .allowBluetooth, .allowAirPlay, .allowBluetoothA2DP]) } else { try audioSession.setCategory(.playAndRecord, mode: .voiceChat, options: [.mixWithOthers, .allowBluetooth, .allowAirPlay, .allowBluetoothA2DP]) @@ -133,7 +133,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse try? await Task.sleep(nanoseconds: UInt64(i) * 300_000000) if let preferred = audioSession.preferredInputDevice() { await MainActor.run { try? audioSession.setPreferredInput(preferred) } - } else if supportsVideo { + } else if hasVideo { await MainActor.run { try? audioSession.overrideOutputAudioPort(.speaker) } } } diff --git a/apps/ios/Shared/Views/Call/CallManager.swift b/apps/ios/Shared/Views/Call/CallManager.swift index f3021815af..a3e1df2301 100644 --- a/apps/ios/Shared/Views/Call/CallManager.swift +++ b/apps/ios/Shared/Views/Call/CallManager.swift @@ -12,7 +12,7 @@ import SimpleXChat class CallManager { func newOutgoingCall(_ contact: Contact, _ media: CallMediaType) -> String { let uuid = UUID().uuidString.lowercased() - let call = Call(direction: .outgoing, contact: contact, callUUID: uuid, callState: .waitCapabilities, localMedia: media) + let call = Call(direction: .outgoing, contact: contact, callUUID: uuid, callState: .waitCapabilities, initialCallType: media) call.speakerEnabled = media == .video ChatModel.shared.activeCall = call return uuid @@ -22,7 +22,7 @@ class CallManager { let m = ChatModel.shared if let call = m.activeCall, call.callUUID == callUUID { m.showCallView = true - Task { await m.callCommand.processCommand(.capabilities(media: call.localMedia)) } + Task { await m.callCommand.processCommand(.capabilities(media: call.initialCallType)) } return true } return false @@ -44,7 +44,7 @@ class CallManager { contact: invitation.contact, callUUID: invitation.callUUID, callState: .invitationAccepted, - localMedia: invitation.callType.media, + initialCallType: invitation.callType.media, sharedKey: invitation.sharedKey ) call.speakerEnabled = invitation.callType.media == .video @@ -68,10 +68,10 @@ class CallManager { } } - func enableMedia(media: CallMediaType, enable: Bool, callUUID: String) -> Bool { + func enableMedia(source: CallMediaSource, enable: Bool, callUUID: String) -> Bool { if let call = ChatModel.shared.activeCall, call.callUUID == callUUID { let m = ChatModel.shared - Task { await m.callCommand.processCommand(.media(media: media, enable: enable)) } + Task { await m.callCommand.processCommand(.media(source: source, enable: enable)) } return true } return false diff --git a/apps/ios/Shared/Views/Call/CallViewRenderers.swift b/apps/ios/Shared/Views/Call/CallViewRenderers.swift index a3201d9351..fbfeb99bf5 100644 --- a/apps/ios/Shared/Views/Call/CallViewRenderers.swift +++ b/apps/ios/Shared/Views/Call/CallViewRenderers.swift @@ -10,40 +10,43 @@ import AVKit struct CallViewRemote: UIViewRepresentable { var client: WebRTCClient - var activeCall: Binding + @ObservedObject var call: Call @State var enablePip: (Bool) -> Void = {_ in } @Binding var activeCallViewIsCollapsed: Bool + @Binding var contentMode: UIView.ContentMode @Binding var pipShown: Bool - init(client: WebRTCClient, activeCall: Binding, activeCallViewIsCollapsed: Binding, pipShown: Binding) { - self.client = client - self.activeCall = activeCall - self._activeCallViewIsCollapsed = activeCallViewIsCollapsed - self._pipShown = pipShown - } - func makeUIView(context: Context) -> UIView { let view = UIView() - if let call = activeCall.wrappedValue { - let remoteRenderer = RTCMTLVideoView(frame: view.frame) - remoteRenderer.videoContentMode = .scaleAspectFill - client.addRemoteRenderer(call, remoteRenderer) - addSubviewAndResize(remoteRenderer, into: view) + let remoteCameraRenderer = RTCMTLVideoView(frame: view.frame) + remoteCameraRenderer.videoContentMode = contentMode + remoteCameraRenderer.tag = 0 + let remoteScreenRenderer = RTCMTLVideoView(frame: view.frame) + remoteScreenRenderer.videoContentMode = contentMode + remoteScreenRenderer.tag = 1 + remoteScreenRenderer.alpha = call.peerMediaSources.screenVideo ? 1 : 0 - if AVPictureInPictureController.isPictureInPictureSupported() { - makeViewWithRTCRenderer(call, remoteRenderer, view, context) - } + context.coordinator.cameraRenderer = remoteCameraRenderer + context.coordinator.screenRenderer = remoteScreenRenderer + client.addRemoteCameraRenderer(remoteCameraRenderer) + client.addRemoteScreenRenderer(remoteScreenRenderer) + addSubviewAndResize(remoteCameraRenderer, remoteScreenRenderer, into: view) + + if AVPictureInPictureController.isPictureInPictureSupported() { + makeViewWithRTCRenderer(remoteCameraRenderer, remoteScreenRenderer, view, context) } return view } - func makeViewWithRTCRenderer(_ call: WebRTCClient.Call, _ remoteRenderer: RTCMTLVideoView, _ view: UIView, _ context: Context) { - let pipRemoteRenderer = RTCMTLVideoView(frame: view.frame) - pipRemoteRenderer.videoContentMode = .scaleAspectFill - + func makeViewWithRTCRenderer(_ remoteCameraRenderer: RTCMTLVideoView, _ remoteScreenRenderer: RTCMTLVideoView, _ view: UIView, _ context: Context) { + let pipRemoteCameraRenderer = RTCMTLVideoView(frame: view.frame) + pipRemoteCameraRenderer.videoContentMode = .scaleAspectFill + + let pipRemoteScreenRenderer = RTCMTLVideoView(frame: view.frame) + pipRemoteScreenRenderer.videoContentMode = .scaleAspectFill + let pipVideoCallViewController = AVPictureInPictureVideoCallViewController() pipVideoCallViewController.preferredContentSize = CGSize(width: 1080, height: 1920) - addSubviewAndResize(pipRemoteRenderer, into: pipVideoCallViewController.view) let pipContentSource = AVPictureInPictureController.ContentSource( activeVideoCallSourceView: view, contentViewController: pipVideoCallViewController @@ -55,7 +58,9 @@ struct CallViewRemote: UIViewRepresentable { context.coordinator.pipController = pipController context.coordinator.willShowHide = { show in if show { - client.addRemoteRenderer(call, pipRemoteRenderer) + client.addRemoteCameraRenderer(pipRemoteCameraRenderer) + client.addRemoteScreenRenderer(pipRemoteScreenRenderer) + context.coordinator.relayout() DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) { activeCallViewIsCollapsed = true } @@ -67,13 +72,29 @@ struct CallViewRemote: UIViewRepresentable { } context.coordinator.didShowHide = { show in if show { - remoteRenderer.isHidden = true + remoteCameraRenderer.isHidden = true + remoteScreenRenderer.isHidden = true } else { - client.removeRemoteRenderer(call, pipRemoteRenderer) - remoteRenderer.isHidden = false + client.removeRemoteCameraRenderer(pipRemoteCameraRenderer) + client.removeRemoteScreenRenderer(pipRemoteScreenRenderer) + remoteCameraRenderer.isHidden = false + remoteScreenRenderer.isHidden = false } pipShown = show } + context.coordinator.relayout = { + let camera = call.peerMediaSources.camera + let screenVideo = call.peerMediaSources.screenVideo + pipRemoteCameraRenderer.alpha = camera ? 1 : 0 + pipRemoteScreenRenderer.alpha = screenVideo ? 1 : 0 + if screenVideo { + addSubviewAndResize(pipRemoteScreenRenderer, pipRemoteCameraRenderer, pip: true, into: pipVideoCallViewController.view) + } else { + addSubviewAndResize(pipRemoteCameraRenderer, pipRemoteScreenRenderer, pip: true, into: pipVideoCallViewController.view) + } + (pipVideoCallViewController.view.subviews[0] as! RTCMTLVideoView).videoContentMode = contentMode + (pipVideoCallViewController.view.subviews[1] as! RTCMTLVideoView).videoContentMode = .scaleAspectFill + } DispatchQueue.main.async { enablePip = { enable in if enable != pipShown /* pipController.isPictureInPictureActive */ { @@ -88,24 +109,50 @@ struct CallViewRemote: UIViewRepresentable { } func makeCoordinator() -> Coordinator { - Coordinator() + Coordinator(client) } func updateUIView(_ view: UIView, context: Context) { logger.debug("CallView.updateUIView remote") + let camera = view.subviews.first(where: { $0.tag == 0 })! + let screen = view.subviews.first(where: { $0.tag == 1 })! + let screenVideo = call.peerMediaSources.screenVideo + if screenVideo && screen.alpha == 0 { + screen.alpha = 1 + addSubviewAndResize(screen, camera, into: view) + } else if !screenVideo && screen.alpha == 1 { + screen.alpha = 0 + addSubviewAndResize(camera, screen, into: view) + } + (view.subviews[0] as! RTCMTLVideoView).videoContentMode = contentMode + (view.subviews[1] as! RTCMTLVideoView).videoContentMode = .scaleAspectFill + + camera.alpha = call.peerMediaSources.camera ? 1 : 0 + screen.alpha = call.peerMediaSources.screenVideo ? 1 : 0 + DispatchQueue.main.async { if activeCallViewIsCollapsed != pipShown { enablePip(activeCallViewIsCollapsed) + } else if pipShown { + context.coordinator.relayout() } } } // MARK: - Coordinator class Coordinator: NSObject, AVPictureInPictureControllerDelegate { + var cameraRenderer: RTCMTLVideoView? + var screenRenderer: RTCMTLVideoView? + var client: WebRTCClient var pipController: AVPictureInPictureController? = nil var willShowHide: (Bool) -> Void = { _ in } var didShowHide: (Bool) -> Void = { _ in } - + var relayout: () -> Void = {} + + required init(_ client: WebRTCClient) { + self.client = client + } + func pictureInPictureControllerWillStartPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) { willShowHide(true) } @@ -127,11 +174,20 @@ struct CallViewRemote: UIViewRepresentable { } deinit { + // TODO: deinit is not called when changing call type from audio to video and back, + // which causes many renderers can be created and added to stream (if enabling/disabling + // video while not yet connected in outgoing call) pipController?.stopPictureInPicture() pipController?.canStartPictureInPictureAutomaticallyFromInline = false pipController?.contentSource = nil pipController?.delegate = nil pipController = nil + if let cameraRenderer { + client.removeRemoteCameraRenderer(cameraRenderer) + } + if let screenRenderer { + client.removeRemoteScreenRenderer(screenRenderer) + } } } @@ -148,51 +204,109 @@ struct CallViewRemote: UIViewRepresentable { struct CallViewLocal: UIViewRepresentable { var client: WebRTCClient - var activeCall: Binding var localRendererAspectRatio: Binding @State var pipStateChanged: (Bool) -> Void = {_ in } @Binding var pipShown: Bool - init(client: WebRTCClient, activeCall: Binding, localRendererAspectRatio: Binding, pipShown: Binding) { + init(client: WebRTCClient, localRendererAspectRatio: Binding, pipShown: Binding) { self.client = client - self.activeCall = activeCall self.localRendererAspectRatio = localRendererAspectRatio self._pipShown = pipShown } func makeUIView(context: Context) -> UIView { let view = UIView() - if let call = activeCall.wrappedValue { - let localRenderer = RTCEAGLVideoView(frame: .zero) - client.addLocalRenderer(call, localRenderer) - client.startCaptureLocalVideo(call) - addSubviewAndResize(localRenderer, into: view) - DispatchQueue.main.async { - pipStateChanged = { shown in - localRenderer.isHidden = shown - } + let localRenderer = RTCEAGLVideoView(frame: .zero) + context.coordinator.renderer = localRenderer + client.addLocalRenderer(localRenderer) + addSubviewAndResize(localRenderer, nil, into: view) + DispatchQueue.main.async { + pipStateChanged = { shown in + localRenderer.isHidden = shown } } return view } + func makeCoordinator() -> Coordinator { + Coordinator(client) + } + func updateUIView(_ view: UIView, context: Context) { logger.debug("CallView.updateUIView local") pipStateChanged(pipShown) } + + // MARK: - Coordinator + class Coordinator: NSObject, AVPictureInPictureControllerDelegate { + var renderer: RTCEAGLVideoView? + var client: WebRTCClient + + required init(_ client: WebRTCClient) { + self.client = client + } + + deinit { + if let renderer { + client.removeLocalRenderer(renderer) + } + } + } } -private func addSubviewAndResize(_ view: UIView, into containerView: UIView) { - containerView.addSubview(view) - view.translatesAutoresizingMaskIntoConstraints = false - containerView.addConstraints(NSLayoutConstraint.constraints(withVisualFormat: "H:|[view]|", - options: [], - metrics: nil, - views: ["view": view])) +private func addSubviewAndResize(_ fullscreen: UIView, _ end: UIView?, pip: Bool = false, into containerView: UIView) { + if containerView.subviews.firstIndex(of: fullscreen) == 0 && ((end == nil && containerView.subviews.count == 1) || (end != nil && containerView.subviews.firstIndex(of: end!) == 1)) { + // Nothing to do, elements on their places + return + } + containerView.removeConstraints(containerView.constraints) + containerView.subviews.forEach { sub in sub.removeFromSuperview()} - containerView.addConstraints(NSLayoutConstraint.constraints(withVisualFormat: "V:|[view]|", + containerView.addSubview(fullscreen) + fullscreen.translatesAutoresizingMaskIntoConstraints = false + fullscreen.layer.cornerRadius = 0 + fullscreen.layer.masksToBounds = false + + if let end { + containerView.addSubview(end) + end.translatesAutoresizingMaskIntoConstraints = false + end.layer.cornerRadius = pip ? 8 : 10 + end.layer.masksToBounds = true + } + + let constraintFullscreenV = NSLayoutConstraint.constraints( + withVisualFormat: "V:|[fullscreen]|", options: [], metrics: nil, - views: ["view": view])) + views: ["fullscreen": fullscreen] + ) + let constraintFullscreenH = NSLayoutConstraint.constraints( + withVisualFormat: "H:|[fullscreen]|", + options: [], + metrics: nil, + views: ["fullscreen": fullscreen] + ) + + containerView.addConstraints(constraintFullscreenV) + containerView.addConstraints(constraintFullscreenH) + + if let end { + let constraintEndWidth = NSLayoutConstraint( + item: end, attribute: .width, relatedBy: .equal, toItem: containerView, attribute: .width, multiplier: pip ? 0.5 : 0.3, constant: 0 + ) + let constraintEndHeight = NSLayoutConstraint( + item: end, attribute: .height, relatedBy: .equal, toItem: containerView, attribute: .width, multiplier: pip ? 0.5 * 1.33 : 0.3 * 1.33, constant: 0 + ) + let constraintEndX = NSLayoutConstraint( + item: end, attribute: .leading, relatedBy: .equal, toItem: containerView, attribute: .trailing, multiplier: pip ? 0.5 : 0.7, constant: pip ? -8 : -17 + ) + let constraintEndY = NSLayoutConstraint( + item: end, attribute: .bottom, relatedBy: .equal, toItem: containerView, attribute: .bottom, multiplier: 1, constant: pip ? -8 : -92 + ) + containerView.addConstraint(constraintEndWidth) + containerView.addConstraint(constraintEndHeight) + containerView.addConstraint(constraintEndX) + containerView.addConstraint(constraintEndY) + } containerView.layoutIfNeeded() } diff --git a/apps/ios/Shared/Views/Call/WebRTC.swift b/apps/ios/Shared/Views/Call/WebRTC.swift index ba990981a1..ef9135761c 100644 --- a/apps/ios/Shared/Views/Call/WebRTC.swift +++ b/apps/ios/Shared/Views/Call/WebRTC.swift @@ -19,14 +19,13 @@ class Call: ObservableObject, Equatable { var direction: CallDirection var contact: Contact var callUUID: String? - var localMedia: CallMediaType + var initialCallType: CallMediaType + @Published var localMediaSources: CallMediaSources @Published var callState: CallState @Published var localCapabilities: CallCapabilities? - @Published var peerMedia: CallMediaType? + @Published var peerMediaSources: CallMediaSources = CallMediaSources() @Published var sharedKey: String? - @Published var audioEnabled = true @Published var speakerEnabled = false - @Published var videoEnabled: Bool @Published var connectionInfo: ConnectionInfo? @Published var connectedAt: Date? = nil @@ -35,32 +34,33 @@ class Call: ObservableObject, Equatable { contact: Contact, callUUID: String?, callState: CallState, - localMedia: CallMediaType, + initialCallType: CallMediaType, sharedKey: String? = nil ) { self.direction = direction self.contact = contact self.callUUID = callUUID self.callState = callState - self.localMedia = localMedia + self.initialCallType = initialCallType self.sharedKey = sharedKey - self.videoEnabled = localMedia == .video + self.localMediaSources = CallMediaSources( + mic: AVCaptureDevice.authorizationStatus(for: .audio) == .authorized, + camera: initialCallType == .video && AVCaptureDevice.authorizationStatus(for: .video) == .authorized) } var encrypted: Bool { get { localEncrypted && sharedKey != nil } } - var localEncrypted: Bool { get { localCapabilities?.encryption ?? false } } + private var localEncrypted: Bool { get { localCapabilities?.encryption ?? false } } var encryptionStatus: LocalizedStringKey { get { switch callState { case .waitCapabilities: return "" case .invitationSent: return localEncrypted ? "e2e encrypted" : "no e2e encryption" case .invitationAccepted: return sharedKey == nil ? "contact has no e2e encryption" : "contact has e2e encryption" - default: return !localEncrypted ? "no e2e encryption" : sharedKey == nil ? "contact has no e2e encryption" : "e2e encrypted" + default: return !localEncrypted ? "no e2e encryption" : sharedKey == nil ? "contact has no e2e encryption" : "e2e encrypted" } } } - var hasMedia: Bool { get { callState == .offerSent || callState == .negotiated || callState == .connected } } - var supportsVideo: Bool { get { peerMedia == .video || localMedia == .video } } + var hasVideo: Bool { get { localMediaSources.hasVideo || peerMediaSources.hasVideo } } } enum CallDirection { @@ -105,18 +105,28 @@ struct WVAPIMessage: Equatable, Decodable, Encodable { var command: WCallCommand? } +struct CallMediaSources: Equatable, Codable { + var mic: Bool = false + var camera: Bool = false + var screenAudio: Bool = false + var screenVideo: Bool = false + + var hasVideo: Bool { get { camera || screenVideo } } +} + enum WCallCommand: Equatable, Encodable, Decodable { case capabilities(media: CallMediaType) case start(media: CallMediaType, aesKey: String? = nil, iceServers: [RTCIceServer]? = nil, relay: Bool? = nil) case offer(offer: String, iceCandidates: String, media: CallMediaType, aesKey: String? = nil, iceServers: [RTCIceServer]? = nil, relay: Bool? = nil) case answer(answer: String, iceCandidates: String) case ice(iceCandidates: String) - case media(media: CallMediaType, enable: Bool) + case media(source: CallMediaSource, enable: Bool) case end enum CodingKeys: String, CodingKey { case type case media + case source case aesKey case offer case answer @@ -167,9 +177,9 @@ enum WCallCommand: Equatable, Encodable, Decodable { case let .ice(iceCandidates): try container.encode("ice", forKey: .type) try container.encode(iceCandidates, forKey: .iceCandidates) - case let .media(media, enable): + case let .media(source, enable): try container.encode("media", forKey: .type) - try container.encode(media, forKey: .media) + try container.encode(source, forKey: .media) try container.encode(enable, forKey: .enable) case .end: try container.encode("end", forKey: .type) @@ -205,9 +215,9 @@ enum WCallCommand: Equatable, Encodable, Decodable { let iceCandidates = try container.decode(String.self, forKey: CodingKeys.iceCandidates) self = .ice(iceCandidates: iceCandidates) case "media": - let media = try container.decode(CallMediaType.self, forKey: CodingKeys.media) + let source = try container.decode(CallMediaSource.self, forKey: CodingKeys.source) let enable = try container.decode(Bool.self, forKey: CodingKeys.enable) - self = .media(media: media, enable: enable) + self = .media(source: source, enable: enable) case "end": self = .end default: @@ -224,6 +234,7 @@ enum WCallResponse: Equatable, Decodable { case ice(iceCandidates: String) case connection(state: ConnectionState) case connected(connectionInfo: ConnectionInfo) + case peerMedia(source: CallMediaSource, enabled: Bool) case ended case ok case error(message: String) @@ -238,6 +249,8 @@ enum WCallResponse: Equatable, Decodable { case state case connectionInfo case message + case source + case enabled } var respType: String { @@ -249,6 +262,7 @@ enum WCallResponse: Equatable, Decodable { case .ice: return "ice" case .connection: return "connection" case .connected: return "connected" + case .peerMedia: return "peerMedia" case .ended: return "ended" case .ok: return "ok" case .error: return "error" @@ -283,6 +297,10 @@ enum WCallResponse: Equatable, Decodable { case "connected": let connectionInfo = try container.decode(ConnectionInfo.self, forKey: CodingKeys.connectionInfo) self = .connected(connectionInfo: connectionInfo) + case "peerMedia": + let source = try container.decode(CallMediaSource.self, forKey: CodingKeys.source) + let enabled = try container.decode(Bool.self, forKey: CodingKeys.enabled) + self = .peerMedia(source: source, enabled: enabled) case "ended": self = .ended case "ok": @@ -324,6 +342,10 @@ extension WCallResponse: Encodable { case let .connected(connectionInfo): try container.encode("connected", forKey: .type) try container.encode(connectionInfo, forKey: .connectionInfo) + case let .peerMedia(source, enabled): + try container.encode("peerMedia", forKey: .type) + try container.encode(source, forKey: .source) + try container.encode(enabled, forKey: .enabled) case .ended: try container.encode("ended", forKey: .type) case .ok: @@ -376,7 +398,7 @@ actor WebRTCCommandProcessor { func shouldRunCommand(_ client: WebRTCClient, _ c: WCallCommand) -> Bool { switch c { case .capabilities, .start, .offer, .end: true - default: client.activeCall.wrappedValue != nil + default: client.activeCall != nil } } } diff --git a/apps/ios/Shared/Views/Call/WebRTCClient.swift b/apps/ios/Shared/Views/Call/WebRTCClient.swift index 79e0dea16c..389e5d0503 100644 --- a/apps/ios/Shared/Views/Call/WebRTCClient.swift +++ b/apps/ios/Shared/Views/Call/WebRTCClient.swift @@ -23,15 +23,24 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg struct Call { var connection: RTCPeerConnection var iceCandidates: IceCandidates - var localMedia: CallMediaType var localCamera: RTCVideoCapturer? - var localVideoSource: RTCVideoSource? - var localStream: RTCVideoTrack? - var remoteStream: RTCVideoTrack? - var device: AVCaptureDevice.Position = .front + var localAudioTrack: RTCAudioTrack? + var localVideoTrack: RTCVideoTrack? + var remoteAudioTrack: RTCAudioTrack? + var remoteVideoTrack: RTCVideoTrack? + var remoteScreenAudioTrack: RTCAudioTrack? + var remoteScreenVideoTrack: RTCVideoTrack? + var device: AVCaptureDevice.Position var aesKey: String? var frameEncryptor: RTCFrameEncryptor? var frameDecryptor: RTCFrameDecryptor? + var peerHasOldVersion: Bool + } + + struct NotConnectedCall { + var audioTrack: RTCAudioTrack? + var localCameraAndTrack: (RTCVideoCapturer, RTCVideoTrack)? + var device: AVCaptureDevice.Position = .front } actor IceCandidates { @@ -51,17 +60,20 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg private let rtcAudioSession = RTCAudioSession.sharedInstance() private let audioQueue = DispatchQueue(label: "chat.simplex.app.audio") private var sendCallResponse: (WVAPIMessage) async -> Void - var activeCall: Binding + var activeCall: Call? + var notConnectedCall: NotConnectedCall? private var localRendererAspectRatio: Binding + var cameraRenderers: [RTCVideoRenderer] = [] + var screenRenderers: [RTCVideoRenderer] = [] + @available(*, unavailable) override init() { fatalError("Unimplemented") } - required init(_ activeCall: Binding, _ sendCallResponse: @escaping (WVAPIMessage) async -> Void, _ localRendererAspectRatio: Binding) { + required init(_ sendCallResponse: @escaping (WVAPIMessage) async -> Void, _ localRendererAspectRatio: Binding) { self.sendCallResponse = sendCallResponse - self.activeCall = activeCall self.localRendererAspectRatio = localRendererAspectRatio rtcAudioSession.useManualAudio = CallController.useCallKit() rtcAudioSession.isAudioEnabled = !CallController.useCallKit() @@ -78,39 +90,45 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg func initializeCall(_ iceServers: [WebRTC.RTCIceServer]?, _ mediaType: CallMediaType, _ aesKey: String?, _ relay: Bool?) -> Call { let connection = createPeerConnection(iceServers ?? getWebRTCIceServers() ?? defaultIceServers, relay) connection.delegate = self - createAudioSender(connection) - var localStream: RTCVideoTrack? = nil - var remoteStream: RTCVideoTrack? = nil + let device = notConnectedCall?.device ?? .front var localCamera: RTCVideoCapturer? = nil - var localVideoSource: RTCVideoSource? = nil - if mediaType == .video { - (localStream, remoteStream, localCamera, localVideoSource) = createVideoSender(connection) + var localAudioTrack: RTCAudioTrack? = nil + var localVideoTrack: RTCVideoTrack? = nil + if let localCameraAndTrack = notConnectedCall?.localCameraAndTrack { + (localCamera, localVideoTrack) = localCameraAndTrack + } else if notConnectedCall == nil && mediaType == .video { + (localCamera, localVideoTrack) = createVideoTrackAndStartCapture(device) } + if let audioTrack = notConnectedCall?.audioTrack { + localAudioTrack = audioTrack + } else if notConnectedCall == nil { + localAudioTrack = createAudioTrack() + } + notConnectedCall?.localCameraAndTrack = nil + notConnectedCall?.audioTrack = nil + var frameEncryptor: RTCFrameEncryptor? = nil var frameDecryptor: RTCFrameDecryptor? = nil if aesKey != nil { let encryptor = RTCFrameEncryptor.init(sizeChange: Int32(WebRTCClient.ivTagBytes)) encryptor.delegate = self frameEncryptor = encryptor - connection.senders.forEach { $0.setRtcFrameEncryptor(encryptor) } let decryptor = RTCFrameDecryptor.init(sizeChange: -Int32(WebRTCClient.ivTagBytes)) decryptor.delegate = self frameDecryptor = decryptor - // Has no video receiver in outgoing call if applied here, see [peerConnection(_ connection: RTCPeerConnection, didChange newState] - // connection.receivers.forEach { $0.setRtcFrameDecryptor(decryptor) } } return Call( connection: connection, iceCandidates: IceCandidates(), - localMedia: mediaType, localCamera: localCamera, - localVideoSource: localVideoSource, - localStream: localStream, - remoteStream: remoteStream, + localAudioTrack: localAudioTrack, + localVideoTrack: localVideoTrack, + device: device, aesKey: aesKey, frameEncryptor: frameEncryptor, - frameDecryptor: frameDecryptor + frameDecryptor: frameDecryptor, + peerHasOldVersion: false ) } @@ -151,18 +169,24 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg func sendCallCommand(command: WCallCommand) async { var resp: WCallResponse? = nil - let pc = activeCall.wrappedValue?.connection + let pc = activeCall?.connection switch command { - case .capabilities: + case let .capabilities(media): // outgoing + let localCameraAndTrack: (RTCVideoCapturer, RTCVideoTrack)? = media == .video + ? createVideoTrackAndStartCapture(.front) + : nil + notConnectedCall = NotConnectedCall(audioTrack: createAudioTrack(), localCameraAndTrack: localCameraAndTrack, device: .front) resp = .capabilities(capabilities: CallCapabilities(encryption: WebRTCClient.enableEncryption)) - case let .start(media: media, aesKey, iceServers, relay): + case let .start(media: media, aesKey, iceServers, relay): // incoming logger.debug("starting incoming call - create webrtc session") - if activeCall.wrappedValue != nil { endCall() } + if activeCall != nil { endCall() } let encryption = WebRTCClient.enableEncryption let call = initializeCall(iceServers?.toWebRTCIceServers(), media, encryption ? aesKey : nil, relay) - activeCall.wrappedValue = call + activeCall = call + setupLocalTracks(true, call) let (offer, error) = await call.connection.offer() if let offer = offer { + setupEncryptionForLocalTracks(call) resp = .offer( offer: compressToBase64(input: encodeJSON(CustomRTCSessionDescription(type: offer.type.toSdpType(), sdp: offer.sdp))), iceCandidates: compressToBase64(input: encodeJSON(await self.getInitialIceCandidates())), @@ -172,18 +196,24 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg } else { resp = .error(message: "offer error: \(error?.localizedDescription ?? "unknown error")") } - case let .offer(offer, iceCandidates, media, aesKey, iceServers, relay): - if activeCall.wrappedValue != nil { + case let .offer(offer, iceCandidates, media, aesKey, iceServers, relay): // outgoing + if activeCall != nil { resp = .error(message: "accept: call already started") } else if !WebRTCClient.enableEncryption && aesKey != nil { resp = .error(message: "accept: encryption is not supported") } else if let offer: CustomRTCSessionDescription = decodeJSON(decompressFromBase64(input: offer)), let remoteIceCandidates: [RTCIceCandidate] = decodeJSON(decompressFromBase64(input: iceCandidates)) { let call = initializeCall(iceServers?.toWebRTCIceServers(), media, WebRTCClient.enableEncryption ? aesKey : nil, relay) - activeCall.wrappedValue = call + activeCall = call let pc = call.connection if let type = offer.type, let sdp = offer.sdp { if (try? await pc.setRemoteDescription(RTCSessionDescription(type: type.toWebRTCSdpType(), sdp: sdp))) != nil { + setupLocalTracks(false, call) + setupEncryptionForLocalTracks(call) + pc.transceivers.forEach { transceiver in + transceiver.setDirection(.sendRecv, error: nil) + } + await adaptToOldVersion(pc.transceivers.count <= 2) let (answer, error) = await pc.answer() if let answer = answer { self.addIceCandidates(pc, remoteIceCandidates) @@ -200,7 +230,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg } } } - case let .answer(answer, iceCandidates): + case let .answer(answer, iceCandidates): // incoming if pc == nil { resp = .error(message: "answer: call not started") } else if pc?.localDescription == nil { @@ -212,6 +242,9 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg let type = answer.type, let sdp = answer.sdp, let pc = pc { if (try? await pc.setRemoteDescription(RTCSessionDescription(type: type.toWebRTCSdpType(), sdp: sdp))) != nil { + var currentDirection: RTCRtpTransceiverDirection = .sendOnly + pc.transceivers[2].currentDirection(¤tDirection) + await adaptToOldVersion(currentDirection == .sendOnly) addIceCandidates(pc, remoteIceCandidates) resp = .ok } else { @@ -226,13 +259,11 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg } else { resp = .error(message: "ice: call not started") } - case let .media(media, enable): - if activeCall.wrappedValue == nil { + case let .media(source, enable): + if activeCall == nil { resp = .error(message: "media: call not started") - } else if activeCall.wrappedValue?.localMedia == .audio && media == .video { - resp = .error(message: "media: no video") } else { - enableMedia(media, enable) + await enableMedia(source, enable) resp = .ok } case .end: @@ -247,7 +278,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg func getInitialIceCandidates() async -> [RTCIceCandidate] { await untilIceComplete(timeoutMs: 750, stepMs: 150) {} - let candidates = await activeCall.wrappedValue?.iceCandidates.getAndClear() ?? [] + let candidates = await activeCall?.iceCandidates.getAndClear() ?? [] logger.debug("WebRTCClient: sending initial ice candidates: \(candidates.count)") return candidates } @@ -255,7 +286,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg func waitForMoreIceCandidates() { Task { await untilIceComplete(timeoutMs: 12000, stepMs: 1500) { - let candidates = await self.activeCall.wrappedValue?.iceCandidates.getAndClear() ?? [] + let candidates = await self.activeCall?.iceCandidates.getAndClear() ?? [] if candidates.count > 0 { logger.debug("WebRTCClient: sending more ice candidates: \(candidates.count)") await self.sendIceCandidates(candidates) @@ -272,25 +303,203 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg ) } - func enableMedia(_ media: CallMediaType, _ enable: Bool) { - logger.debug("WebRTCClient: enabling media \(media.rawValue) \(enable)") - media == .video ? setVideoEnabled(enable) : setAudioEnabled(enable) + func setupMuteUnmuteListener(_ transceiver: RTCRtpTransceiver, _ track: RTCMediaStreamTrack) { + // logger.log("Setting up mute/unmute listener in the call without encryption for mid = \(transceiver.mid)") + Task { + // for some reason even for disabled tracks one packet arrives (seeing this on screenVideo track) + var lastPacketsReceived = 1 + // muted initially + var mutedSeconds = 4 + while let call = self.activeCall, transceiver.receiver.track?.readyState == .live { + let stats: RTCStatisticsReport = await call.connection.statistics(for: transceiver.receiver) + let stat = stats.statistics.values.first(where: { stat in stat.type == "inbound-rtp"}) + if let stat { + //logger.debug("Stat \(stat.debugDescription)") + let packets = stat.values["packetsReceived"] as! Int + if packets <= lastPacketsReceived { + mutedSeconds += 1 + if mutedSeconds == 3 { + await MainActor.run { + self.onMediaMuteUnmute(transceiver.mid, true) + } + } + } else { + if mutedSeconds >= 3 { + await MainActor.run { + self.onMediaMuteUnmute(transceiver.mid, false) + } + } + lastPacketsReceived = packets + mutedSeconds = 0 + } + } + try? await Task.sleep(nanoseconds: 1000_000000) + } + } } - func addLocalRenderer(_ activeCall: Call, _ renderer: RTCEAGLVideoView) { - activeCall.localStream?.add(renderer) + @MainActor + func onMediaMuteUnmute(_ transceiverMid: String?, _ mute: Bool) { + guard let activeCall = ChatModel.shared.activeCall else { return } + let source = mediaSourceFromTransceiverMid(transceiverMid) + logger.log("Mute/unmute \(source.rawValue) track = \(mute) with mid = \(transceiverMid ?? "nil")") + if source == .mic && activeCall.peerMediaSources.mic == mute { + activeCall.peerMediaSources.mic = !mute + } else if (source == .camera && activeCall.peerMediaSources.camera == mute) { + activeCall.peerMediaSources.camera = !mute + } else if (source == .screenAudio && activeCall.peerMediaSources.screenAudio == mute) { + activeCall.peerMediaSources.screenAudio = !mute + } else if (source == .screenVideo && activeCall.peerMediaSources.screenVideo == mute) { + activeCall.peerMediaSources.screenVideo = !mute + } + } + + @MainActor + func enableMedia(_ source: CallMediaSource, _ enable: Bool) { + logger.debug("WebRTCClient: enabling media \(source.rawValue) \(enable)") + source == .camera ? setCameraEnabled(enable) : setAudioEnabled(enable) + } + + @MainActor + func adaptToOldVersion(_ peerHasOldVersion: Bool) { + activeCall?.peerHasOldVersion = peerHasOldVersion + if peerHasOldVersion { + logger.debug("The peer has an old version. Remote audio track is nil = \(self.activeCall?.remoteAudioTrack == nil), video = \(self.activeCall?.remoteVideoTrack == nil)") + onMediaMuteUnmute("0", false) + if activeCall?.remoteVideoTrack != nil { + onMediaMuteUnmute("1", false) + } + if ChatModel.shared.activeCall?.localMediaSources.camera == true && ChatModel.shared.activeCall?.peerMediaSources.camera == false { + logger.debug("Stopping video track for the old version") + activeCall?.connection.senders[1].track = nil + ChatModel.shared.activeCall?.localMediaSources.camera = false + (activeCall?.localCamera as? RTCCameraVideoCapturer)?.stopCapture() + activeCall?.localCamera = nil + activeCall?.localVideoTrack = nil + } + } + } + + func addLocalRenderer(_ renderer: RTCEAGLVideoView) { + if let activeCall { + if let track = activeCall.localVideoTrack { + track.add(renderer) + } + } else if let notConnectedCall { + if let track = notConnectedCall.localCameraAndTrack?.1 { + track.add(renderer) + } + } // To get width and height of a frame, see videoView(videoView:, didChangeVideoSize) renderer.delegate = self } + func removeLocalRenderer(_ renderer: RTCEAGLVideoView) { + if let activeCall { + if let track = activeCall.localVideoTrack { + track.remove(renderer) + } + } else if let notConnectedCall { + if let track = notConnectedCall.localCameraAndTrack?.1 { + track.remove(renderer) + } + } + renderer.delegate = nil + } + func videoView(_ videoView: RTCVideoRenderer, didChangeVideoSize size: CGSize) { guard size.height > 0 else { return } localRendererAspectRatio.wrappedValue = size.width / size.height } + func setupLocalTracks(_ incomingCall: Bool, _ call: Call) { + let pc = call.connection + let transceivers = call.connection.transceivers + let audioTrack = call.localAudioTrack + let videoTrack = call.localVideoTrack + + if incomingCall { + let micCameraInit = RTCRtpTransceiverInit() + // streamIds required for old versions which adds tracks from stream, not from track property + micCameraInit.streamIds = ["micCamera"] + + let screenAudioVideoInit = RTCRtpTransceiverInit() + screenAudioVideoInit.streamIds = ["screenAudioVideo"] + + // incoming call, no transceivers yet. But they should be added in order: mic, camera, screen audio, screen video + // mid = 0, mic + if let audioTrack { + pc.addTransceiver(with: audioTrack, init: micCameraInit) + } else { + pc.addTransceiver(of: .audio, init: micCameraInit) + } + // mid = 1, camera + if let videoTrack { + pc.addTransceiver(with: videoTrack, init: micCameraInit) + } else { + pc.addTransceiver(of: .video, init: micCameraInit) + } + // mid = 2, screenAudio + pc.addTransceiver(of: .audio, init: screenAudioVideoInit) + // mid = 3, screenVideo + pc.addTransceiver(of: .video, init: screenAudioVideoInit) + } else { + // new version + if transceivers.count > 2 { + // Outgoing call. All transceivers are ready. Don't addTrack() because it will create new transceivers, replace existing (nil) tracks + transceivers + .first(where: { elem in mediaSourceFromTransceiverMid(elem.mid) == .mic })? + .sender.track = audioTrack + transceivers + .first(where: { elem in mediaSourceFromTransceiverMid(elem.mid) == .camera })? + .sender.track = videoTrack + } else { + // old version, only two transceivers + if let audioTrack { + pc.add(audioTrack, streamIds: ["micCamera"]) + } else { + // it's important to have any track in order to be able to turn it on again (currently it's off) + let sender = pc.add(createAudioTrack(), streamIds: ["micCamera"]) + sender?.track = nil + } + if let videoTrack { + pc.add(videoTrack, streamIds: ["micCamera"]) + } else { + // it's important to have any track in order to be able to turn it on again (currently it's off) + let localVideoSource = WebRTCClient.factory.videoSource() + let localVideoTrack = WebRTCClient.factory.videoTrack(with: localVideoSource, trackId: "video0") + let sender = pc.add(localVideoTrack, streamIds: ["micCamera"]) + sender?.track = nil + } + } + } + } + + func mediaSourceFromTransceiverMid(_ mid: String?) -> CallMediaSource { + switch mid { + case "0": + return .mic + case "1": + return .camera + case "2": + return .screenAudio + case "3": + return .screenVideo + default: + return .unknown + } + } + + // Should be called after local description set + func setupEncryptionForLocalTracks(_ call: Call) { + if let encryptor = call.frameEncryptor { + call.connection.senders.forEach { $0.setRtcFrameEncryptor(encryptor) } + } + } + func frameDecryptor(_ decryptor: RTCFrameDecryptor, mediaType: RTCRtpMediaType, withFrame encrypted: Data) -> Data? { guard encrypted.count > 0 else { return nil } - if var key: [CChar] = activeCall.wrappedValue?.aesKey?.cString(using: .utf8), + if var key: [CChar] = activeCall?.aesKey?.cString(using: .utf8), let pointer: UnsafeMutableRawPointer = malloc(encrypted.count) { memcpy(pointer, (encrypted as NSData).bytes, encrypted.count) let isKeyFrame = encrypted[0] & 1 == 0 @@ -304,7 +513,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg func frameEncryptor(_ encryptor: RTCFrameEncryptor, mediaType: RTCRtpMediaType, withFrame unencrypted: Data) -> Data? { guard unencrypted.count > 0 else { return nil } - if var key: [CChar] = activeCall.wrappedValue?.aesKey?.cString(using: .utf8), + if var key: [CChar] = activeCall?.aesKey?.cString(using: .utf8), let pointer: UnsafeMutableRawPointer = malloc(unencrypted.count + WebRTCClient.ivTagBytes) { memcpy(pointer, (unencrypted as NSData).bytes, unencrypted.count) let isKeyFrame = unencrypted[0] & 1 == 0 @@ -327,18 +536,42 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg } } - func addRemoteRenderer(_ activeCall: Call, _ renderer: RTCVideoRenderer) { - activeCall.remoteStream?.add(renderer) + func addRemoteCameraRenderer(_ renderer: RTCVideoRenderer) { + if activeCall?.remoteVideoTrack != nil { + activeCall?.remoteVideoTrack?.add(renderer) + } else { + cameraRenderers.append(renderer) + } } - func removeRemoteRenderer(_ activeCall: Call, _ renderer: RTCVideoRenderer) { - activeCall.remoteStream?.remove(renderer) + func removeRemoteCameraRenderer(_ renderer: RTCVideoRenderer) { + if activeCall?.remoteVideoTrack != nil { + activeCall?.remoteVideoTrack?.remove(renderer) + } else { + cameraRenderers.removeAll(where: { $0.isEqual(renderer) }) + } } - func startCaptureLocalVideo(_ activeCall: Call) { + func addRemoteScreenRenderer(_ renderer: RTCVideoRenderer) { + if activeCall?.remoteScreenVideoTrack != nil { + activeCall?.remoteScreenVideoTrack?.add(renderer) + } else { + screenRenderers.append(renderer) + } + } + + func removeRemoteScreenRenderer(_ renderer: RTCVideoRenderer) { + if activeCall?.remoteScreenVideoTrack != nil { + activeCall?.remoteScreenVideoTrack?.remove(renderer) + } else { + screenRenderers.removeAll(where: { $0.isEqual(renderer) }) + } + } + + func startCaptureLocalVideo(_ device: AVCaptureDevice.Position?, _ capturer: RTCVideoCapturer?) { #if targetEnvironment(simulator) guard - let capturer = activeCall.localCamera as? RTCFileVideoCapturer + let capturer = (activeCall?.localCamera ?? notConnectedCall?.localCameraAndTrack?.0) as? RTCFileVideoCapturer else { logger.error("Unable to work with a file capturer") return @@ -348,10 +581,10 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg capturer.startCapturing(fromFileNamed: "sounds/video.mp4") #else guard - let capturer = activeCall.localCamera as? RTCCameraVideoCapturer, - let camera = (RTCCameraVideoCapturer.captureDevices().first { $0.position == activeCall.device }) + let capturer = capturer as? RTCCameraVideoCapturer, + let camera = (RTCCameraVideoCapturer.captureDevices().first { $0.position == device }) else { - logger.error("Unable to find a camera") + logger.error("Unable to find a camera or local track") return } @@ -377,19 +610,6 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg #endif } - private func createAudioSender(_ connection: RTCPeerConnection) { - let streamId = "stream" - let audioTrack = createAudioTrack() - connection.add(audioTrack, streamIds: [streamId]) - } - - private func createVideoSender(_ connection: RTCPeerConnection) -> (RTCVideoTrack?, RTCVideoTrack?, RTCVideoCapturer?, RTCVideoSource?) { - let streamId = "stream" - let (localVideoTrack, localCamera, localVideoSource) = createVideoTrack() - connection.add(localVideoTrack, streamIds: [streamId]) - return (localVideoTrack, connection.transceivers.first { $0.mediaType == .video }?.receiver.track as? RTCVideoTrack, localCamera, localVideoSource) - } - private func createAudioTrack() -> RTCAudioTrack { let audioConstrains = RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil) let audioSource = WebRTCClient.factory.audioSource(with: audioConstrains) @@ -397,7 +617,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg return audioTrack } - private func createVideoTrack() -> (RTCVideoTrack, RTCVideoCapturer, RTCVideoSource) { + private func createVideoTrackAndStartCapture(_ device: AVCaptureDevice.Position) -> (RTCVideoCapturer, RTCVideoTrack) { let localVideoSource = WebRTCClient.factory.videoSource() #if targetEnvironment(simulator) @@ -407,7 +627,8 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg #endif let localVideoTrack = WebRTCClient.factory.videoTrack(with: localVideoSource, trackId: "video0") - return (localVideoTrack, localCamera, localVideoSource) + startCaptureLocalVideo(device, localCamera) + return (localCamera, localVideoTrack) } func endCall() { @@ -420,15 +641,16 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg } private func _endCall() { - guard let call = activeCall.wrappedValue else { return } + (notConnectedCall?.localCameraAndTrack?.0 as? RTCCameraVideoCapturer)?.stopCapture() + guard let call = activeCall else { return } logger.debug("WebRTCClient: ending the call") - activeCall.wrappedValue = nil - (call.localCamera as? RTCCameraVideoCapturer)?.stopCapture() call.connection.close() call.connection.delegate = nil call.frameEncryptor?.delegate = nil call.frameDecryptor?.delegate = nil + (call.localCamera as? RTCCameraVideoCapturer)?.stopCapture() audioSessionToDefaults() + activeCall = nil } func untilIceComplete(timeoutMs: UInt64, stepMs: UInt64, action: @escaping () async -> Void) async { @@ -437,7 +659,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg _ = try? await Task.sleep(nanoseconds: stepMs * 1000000) t += stepMs await action() - } while t < timeoutMs && activeCall.wrappedValue?.connection.iceGatheringState != .complete + } while t < timeoutMs && activeCall?.connection.iceGatheringState != .complete } } @@ -498,11 +720,40 @@ extension WebRTCClient: RTCPeerConnectionDelegate { logger.debug("Connection should negotiate") } + func peerConnection(_ peerConnection: RTCPeerConnection, didStartReceivingOn transceiver: RTCRtpTransceiver) { + if let track = transceiver.receiver.track { + DispatchQueue.main.async { + // Doesn't work for outgoing video call (audio in video call works ok still, same as incoming call) +// if let decryptor = self.activeCall?.frameDecryptor { +// transceiver.receiver.setRtcFrameDecryptor(decryptor) +// } + let source = self.mediaSourceFromTransceiverMid(transceiver.mid) + switch source { + case .mic: self.activeCall?.remoteAudioTrack = track as? RTCAudioTrack + case .camera: + self.activeCall?.remoteVideoTrack = track as? RTCVideoTrack + self.cameraRenderers.forEach({ renderer in + self.activeCall?.remoteVideoTrack?.add(renderer) + }) + self.cameraRenderers.removeAll() + case .screenAudio: self.activeCall?.remoteScreenAudioTrack = track as? RTCAudioTrack + case .screenVideo: + self.activeCall?.remoteScreenVideoTrack = track as? RTCVideoTrack + self.screenRenderers.forEach({ renderer in + self.activeCall?.remoteScreenVideoTrack?.add(renderer) + }) + self.screenRenderers.removeAll() + case .unknown: () + } + } + self.setupMuteUnmuteListener(transceiver, track) + } + } + func peerConnection(_ connection: RTCPeerConnection, didChange newState: RTCIceConnectionState) { debugPrint("Connection new connection state: \(newState.toString() ?? "" + newState.rawValue.description) \(connection.receivers)") - guard let call = activeCall.wrappedValue, - let connectionStateString = newState.toString(), + guard let connectionStateString = newState.toString(), let iceConnectionStateString = connection.iceConnectionState.toString(), let iceGatheringStateString = connection.iceGatheringState.toString(), let signalingStateString = connection.signalingState.toString() @@ -523,18 +774,14 @@ extension WebRTCClient: RTCPeerConnectionDelegate { switch newState { case .checking: - if let frameDecryptor = activeCall.wrappedValue?.frameDecryptor { + if let frameDecryptor = activeCall?.frameDecryptor { connection.receivers.forEach { $0.setRtcFrameDecryptor(frameDecryptor) } } - let enableSpeaker: Bool - switch call.localMedia { - case .video: enableSpeaker = true - default: enableSpeaker = false - } + let enableSpeaker: Bool = ChatModel.shared.activeCall?.localMediaSources.hasVideo == true setSpeakerEnabledAndConfigureSession(enableSpeaker) case .connected: sendConnectedEvent(connection) case .disconnected, .failed: endCall() - default: do {} + default: () } } } @@ -546,7 +793,7 @@ extension WebRTCClient: RTCPeerConnectionDelegate { func peerConnection(_ connection: RTCPeerConnection, didGenerate candidate: WebRTC.RTCIceCandidate) { // logger.debug("Connection generated candidate \(candidate.debugDescription)") Task { - await self.activeCall.wrappedValue?.iceCandidates.append(candidate.toCandidate(nil, nil)) + await self.activeCall?.iceCandidates.append(candidate.toCandidate(nil, nil)) } } @@ -601,11 +848,42 @@ extension WebRTCClient: RTCPeerConnectionDelegate { } extension WebRTCClient { - func setAudioEnabled(_ enabled: Bool) { - setTrackEnabled(RTCAudioTrack.self, enabled) + static func isAuthorized(for type: AVMediaType) async -> Bool { + let status = AVCaptureDevice.authorizationStatus(for: type) + var isAuthorized = status == .authorized + if status == .notDetermined { + isAuthorized = await AVCaptureDevice.requestAccess(for: type) + } + return isAuthorized } - func setSpeakerEnabledAndConfigureSession( _ enabled: Bool) { + static func showUnauthorizedAlert(for type: AVMediaType) { + if type == .audio { + AlertManager.shared.showAlert(Alert( + title: Text("No permission to record speech"), + message: Text("To record speech please grant permission to use Microphone."), + primaryButton: .default(Text("Open Settings")) { + DispatchQueue.main.async { + UIApplication.shared.open(URL(string: UIApplication.openSettingsURLString)!, options: [:], completionHandler: nil) + } + }, + secondaryButton: .cancel() + )) + } else if type == .video { + AlertManager.shared.showAlert(Alert( + title: Text("No permission to record video"), + message: Text("To record video please grant permission to use Camera."), + primaryButton: .default(Text("Open Settings")) { + DispatchQueue.main.async { + UIApplication.shared.open(URL(string: UIApplication.openSettingsURLString)!, options: [:], completionHandler: nil) + } + }, + secondaryButton: .cancel() + )) + } + } + + func setSpeakerEnabledAndConfigureSession( _ enabled: Bool, skipExternalDevice: Bool = false) { logger.debug("WebRTCClient: configuring session with speaker enabled \(enabled)") audioQueue.async { [weak self] in guard let self = self else { return } @@ -618,7 +896,7 @@ extension WebRTCClient { if enabled { try self.rtcAudioSession.setCategory(AVAudioSession.Category.playAndRecord.rawValue, with: [.defaultToSpeaker, .allowBluetooth, .allowAirPlay, .allowBluetoothA2DP]) try self.rtcAudioSession.setMode(AVAudioSession.Mode.videoChat.rawValue) - if hasExternalAudioDevice, let preferred = self.rtcAudioSession.session.preferredInputDevice() { + if hasExternalAudioDevice && !skipExternalDevice, let preferred = self.rtcAudioSession.session.preferredInputDevice() { try self.rtcAudioSession.setPreferredInput(preferred) } else { try self.rtcAudioSession.overrideOutputAudioPort(.speaker) @@ -628,7 +906,7 @@ extension WebRTCClient { try self.rtcAudioSession.setMode(AVAudioSession.Mode.voiceChat.rawValue) try self.rtcAudioSession.overrideOutputAudioPort(.none) } - if hasExternalAudioDevice { + if hasExternalAudioDevice && !skipExternalDevice { logger.debug("WebRTCClient: configuring session with external device available, skip configuring speaker") } try self.rtcAudioSession.setActive(true) @@ -659,25 +937,59 @@ extension WebRTCClient { } } - func setVideoEnabled(_ enabled: Bool) { - setTrackEnabled(RTCVideoTrack.self, enabled) + @MainActor + func setAudioEnabled(_ enabled: Bool) { + if activeCall != nil { + activeCall?.localAudioTrack = enabled ? createAudioTrack() : nil + activeCall?.connection.transceivers.first(where: { t in mediaSourceFromTransceiverMid(t.mid) == .mic })?.sender.track = activeCall?.localAudioTrack + } else if notConnectedCall != nil { + notConnectedCall?.audioTrack = enabled ? createAudioTrack() : nil + } + ChatModel.shared.activeCall?.localMediaSources.mic = enabled + } + + @MainActor + func setCameraEnabled(_ enabled: Bool) { + if let call = activeCall { + if enabled { + if call.localVideoTrack == nil { + let device = activeCall?.device ?? notConnectedCall?.device ?? .front + let (camera, track) = createVideoTrackAndStartCapture(device) + activeCall?.localCamera = camera + activeCall?.localVideoTrack = track + } + } else { + (call.localCamera as? RTCCameraVideoCapturer)?.stopCapture() + activeCall?.localCamera = nil + activeCall?.localVideoTrack = nil + } + call.connection.transceivers + .first(where: { t in mediaSourceFromTransceiverMid(t.mid) == .camera })? + .sender.track = activeCall?.localVideoTrack + ChatModel.shared.activeCall?.localMediaSources.camera = activeCall?.localVideoTrack != nil + } else if let call = notConnectedCall { + if enabled { + let device = activeCall?.device ?? notConnectedCall?.device ?? .front + notConnectedCall?.localCameraAndTrack = createVideoTrackAndStartCapture(device) + } else { + (call.localCameraAndTrack?.0 as? RTCCameraVideoCapturer)?.stopCapture() + notConnectedCall?.localCameraAndTrack = nil + } + ChatModel.shared.activeCall?.localMediaSources.camera = notConnectedCall?.localCameraAndTrack != nil + } } func flipCamera() { - switch activeCall.wrappedValue?.device { - case .front: activeCall.wrappedValue?.device = .back - case .back: activeCall.wrappedValue?.device = .front - default: () + let device = activeCall?.device ?? notConnectedCall?.device + if activeCall != nil { + activeCall?.device = device == .front ? .back : .front + } else { + notConnectedCall?.device = device == .front ? .back : .front } - if let call = activeCall.wrappedValue { - startCaptureLocalVideo(call) - } - } - - private func setTrackEnabled(_ type: T.Type, _ enabled: Bool) { - activeCall.wrappedValue?.connection.transceivers - .compactMap { $0.sender.track as? T } - .forEach { $0.isEnabled = enabled } + startCaptureLocalVideo( + activeCall?.device ?? notConnectedCall?.device, + (activeCall?.localCamera ?? notConnectedCall?.localCameraAndTrack?.0) as? RTCCameraVideoCapturer + ) } } diff --git a/apps/ios/SimpleXChat/CallTypes.swift b/apps/ios/SimpleXChat/CallTypes.swift index 9f6d98e518..da1720c134 100644 --- a/apps/ios/SimpleXChat/CallTypes.swift +++ b/apps/ios/SimpleXChat/CallTypes.swift @@ -80,6 +80,14 @@ public enum CallMediaType: String, Codable, Equatable { case audio = "audio" } +public enum CallMediaSource: String, Codable, Equatable { + case mic = "mic" + case camera = "camera" + case screenAudio = "screenAudio" + case screenVideo = "screenVideo" + case unknown = "unknown" +} + public enum VideoCamera: String, Codable, Equatable { case user = "user" case environment = "environment" From ad2a0024f2ec5978cbc230203a5f4553495b2359 Mon Sep 17 00:00:00 2001 From: spaced4ndy <8711996+spaced4ndy@users.noreply.github.com> Date: Tue, 8 Oct 2024 15:26:16 +0400 Subject: [PATCH 3/4] core: add missing indexes (#4993) --- simplex-chat.cabal | 1 + .../Chat/Migrations/M20241008_indexes.hs | 18 ++++++++++++++++++ src/Simplex/Chat/Migrations/chat_schema.sql | 3 +++ src/Simplex/Chat/Store/Migrations.hs | 4 +++- 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 src/Simplex/Chat/Migrations/M20241008_indexes.hs diff --git a/simplex-chat.cabal b/simplex-chat.cabal index fbd1d222e6..cf9b71cba1 100644 --- a/simplex-chat.cabal +++ b/simplex-chat.cabal @@ -148,6 +148,7 @@ library Simplex.Chat.Migrations.M20240528_quota_err_counter Simplex.Chat.Migrations.M20240827_calls_uuid Simplex.Chat.Migrations.M20240920_user_order + Simplex.Chat.Migrations.M20241008_indexes Simplex.Chat.Mobile Simplex.Chat.Mobile.File Simplex.Chat.Mobile.Shared diff --git a/src/Simplex/Chat/Migrations/M20241008_indexes.hs b/src/Simplex/Chat/Migrations/M20241008_indexes.hs new file mode 100644 index 0000000000..94cffa8d74 --- /dev/null +++ b/src/Simplex/Chat/Migrations/M20241008_indexes.hs @@ -0,0 +1,18 @@ +{-# LANGUAGE QuasiQuotes #-} + +module Simplex.Chat.Migrations.M20241008_indexes where + +import Database.SQLite.Simple (Query) +import Database.SQLite.Simple.QQ (sql) + +m20241008_indexes :: Query +m20241008_indexes = + [sql| +CREATE INDEX idx_received_probes_group_member_id on received_probes(group_member_id); +|] + +down_m20241008_indexes :: Query +down_m20241008_indexes = + [sql| +DROP INDEX idx_received_probes_group_member_id; +|] diff --git a/src/Simplex/Chat/Migrations/chat_schema.sql b/src/Simplex/Chat/Migrations/chat_schema.sql index 214c0e72cc..ad13fb5db9 100644 --- a/src/Simplex/Chat/Migrations/chat_schema.sql +++ b/src/Simplex/Chat/Migrations/chat_schema.sql @@ -885,3 +885,6 @@ CREATE INDEX idx_chat_items_fwd_from_group_id ON chat_items(fwd_from_group_id); CREATE INDEX idx_chat_items_fwd_from_chat_item_id ON chat_items( fwd_from_chat_item_id ); +CREATE INDEX idx_received_probes_group_member_id on received_probes( + group_member_id +); diff --git a/src/Simplex/Chat/Store/Migrations.hs b/src/Simplex/Chat/Store/Migrations.hs index a09baee272..a546b1851a 100644 --- a/src/Simplex/Chat/Store/Migrations.hs +++ b/src/Simplex/Chat/Store/Migrations.hs @@ -112,6 +112,7 @@ import Simplex.Chat.Migrations.M20240515_rcv_files_user_approved_relays import Simplex.Chat.Migrations.M20240528_quota_err_counter import Simplex.Chat.Migrations.M20240827_calls_uuid import Simplex.Chat.Migrations.M20240920_user_order +import Simplex.Chat.Migrations.M20241008_indexes import Simplex.Messaging.Agent.Store.SQLite.Migrations (Migration (..)) schemaMigrations :: [(String, Query, Maybe Query)] @@ -223,7 +224,8 @@ schemaMigrations = ("20240515_rcv_files_user_approved_relays", m20240515_rcv_files_user_approved_relays, Just down_m20240515_rcv_files_user_approved_relays), ("20240528_quota_err_counter", m20240528_quota_err_counter, Just down_m20240528_quota_err_counter), ("20240827_calls_uuid", m20240827_calls_uuid, Just down_m20240827_calls_uuid), - ("20240920_user_order", m20240920_user_order, Just down_m20240920_user_order) + ("20240920_user_order", m20240920_user_order, Just down_m20240920_user_order), + ("20241008_indexes", m20241008_indexes, Just down_m20241008_indexes) ] -- | The list of migrations in ascending order by date From a00e5901de18d48db85702d15d4d7ef6fb513856 Mon Sep 17 00:00:00 2001 From: spaced4ndy <8711996+spaced4ndy@users.noreply.github.com> Date: Tue, 8 Oct 2024 16:42:36 +0400 Subject: [PATCH 4/4] core: 6.1.0.7 --- package.yaml | 2 +- simplex-chat.cabal | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.yaml b/package.yaml index 3d794ea1ec..c7f844e077 100644 --- a/package.yaml +++ b/package.yaml @@ -1,5 +1,5 @@ name: simplex-chat -version: 6.1.0.6 +version: 6.1.0.7 #synopsis: #description: homepage: https://github.com/simplex-chat/simplex-chat#readme diff --git a/simplex-chat.cabal b/simplex-chat.cabal index cf9b71cba1..c637789ad2 100644 --- a/simplex-chat.cabal +++ b/simplex-chat.cabal @@ -5,7 +5,7 @@ cabal-version: 1.12 -- see: https://github.com/sol/hpack name: simplex-chat -version: 6.1.0.6 +version: 6.1.0.7 category: Web, System, Services, Cryptography homepage: https://github.com/simplex-chat/simplex-chat#readme author: simplex.chat