Merge branch 'master' into master-android

This commit is contained in:
Evgeny Poberezkin
2024-10-08 14:08:16 +01:00
19 changed files with 904 additions and 304 deletions
+12 -3
View File
@@ -297,9 +297,18 @@ struct ContentView: View {
if let contactId = contacts?.first?.personHandle?.value,
let chat = chatModel.getChat(contactId),
case let .direct(contact) = chat.chatInfo {
logger.debug("callToRecentContact: schedule call")
DispatchQueue.main.asyncAfter(deadline: .now() + 1) {
CallController.shared.startCall(contact, mediaType)
let activeCall = chatModel.activeCall
// This line works when a user clicks on a video button in CallKit UI while in call.
// The app tries to make another call to the same contact and overwite activeCall instance making its state broken
if let activeCall, contactId == activeCall.contact.id, mediaType == .video, !activeCall.hasVideo {
Task {
await chatModel.callCommand.processCommand(.media(source: .camera, enable: true))
}
} else if activeCall == nil {
logger.debug("callToRecentContact: schedule call")
DispatchQueue.main.asyncAfter(deadline: .now() + 1) {
CallController.shared.startCall(contact, mediaType)
}
}
}
}
-1
View File
@@ -2084,7 +2084,6 @@ func processReceivedMsg(_ res: ChatResponse) async {
await withCall(contact) { call in
await MainActor.run {
call.callState = .offerReceived
call.peerMedia = callType.media
call.sharedKey = sharedKey
}
let useRelay = UserDefaults.standard.bool(forKey: DEFAULT_WEBRTC_POLICY_RELAY)
+161 -109
View File
@@ -17,8 +17,8 @@ struct ActiveCallView: View {
@ObservedObject var call: Call
@Environment(\.scenePhase) var scenePhase
@State private var client: WebRTCClient? = nil
@State private var activeCall: WebRTCClient.Call? = nil
@State private var localRendererAspectRatio: CGFloat? = nil
@State var remoteContentMode: UIView.ContentMode = .scaleAspectFill
@Binding var canConnectCall: Bool
@State var prevColorScheme: ColorScheme = .dark
@State var pipShown = false
@@ -27,24 +27,39 @@ struct ActiveCallView: View {
var body: some View {
ZStack(alignment: .topLeading) {
ZStack(alignment: .bottom) {
if let client = client, [call.peerMedia, call.localMedia].contains(.video), activeCall != nil {
if let client = client, call.hasVideo {
GeometryReader { g in
let width = g.size.width * 0.3
ZStack(alignment: .topTrailing) {
CallViewRemote(client: client, activeCall: $activeCall, activeCallViewIsCollapsed: $m.activeCallViewIsCollapsed, pipShown: $pipShown)
CallViewLocal(client: client, activeCall: $activeCall, localRendererAspectRatio: $localRendererAspectRatio, pipShown: $pipShown)
.cornerRadius(10)
.frame(width: width, height: width / (localRendererAspectRatio ?? 1))
.padding([.top, .trailing], 17)
ZStack(alignment: .center) {
// For some reason, when the view in GeometryReader and ZStack is visible, it steals clicks on a back button, so showing something on top like this with background color helps (.clear color doesn't work)
}
.frame(maxWidth: .infinity, maxHeight: .infinity)
.background(Color.primary.opacity(0.000001))
CallViewRemote(client: client, call: call, activeCallViewIsCollapsed: $m.activeCallViewIsCollapsed, contentMode: $remoteContentMode, pipShown: $pipShown)
.onTapGesture {
remoteContentMode = remoteContentMode == .scaleAspectFill ? .scaleAspectFit : .scaleAspectFill
}
Group {
let localVideoTrack = client.activeCall?.localVideoTrack ?? client.notConnectedCall?.localCameraAndTrack?.1
if localVideoTrack != nil {
CallViewLocal(client: client, localRendererAspectRatio: $localRendererAspectRatio, pipShown: $pipShown)
.onDisappear {
localRendererAspectRatio = nil
}
} else {
Rectangle().fill(.black)
}
}
.cornerRadius(10)
.frame(width: width, height: localRendererAspectRatio == nil ? (g.size.width < g.size.height ? width * 1.33 : width / 1.33) : width / (localRendererAspectRatio ?? 1))
.padding([.top, .trailing], 17)
}
}
}
if let call = m.activeCall, let client = client, (!pipShown || !call.supportsVideo) {
if let call = m.activeCall, let client = client, (!pipShown || !call.hasVideo) {
ActiveCallOverlay(call: call, client: client)
}
}
@@ -54,6 +69,9 @@ struct ActiveCallView: View {
.onAppear {
logger.debug("ActiveCallView: appear client is nil \(client == nil), scenePhase \(String(describing: scenePhase)), canConnectCall \(canConnectCall)")
AppDelegate.keepScreenOn(true)
Task {
await askRequiredPermissions()
}
createWebRTCClient()
dismissAllSheets()
hideKeyboard()
@@ -84,7 +102,7 @@ struct ActiveCallView: View {
private func createWebRTCClient() {
if client == nil && canConnectCall {
client = WebRTCClient($activeCall, { msg in await MainActor.run { processRtcMessage(msg: msg) } }, $localRendererAspectRatio)
client = WebRTCClient({ msg in await MainActor.run { processRtcMessage(msg: msg) } }, $localRendererAspectRatio)
Task {
await m.callCommand.setClient(client)
}
@@ -99,7 +117,7 @@ struct ActiveCallView: View {
logger.debug("ActiveCallView: response \(msg.resp.respType)")
switch msg.resp {
case let .capabilities(capabilities):
let callType = CallType(media: call.localMedia, capabilities: capabilities)
let callType = CallType(media: call.initialCallType, capabilities: capabilities)
Task {
do {
try await apiSendCallInvitation(call.contact, callType)
@@ -110,7 +128,7 @@ struct ActiveCallView: View {
call.callState = .invitationSent
call.localCapabilities = capabilities
}
if call.supportsVideo && !AVAudioSession.sharedInstance().hasExternalAudioDevice() {
if call.hasVideo && !AVAudioSession.sharedInstance().hasExternalAudioDevice() {
try? AVAudioSession.sharedInstance().setCategory(.playback, options: [.allowBluetooth, .allowAirPlay, .allowBluetoothA2DP])
}
CallSoundsPlayer.shared.startConnectingCallSound()
@@ -120,7 +138,7 @@ struct ActiveCallView: View {
Task {
do {
try await apiSendCallOffer(call.contact, offer, iceCandidates,
media: call.localMedia, capabilities: capabilities)
media: call.initialCallType, capabilities: capabilities)
} catch {
logger.error("apiSendCallOffer \(responseError(error))")
}
@@ -164,6 +182,9 @@ struct ActiveCallView: View {
}
if state.connectionState == "closed" {
closeCallView(client)
if let callUUID = m.activeCall?.callUUID {
CallController.shared.endCall(callUUID: callUUID)
}
m.activeCall = nil
m.activeCallViewIsCollapsed = false
}
@@ -182,6 +203,14 @@ struct ActiveCallView: View {
CallSoundsPlayer.shared.vibrate(long: false)
wasConnected = true
}
case let .peerMedia(source, enabled):
switch source {
case .mic: call.peerMediaSources.mic = enabled
case .camera: call.peerMediaSources.camera = enabled
case .screenAudio: call.peerMediaSources.screenAudio = enabled
case .screenVideo: call.peerMediaSources.screenVideo = enabled
case .unknown: ()
}
case .ended:
closeCallView(client)
call.callState = .ended
@@ -226,6 +255,26 @@ struct ActiveCallView: View {
}
}
private func askRequiredPermissions() async {
let mic = await WebRTCClient.isAuthorized(for: .audio)
await MainActor.run {
call.localMediaSources.mic = mic
}
let cameraAuthorized = AVCaptureDevice.authorizationStatus(for: .video) == .authorized
var camera = call.initialCallType == .audio || cameraAuthorized
if call.initialCallType == .video && !cameraAuthorized {
camera = await WebRTCClient.isAuthorized(for: .video)
await MainActor.run {
if camera, let client {
client.setCameraEnabled(true)
}
}
}
if !mic || !camera {
WebRTCClient.showUnauthorizedAlert(for: !mic ? .audio : .video)
}
}
private func closeCallView(_ client: WebRTCClient) {
if m.activeCall != nil {
m.showCallView = false
@@ -241,44 +290,16 @@ struct ActiveCallOverlay: View {
var body: some View {
VStack {
switch call.localMedia {
case .video:
switch call.hasVideo {
case true:
videoCallInfoView(call)
.foregroundColor(.white)
.opacity(0.8)
.padding()
Spacer()
HStack {
toggleAudioButton()
Spacer()
if deviceManager.availableInputs.allSatisfy({ $0.portType == .builtInMic }) {
toggleSpeakerButton()
.frame(width: 40, height: 40)
} else if call.hasMedia {
AudioDevicePicker()
.scaleEffect(2)
.frame(maxWidth: 40, maxHeight: 40)
} else {
Color.clear.frame(width: 40, height: 40)
}
Spacer()
endCallButton()
Spacer()
if call.videoEnabled {
flipCameraButton()
} else {
Color.clear.frame(width: 40, height: 40)
}
Spacer()
toggleVideoButton()
}
.padding(.horizontal, 20)
.padding(.bottom, 16)
.frame(maxWidth: .infinity, alignment: .center)
case .audio:
.padding(.horizontal)
// Fixed vertical padding required for preserving position of buttons row when changing audio-to-video and back in landscape orientation.
// Otherwise, bigger padding is added by SwiftUI when switching call types
.padding(.vertical, 10)
case false:
ZStack(alignment: .topLeading) {
Button {
chatModel.activeCallViewIsCollapsed = true
@@ -293,35 +314,32 @@ struct ActiveCallOverlay: View {
}
.foregroundColor(.white)
.opacity(0.8)
.padding()
.padding(.horizontal)
.padding(.vertical, 10)
.frame(maxHeight: .infinity)
}
Spacer()
ZStack(alignment: .bottom) {
toggleAudioButton()
.frame(maxWidth: .infinity, alignment: .leading)
endCallButton()
// Check if the only input is microphone. And in this case show toggle button,
// If there are more inputs, it probably means something like bluetooth headphones are available
// and in this case show iOS button for choosing different output.
// There is no way to get available outputs, only inputs
if deviceManager.availableInputs.allSatisfy({ $0.portType == .builtInMic }) {
toggleSpeakerButton()
.frame(maxWidth: .infinity, alignment: .trailing)
} else if call.hasMedia {
AudioDevicePicker()
.scaleEffect(2)
.frame(maxWidth: 50, maxHeight: 40)
.frame(maxWidth: .infinity, alignment: .trailing)
} else {
Color.clear.frame(width: 50, height: 40)
}
}
.padding(.bottom, 60)
.padding(.horizontal, 48)
}
Spacer()
HStack {
toggleMicButton()
Spacer()
audioDeviceButton()
Spacer()
endCallButton()
Spacer()
if call.localMediaSources.camera {
flipCameraButton()
} else {
Color.clear.frame(width: 60, height: 60)
}
Spacer()
toggleCameraButton()
}
.padding(.horizontal, 20)
.padding(.bottom, 16)
.frame(maxWidth: 440, alignment: .center)
}
.frame(maxWidth: .infinity)
.onAppear {
@@ -383,36 +401,56 @@ struct ActiveCallOverlay: View {
private func endCallButton() -> some View {
let cc = CallController.shared
return callButton("phone.down.fill", width: 60, height: 60) {
return callButton("phone.down.fill", padding: 10) {
if let uuid = call.callUUID {
cc.endCall(callUUID: uuid)
} else {
cc.endCall(call: call) {}
}
}
.foregroundColor(.red)
.background(.red)
.clipShape(.circle)
}
private func toggleAudioButton() -> some View {
controlButton(call, call.audioEnabled ? "mic.fill" : "mic.slash") {
private func toggleMicButton() -> some View {
controlButton(call, call.localMediaSources.mic ? "mic.fill" : "mic.slash", padding: 14) {
Task {
client.setAudioEnabled(!call.audioEnabled)
DispatchQueue.main.async {
call.audioEnabled = !call.audioEnabled
}
if await WebRTCClient.isAuthorized(for: .audio) {
client.setAudioEnabled(!call.localMediaSources.mic)
} else { WebRTCClient.showUnauthorizedAlert(for: .audio) }
}
}
}
func audioDeviceButton() -> some View {
// Check if the only input is microphone. And in this case show toggle button,
// If there are more inputs, it probably means something like bluetooth headphones are available
// and in this case show iOS button for choosing different output.
// There is no way to get available outputs, only inputs
Group {
if deviceManager.availableInputs.allSatisfy({ $0.portType == .builtInMic }) {
toggleSpeakerButton()
} else {
audioDevicePickerButton()
}
}
.onChange(of: call.hasVideo) { hasVideo in
let current = AVAudioSession.sharedInstance().currentRoute.outputs.first?.portType
let speakerEnabled = current == .builtInSpeaker
let receiverEnabled = current == .builtInReceiver
// react automatically only when speaker or receiver were selected, otherwise keep an external device selected
if hasVideo != speakerEnabled && (speakerEnabled || receiverEnabled) {
client.setSpeakerEnabledAndConfigureSession(!speakerEnabled, skipExternalDevice: true)
call.speakerEnabled = !speakerEnabled
}
}
}
private func toggleSpeakerButton() -> some View {
controlButton(call, call.speakerEnabled ? "speaker.wave.2.fill" : "speaker.wave.1.fill") {
Task {
let speakerEnabled = AVAudioSession.sharedInstance().currentRoute.outputs.first?.portType == .builtInSpeaker
client.setSpeakerEnabledAndConfigureSession(!speakerEnabled)
DispatchQueue.main.async {
call.speakerEnabled = !speakerEnabled
}
}
controlButton(call, !call.peerMediaSources.mic ? "speaker.slash" : call.speakerEnabled ? "speaker.wave.2.fill" : "speaker.wave.1.fill", padding: !call.peerMediaSources.mic ? 16 : call.speakerEnabled ? 15 : 17) {
let speakerEnabled = AVAudioSession.sharedInstance().currentRoute.outputs.first?.portType == .builtInSpeaker
client.setSpeakerEnabledAndConfigureSession(!speakerEnabled)
call.speakerEnabled = !speakerEnabled
}
.onAppear {
deviceManager.call = call
@@ -420,53 +458,67 @@ struct ActiveCallOverlay: View {
}
}
private func toggleVideoButton() -> some View {
controlButton(call, call.videoEnabled ? "video.fill" : "video.slash") {
private func toggleCameraButton() -> some View {
controlButton(call, call.localMediaSources.camera ? "video.fill" : "video.slash", padding: call.localMediaSources.camera ? 16 : 14) {
Task {
client.setVideoEnabled(!call.videoEnabled)
DispatchQueue.main.async {
call.videoEnabled = !call.videoEnabled
}
if await WebRTCClient.isAuthorized(for: .video) {
client.setCameraEnabled(!call.localMediaSources.camera)
} else { WebRTCClient.showUnauthorizedAlert(for: .video) }
}
}
.disabled(call.initialCallType == .audio && client.activeCall?.peerHasOldVersion == true)
}
@ViewBuilder private func flipCameraButton() -> some View {
controlButton(call, "arrow.triangle.2.circlepath") {
controlButton(call, "arrow.triangle.2.circlepath", padding: 12) {
Task {
client.flipCamera()
if await WebRTCClient.isAuthorized(for: .video) {
client.flipCamera()
}
}
}
}
@ViewBuilder private func controlButton(_ call: Call, _ imageName: String, _ perform: @escaping () -> Void) -> some View {
if call.hasMedia {
callButton(imageName, width: 50, height: 38, perform)
.foregroundColor(.white)
.opacity(0.85)
} else {
Color.clear.frame(width: 50, height: 38)
}
@ViewBuilder private func controlButton(_ call: Call, _ imageName: String, padding: CGFloat, _ perform: @escaping () -> Void) -> some View {
callButton(imageName, padding: padding, perform)
.background(call.peerMediaSources.hasVideo ? Color.black.opacity(0.2) : Color.white.opacity(0.2))
.clipShape(.circle)
}
private func callButton(_ imageName: String, width: CGFloat, height: CGFloat, _ perform: @escaping () -> Void) -> some View {
@ViewBuilder private func audioDevicePickerButton() -> some View {
AudioDevicePicker()
.opacity(0.8)
.scaleEffect(2)
.padding(10)
.frame(width: 60, height: 60)
.background(call.peerMediaSources.hasVideo ? Color.black.opacity(0.2) : Color.white.opacity(0.2))
.clipShape(.circle)
}
private func callButton(_ imageName: String, padding: CGFloat, _ perform: @escaping () -> Void) -> some View {
Button {
perform()
} label: {
Image(systemName: imageName)
.resizable()
.scaledToFit()
.frame(maxWidth: width, maxHeight: height)
.padding(padding)
.frame(width: 60, height: 60)
}
.foregroundColor(whiteColorWithAlpha)
}
private var whiteColorWithAlpha: Color {
get { Color(red: 204 / 255, green: 204 / 255, blue: 204 / 255) }
}
}
struct ActiveCallOverlay_Previews: PreviewProvider {
static var previews: some View {
Group{
ActiveCallOverlay(call: Call(direction: .incoming, contact: Contact.sampleData, callUUID: UUID().uuidString.lowercased(), callState: .offerSent, localMedia: .video), client: WebRTCClient(Binding.constant(nil), { _ in }, Binding.constant(nil)))
ActiveCallOverlay(call: Call(direction: .incoming, contact: Contact.sampleData, callUUID: UUID().uuidString.lowercased(), callState: .offerSent, initialCallType: .video), client: WebRTCClient({ _ in }, Binding.constant(nil)))
.background(.black)
ActiveCallOverlay(call: Call(direction: .incoming, contact: Contact.sampleData, callUUID: UUID().uuidString.lowercased(), callState: .offerSent, localMedia: .audio), client: WebRTCClient(Binding.constant(nil), { _ in }, Binding.constant(nil)))
ActiveCallOverlay(call: Call(direction: .incoming, contact: Contact.sampleData, callUUID: UUID().uuidString.lowercased(), callState: .offerSent, initialCallType: .audio), client: WebRTCClient({ _ in }, Binding.constant(nil)))
.background(.black)
}
}
@@ -104,7 +104,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
}
func provider(_ provider: CXProvider, perform action: CXSetMutedCallAction) {
if callManager.enableMedia(media: .audio, enable: !action.isMuted, callUUID: action.callUUID.uuidString.lowercased()) {
if callManager.enableMedia(source: .mic, enable: !action.isMuted, callUUID: action.callUUID.uuidString.lowercased()) {
action.fulfill()
} else {
action.fail()
@@ -121,8 +121,8 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
RTCAudioSession.sharedInstance().audioSessionDidActivate(audioSession)
RTCAudioSession.sharedInstance().isAudioEnabled = true
do {
let supportsVideo = ChatModel.shared.activeCall?.supportsVideo == true
if supportsVideo {
let hasVideo = ChatModel.shared.activeCall?.hasVideo == true
if hasVideo {
try audioSession.setCategory(.playAndRecord, mode: .videoChat, options: [.defaultToSpeaker, .mixWithOthers, .allowBluetooth, .allowAirPlay, .allowBluetoothA2DP])
} else {
try audioSession.setCategory(.playAndRecord, mode: .voiceChat, options: [.mixWithOthers, .allowBluetooth, .allowAirPlay, .allowBluetoothA2DP])
@@ -133,7 +133,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse
try? await Task.sleep(nanoseconds: UInt64(i) * 300_000000)
if let preferred = audioSession.preferredInputDevice() {
await MainActor.run { try? audioSession.setPreferredInput(preferred) }
} else if supportsVideo {
} else if hasVideo {
await MainActor.run { try? audioSession.overrideOutputAudioPort(.speaker) }
}
}
+5 -5
View File
@@ -12,7 +12,7 @@ import SimpleXChat
class CallManager {
func newOutgoingCall(_ contact: Contact, _ media: CallMediaType) -> String {
let uuid = UUID().uuidString.lowercased()
let call = Call(direction: .outgoing, contact: contact, callUUID: uuid, callState: .waitCapabilities, localMedia: media)
let call = Call(direction: .outgoing, contact: contact, callUUID: uuid, callState: .waitCapabilities, initialCallType: media)
call.speakerEnabled = media == .video
ChatModel.shared.activeCall = call
return uuid
@@ -22,7 +22,7 @@ class CallManager {
let m = ChatModel.shared
if let call = m.activeCall, call.callUUID == callUUID {
m.showCallView = true
Task { await m.callCommand.processCommand(.capabilities(media: call.localMedia)) }
Task { await m.callCommand.processCommand(.capabilities(media: call.initialCallType)) }
return true
}
return false
@@ -44,7 +44,7 @@ class CallManager {
contact: invitation.contact,
callUUID: invitation.callUUID,
callState: .invitationAccepted,
localMedia: invitation.callType.media,
initialCallType: invitation.callType.media,
sharedKey: invitation.sharedKey
)
call.speakerEnabled = invitation.callType.media == .video
@@ -68,10 +68,10 @@ class CallManager {
}
}
func enableMedia(media: CallMediaType, enable: Bool, callUUID: String) -> Bool {
func enableMedia(source: CallMediaSource, enable: Bool, callUUID: String) -> Bool {
if let call = ChatModel.shared.activeCall, call.callUUID == callUUID {
let m = ChatModel.shared
Task { await m.callCommand.processCommand(.media(media: media, enable: enable)) }
Task { await m.callCommand.processCommand(.media(source: source, enable: enable)) }
return true
}
return false
@@ -10,40 +10,43 @@ import AVKit
struct CallViewRemote: UIViewRepresentable {
var client: WebRTCClient
var activeCall: Binding<WebRTCClient.Call?>
@ObservedObject var call: Call
@State var enablePip: (Bool) -> Void = {_ in }
@Binding var activeCallViewIsCollapsed: Bool
@Binding var contentMode: UIView.ContentMode
@Binding var pipShown: Bool
init(client: WebRTCClient, activeCall: Binding<WebRTCClient.Call?>, activeCallViewIsCollapsed: Binding<Bool>, pipShown: Binding<Bool>) {
self.client = client
self.activeCall = activeCall
self._activeCallViewIsCollapsed = activeCallViewIsCollapsed
self._pipShown = pipShown
}
func makeUIView(context: Context) -> UIView {
let view = UIView()
if let call = activeCall.wrappedValue {
let remoteRenderer = RTCMTLVideoView(frame: view.frame)
remoteRenderer.videoContentMode = .scaleAspectFill
client.addRemoteRenderer(call, remoteRenderer)
addSubviewAndResize(remoteRenderer, into: view)
let remoteCameraRenderer = RTCMTLVideoView(frame: view.frame)
remoteCameraRenderer.videoContentMode = contentMode
remoteCameraRenderer.tag = 0
let remoteScreenRenderer = RTCMTLVideoView(frame: view.frame)
remoteScreenRenderer.videoContentMode = contentMode
remoteScreenRenderer.tag = 1
remoteScreenRenderer.alpha = call.peerMediaSources.screenVideo ? 1 : 0
if AVPictureInPictureController.isPictureInPictureSupported() {
makeViewWithRTCRenderer(call, remoteRenderer, view, context)
}
context.coordinator.cameraRenderer = remoteCameraRenderer
context.coordinator.screenRenderer = remoteScreenRenderer
client.addRemoteCameraRenderer(remoteCameraRenderer)
client.addRemoteScreenRenderer(remoteScreenRenderer)
addSubviewAndResize(remoteCameraRenderer, remoteScreenRenderer, into: view)
if AVPictureInPictureController.isPictureInPictureSupported() {
makeViewWithRTCRenderer(remoteCameraRenderer, remoteScreenRenderer, view, context)
}
return view
}
func makeViewWithRTCRenderer(_ call: WebRTCClient.Call, _ remoteRenderer: RTCMTLVideoView, _ view: UIView, _ context: Context) {
let pipRemoteRenderer = RTCMTLVideoView(frame: view.frame)
pipRemoteRenderer.videoContentMode = .scaleAspectFill
func makeViewWithRTCRenderer(_ remoteCameraRenderer: RTCMTLVideoView, _ remoteScreenRenderer: RTCMTLVideoView, _ view: UIView, _ context: Context) {
let pipRemoteCameraRenderer = RTCMTLVideoView(frame: view.frame)
pipRemoteCameraRenderer.videoContentMode = .scaleAspectFill
let pipRemoteScreenRenderer = RTCMTLVideoView(frame: view.frame)
pipRemoteScreenRenderer.videoContentMode = .scaleAspectFill
let pipVideoCallViewController = AVPictureInPictureVideoCallViewController()
pipVideoCallViewController.preferredContentSize = CGSize(width: 1080, height: 1920)
addSubviewAndResize(pipRemoteRenderer, into: pipVideoCallViewController.view)
let pipContentSource = AVPictureInPictureController.ContentSource(
activeVideoCallSourceView: view,
contentViewController: pipVideoCallViewController
@@ -55,7 +58,9 @@ struct CallViewRemote: UIViewRepresentable {
context.coordinator.pipController = pipController
context.coordinator.willShowHide = { show in
if show {
client.addRemoteRenderer(call, pipRemoteRenderer)
client.addRemoteCameraRenderer(pipRemoteCameraRenderer)
client.addRemoteScreenRenderer(pipRemoteScreenRenderer)
context.coordinator.relayout()
DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) {
activeCallViewIsCollapsed = true
}
@@ -67,13 +72,29 @@ struct CallViewRemote: UIViewRepresentable {
}
context.coordinator.didShowHide = { show in
if show {
remoteRenderer.isHidden = true
remoteCameraRenderer.isHidden = true
remoteScreenRenderer.isHidden = true
} else {
client.removeRemoteRenderer(call, pipRemoteRenderer)
remoteRenderer.isHidden = false
client.removeRemoteCameraRenderer(pipRemoteCameraRenderer)
client.removeRemoteScreenRenderer(pipRemoteScreenRenderer)
remoteCameraRenderer.isHidden = false
remoteScreenRenderer.isHidden = false
}
pipShown = show
}
context.coordinator.relayout = {
let camera = call.peerMediaSources.camera
let screenVideo = call.peerMediaSources.screenVideo
pipRemoteCameraRenderer.alpha = camera ? 1 : 0
pipRemoteScreenRenderer.alpha = screenVideo ? 1 : 0
if screenVideo {
addSubviewAndResize(pipRemoteScreenRenderer, pipRemoteCameraRenderer, pip: true, into: pipVideoCallViewController.view)
} else {
addSubviewAndResize(pipRemoteCameraRenderer, pipRemoteScreenRenderer, pip: true, into: pipVideoCallViewController.view)
}
(pipVideoCallViewController.view.subviews[0] as! RTCMTLVideoView).videoContentMode = contentMode
(pipVideoCallViewController.view.subviews[1] as! RTCMTLVideoView).videoContentMode = .scaleAspectFill
}
DispatchQueue.main.async {
enablePip = { enable in
if enable != pipShown /* pipController.isPictureInPictureActive */ {
@@ -88,24 +109,50 @@ struct CallViewRemote: UIViewRepresentable {
}
func makeCoordinator() -> Coordinator {
Coordinator()
Coordinator(client)
}
func updateUIView(_ view: UIView, context: Context) {
logger.debug("CallView.updateUIView remote")
let camera = view.subviews.first(where: { $0.tag == 0 })!
let screen = view.subviews.first(where: { $0.tag == 1 })!
let screenVideo = call.peerMediaSources.screenVideo
if screenVideo && screen.alpha == 0 {
screen.alpha = 1
addSubviewAndResize(screen, camera, into: view)
} else if !screenVideo && screen.alpha == 1 {
screen.alpha = 0
addSubviewAndResize(camera, screen, into: view)
}
(view.subviews[0] as! RTCMTLVideoView).videoContentMode = contentMode
(view.subviews[1] as! RTCMTLVideoView).videoContentMode = .scaleAspectFill
camera.alpha = call.peerMediaSources.camera ? 1 : 0
screen.alpha = call.peerMediaSources.screenVideo ? 1 : 0
DispatchQueue.main.async {
if activeCallViewIsCollapsed != pipShown {
enablePip(activeCallViewIsCollapsed)
} else if pipShown {
context.coordinator.relayout()
}
}
}
// MARK: - Coordinator
class Coordinator: NSObject, AVPictureInPictureControllerDelegate {
var cameraRenderer: RTCMTLVideoView?
var screenRenderer: RTCMTLVideoView?
var client: WebRTCClient
var pipController: AVPictureInPictureController? = nil
var willShowHide: (Bool) -> Void = { _ in }
var didShowHide: (Bool) -> Void = { _ in }
var relayout: () -> Void = {}
required init(_ client: WebRTCClient) {
self.client = client
}
func pictureInPictureControllerWillStartPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) {
willShowHide(true)
}
@@ -127,11 +174,20 @@ struct CallViewRemote: UIViewRepresentable {
}
deinit {
// TODO: deinit is not called when changing call type from audio to video and back,
// which causes many renderers can be created and added to stream (if enabling/disabling
// video while not yet connected in outgoing call)
pipController?.stopPictureInPicture()
pipController?.canStartPictureInPictureAutomaticallyFromInline = false
pipController?.contentSource = nil
pipController?.delegate = nil
pipController = nil
if let cameraRenderer {
client.removeRemoteCameraRenderer(cameraRenderer)
}
if let screenRenderer {
client.removeRemoteScreenRenderer(screenRenderer)
}
}
}
@@ -148,51 +204,109 @@ struct CallViewRemote: UIViewRepresentable {
struct CallViewLocal: UIViewRepresentable {
var client: WebRTCClient
var activeCall: Binding<WebRTCClient.Call?>
var localRendererAspectRatio: Binding<CGFloat?>
@State var pipStateChanged: (Bool) -> Void = {_ in }
@Binding var pipShown: Bool
init(client: WebRTCClient, activeCall: Binding<WebRTCClient.Call?>, localRendererAspectRatio: Binding<CGFloat?>, pipShown: Binding<Bool>) {
init(client: WebRTCClient, localRendererAspectRatio: Binding<CGFloat?>, pipShown: Binding<Bool>) {
self.client = client
self.activeCall = activeCall
self.localRendererAspectRatio = localRendererAspectRatio
self._pipShown = pipShown
}
func makeUIView(context: Context) -> UIView {
let view = UIView()
if let call = activeCall.wrappedValue {
let localRenderer = RTCEAGLVideoView(frame: .zero)
client.addLocalRenderer(call, localRenderer)
client.startCaptureLocalVideo(call)
addSubviewAndResize(localRenderer, into: view)
DispatchQueue.main.async {
pipStateChanged = { shown in
localRenderer.isHidden = shown
}
let localRenderer = RTCEAGLVideoView(frame: .zero)
context.coordinator.renderer = localRenderer
client.addLocalRenderer(localRenderer)
addSubviewAndResize(localRenderer, nil, into: view)
DispatchQueue.main.async {
pipStateChanged = { shown in
localRenderer.isHidden = shown
}
}
return view
}
func makeCoordinator() -> Coordinator {
Coordinator(client)
}
func updateUIView(_ view: UIView, context: Context) {
logger.debug("CallView.updateUIView local")
pipStateChanged(pipShown)
}
// MARK: - Coordinator
class Coordinator: NSObject, AVPictureInPictureControllerDelegate {
var renderer: RTCEAGLVideoView?
var client: WebRTCClient
required init(_ client: WebRTCClient) {
self.client = client
}
deinit {
if let renderer {
client.removeLocalRenderer(renderer)
}
}
}
}
private func addSubviewAndResize(_ view: UIView, into containerView: UIView) {
containerView.addSubview(view)
view.translatesAutoresizingMaskIntoConstraints = false
containerView.addConstraints(NSLayoutConstraint.constraints(withVisualFormat: "H:|[view]|",
options: [],
metrics: nil,
views: ["view": view]))
private func addSubviewAndResize(_ fullscreen: UIView, _ end: UIView?, pip: Bool = false, into containerView: UIView) {
if containerView.subviews.firstIndex(of: fullscreen) == 0 && ((end == nil && containerView.subviews.count == 1) || (end != nil && containerView.subviews.firstIndex(of: end!) == 1)) {
// Nothing to do, elements on their places
return
}
containerView.removeConstraints(containerView.constraints)
containerView.subviews.forEach { sub in sub.removeFromSuperview()}
containerView.addConstraints(NSLayoutConstraint.constraints(withVisualFormat: "V:|[view]|",
containerView.addSubview(fullscreen)
fullscreen.translatesAutoresizingMaskIntoConstraints = false
fullscreen.layer.cornerRadius = 0
fullscreen.layer.masksToBounds = false
if let end {
containerView.addSubview(end)
end.translatesAutoresizingMaskIntoConstraints = false
end.layer.cornerRadius = pip ? 8 : 10
end.layer.masksToBounds = true
}
let constraintFullscreenV = NSLayoutConstraint.constraints(
withVisualFormat: "V:|[fullscreen]|",
options: [],
metrics: nil,
views: ["view": view]))
views: ["fullscreen": fullscreen]
)
let constraintFullscreenH = NSLayoutConstraint.constraints(
withVisualFormat: "H:|[fullscreen]|",
options: [],
metrics: nil,
views: ["fullscreen": fullscreen]
)
containerView.addConstraints(constraintFullscreenV)
containerView.addConstraints(constraintFullscreenH)
if let end {
let constraintEndWidth = NSLayoutConstraint(
item: end, attribute: .width, relatedBy: .equal, toItem: containerView, attribute: .width, multiplier: pip ? 0.5 : 0.3, constant: 0
)
let constraintEndHeight = NSLayoutConstraint(
item: end, attribute: .height, relatedBy: .equal, toItem: containerView, attribute: .width, multiplier: pip ? 0.5 * 1.33 : 0.3 * 1.33, constant: 0
)
let constraintEndX = NSLayoutConstraint(
item: end, attribute: .leading, relatedBy: .equal, toItem: containerView, attribute: .trailing, multiplier: pip ? 0.5 : 0.7, constant: pip ? -8 : -17
)
let constraintEndY = NSLayoutConstraint(
item: end, attribute: .bottom, relatedBy: .equal, toItem: containerView, attribute: .bottom, multiplier: 1, constant: pip ? -8 : -92
)
containerView.addConstraint(constraintEndWidth)
containerView.addConstraint(constraintEndHeight)
containerView.addConstraint(constraintEndX)
containerView.addConstraint(constraintEndY)
}
containerView.layoutIfNeeded()
}
+39 -17
View File
@@ -19,14 +19,13 @@ class Call: ObservableObject, Equatable {
var direction: CallDirection
var contact: Contact
var callUUID: String?
var localMedia: CallMediaType
var initialCallType: CallMediaType
@Published var localMediaSources: CallMediaSources
@Published var callState: CallState
@Published var localCapabilities: CallCapabilities?
@Published var peerMedia: CallMediaType?
@Published var peerMediaSources: CallMediaSources = CallMediaSources()
@Published var sharedKey: String?
@Published var audioEnabled = true
@Published var speakerEnabled = false
@Published var videoEnabled: Bool
@Published var connectionInfo: ConnectionInfo?
@Published var connectedAt: Date? = nil
@@ -35,32 +34,33 @@ class Call: ObservableObject, Equatable {
contact: Contact,
callUUID: String?,
callState: CallState,
localMedia: CallMediaType,
initialCallType: CallMediaType,
sharedKey: String? = nil
) {
self.direction = direction
self.contact = contact
self.callUUID = callUUID
self.callState = callState
self.localMedia = localMedia
self.initialCallType = initialCallType
self.sharedKey = sharedKey
self.videoEnabled = localMedia == .video
self.localMediaSources = CallMediaSources(
mic: AVCaptureDevice.authorizationStatus(for: .audio) == .authorized,
camera: initialCallType == .video && AVCaptureDevice.authorizationStatus(for: .video) == .authorized)
}
var encrypted: Bool { get { localEncrypted && sharedKey != nil } }
var localEncrypted: Bool { get { localCapabilities?.encryption ?? false } }
private var localEncrypted: Bool { get { localCapabilities?.encryption ?? false } }
var encryptionStatus: LocalizedStringKey {
get {
switch callState {
case .waitCapabilities: return ""
case .invitationSent: return localEncrypted ? "e2e encrypted" : "no e2e encryption"
case .invitationAccepted: return sharedKey == nil ? "contact has no e2e encryption" : "contact has e2e encryption"
default: return !localEncrypted ? "no e2e encryption" : sharedKey == nil ? "contact has no e2e encryption" : "e2e encrypted"
default: return !localEncrypted ? "no e2e encryption" : sharedKey == nil ? "contact has no e2e encryption" : "e2e encrypted"
}
}
}
var hasMedia: Bool { get { callState == .offerSent || callState == .negotiated || callState == .connected } }
var supportsVideo: Bool { get { peerMedia == .video || localMedia == .video } }
var hasVideo: Bool { get { localMediaSources.hasVideo || peerMediaSources.hasVideo } }
}
enum CallDirection {
@@ -105,18 +105,28 @@ struct WVAPIMessage: Equatable, Decodable, Encodable {
var command: WCallCommand?
}
struct CallMediaSources: Equatable, Codable {
var mic: Bool = false
var camera: Bool = false
var screenAudio: Bool = false
var screenVideo: Bool = false
var hasVideo: Bool { get { camera || screenVideo } }
}
enum WCallCommand: Equatable, Encodable, Decodable {
case capabilities(media: CallMediaType)
case start(media: CallMediaType, aesKey: String? = nil, iceServers: [RTCIceServer]? = nil, relay: Bool? = nil)
case offer(offer: String, iceCandidates: String, media: CallMediaType, aesKey: String? = nil, iceServers: [RTCIceServer]? = nil, relay: Bool? = nil)
case answer(answer: String, iceCandidates: String)
case ice(iceCandidates: String)
case media(media: CallMediaType, enable: Bool)
case media(source: CallMediaSource, enable: Bool)
case end
enum CodingKeys: String, CodingKey {
case type
case media
case source
case aesKey
case offer
case answer
@@ -167,9 +177,9 @@ enum WCallCommand: Equatable, Encodable, Decodable {
case let .ice(iceCandidates):
try container.encode("ice", forKey: .type)
try container.encode(iceCandidates, forKey: .iceCandidates)
case let .media(media, enable):
case let .media(source, enable):
try container.encode("media", forKey: .type)
try container.encode(media, forKey: .media)
try container.encode(source, forKey: .media)
try container.encode(enable, forKey: .enable)
case .end:
try container.encode("end", forKey: .type)
@@ -205,9 +215,9 @@ enum WCallCommand: Equatable, Encodable, Decodable {
let iceCandidates = try container.decode(String.self, forKey: CodingKeys.iceCandidates)
self = .ice(iceCandidates: iceCandidates)
case "media":
let media = try container.decode(CallMediaType.self, forKey: CodingKeys.media)
let source = try container.decode(CallMediaSource.self, forKey: CodingKeys.source)
let enable = try container.decode(Bool.self, forKey: CodingKeys.enable)
self = .media(media: media, enable: enable)
self = .media(source: source, enable: enable)
case "end":
self = .end
default:
@@ -224,6 +234,7 @@ enum WCallResponse: Equatable, Decodable {
case ice(iceCandidates: String)
case connection(state: ConnectionState)
case connected(connectionInfo: ConnectionInfo)
case peerMedia(source: CallMediaSource, enabled: Bool)
case ended
case ok
case error(message: String)
@@ -238,6 +249,8 @@ enum WCallResponse: Equatable, Decodable {
case state
case connectionInfo
case message
case source
case enabled
}
var respType: String {
@@ -249,6 +262,7 @@ enum WCallResponse: Equatable, Decodable {
case .ice: return "ice"
case .connection: return "connection"
case .connected: return "connected"
case .peerMedia: return "peerMedia"
case .ended: return "ended"
case .ok: return "ok"
case .error: return "error"
@@ -283,6 +297,10 @@ enum WCallResponse: Equatable, Decodable {
case "connected":
let connectionInfo = try container.decode(ConnectionInfo.self, forKey: CodingKeys.connectionInfo)
self = .connected(connectionInfo: connectionInfo)
case "peerMedia":
let source = try container.decode(CallMediaSource.self, forKey: CodingKeys.source)
let enabled = try container.decode(Bool.self, forKey: CodingKeys.enabled)
self = .peerMedia(source: source, enabled: enabled)
case "ended":
self = .ended
case "ok":
@@ -324,6 +342,10 @@ extension WCallResponse: Encodable {
case let .connected(connectionInfo):
try container.encode("connected", forKey: .type)
try container.encode(connectionInfo, forKey: .connectionInfo)
case let .peerMedia(source, enabled):
try container.encode("peerMedia", forKey: .type)
try container.encode(source, forKey: .source)
try container.encode(enabled, forKey: .enabled)
case .ended:
try container.encode("ended", forKey: .type)
case .ok:
@@ -376,7 +398,7 @@ actor WebRTCCommandProcessor {
func shouldRunCommand(_ client: WebRTCClient, _ c: WCallCommand) -> Bool {
switch c {
case .capabilities, .start, .offer, .end: true
default: client.activeCall.wrappedValue != nil
default: client.activeCall != nil
}
}
}
+415 -103
View File
@@ -23,15 +23,24 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
struct Call {
var connection: RTCPeerConnection
var iceCandidates: IceCandidates
var localMedia: CallMediaType
var localCamera: RTCVideoCapturer?
var localVideoSource: RTCVideoSource?
var localStream: RTCVideoTrack?
var remoteStream: RTCVideoTrack?
var device: AVCaptureDevice.Position = .front
var localAudioTrack: RTCAudioTrack?
var localVideoTrack: RTCVideoTrack?
var remoteAudioTrack: RTCAudioTrack?
var remoteVideoTrack: RTCVideoTrack?
var remoteScreenAudioTrack: RTCAudioTrack?
var remoteScreenVideoTrack: RTCVideoTrack?
var device: AVCaptureDevice.Position
var aesKey: String?
var frameEncryptor: RTCFrameEncryptor?
var frameDecryptor: RTCFrameDecryptor?
var peerHasOldVersion: Bool
}
struct NotConnectedCall {
var audioTrack: RTCAudioTrack?
var localCameraAndTrack: (RTCVideoCapturer, RTCVideoTrack)?
var device: AVCaptureDevice.Position = .front
}
actor IceCandidates {
@@ -51,17 +60,20 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
private let rtcAudioSession = RTCAudioSession.sharedInstance()
private let audioQueue = DispatchQueue(label: "chat.simplex.app.audio")
private var sendCallResponse: (WVAPIMessage) async -> Void
var activeCall: Binding<Call?>
var activeCall: Call?
var notConnectedCall: NotConnectedCall?
private var localRendererAspectRatio: Binding<CGFloat?>
var cameraRenderers: [RTCVideoRenderer] = []
var screenRenderers: [RTCVideoRenderer] = []
@available(*, unavailable)
override init() {
fatalError("Unimplemented")
}
required init(_ activeCall: Binding<Call?>, _ sendCallResponse: @escaping (WVAPIMessage) async -> Void, _ localRendererAspectRatio: Binding<CGFloat?>) {
required init(_ sendCallResponse: @escaping (WVAPIMessage) async -> Void, _ localRendererAspectRatio: Binding<CGFloat?>) {
self.sendCallResponse = sendCallResponse
self.activeCall = activeCall
self.localRendererAspectRatio = localRendererAspectRatio
rtcAudioSession.useManualAudio = CallController.useCallKit()
rtcAudioSession.isAudioEnabled = !CallController.useCallKit()
@@ -78,39 +90,45 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
func initializeCall(_ iceServers: [WebRTC.RTCIceServer]?, _ mediaType: CallMediaType, _ aesKey: String?, _ relay: Bool?) -> Call {
let connection = createPeerConnection(iceServers ?? getWebRTCIceServers() ?? defaultIceServers, relay)
connection.delegate = self
createAudioSender(connection)
var localStream: RTCVideoTrack? = nil
var remoteStream: RTCVideoTrack? = nil
let device = notConnectedCall?.device ?? .front
var localCamera: RTCVideoCapturer? = nil
var localVideoSource: RTCVideoSource? = nil
if mediaType == .video {
(localStream, remoteStream, localCamera, localVideoSource) = createVideoSender(connection)
var localAudioTrack: RTCAudioTrack? = nil
var localVideoTrack: RTCVideoTrack? = nil
if let localCameraAndTrack = notConnectedCall?.localCameraAndTrack {
(localCamera, localVideoTrack) = localCameraAndTrack
} else if notConnectedCall == nil && mediaType == .video {
(localCamera, localVideoTrack) = createVideoTrackAndStartCapture(device)
}
if let audioTrack = notConnectedCall?.audioTrack {
localAudioTrack = audioTrack
} else if notConnectedCall == nil {
localAudioTrack = createAudioTrack()
}
notConnectedCall?.localCameraAndTrack = nil
notConnectedCall?.audioTrack = nil
var frameEncryptor: RTCFrameEncryptor? = nil
var frameDecryptor: RTCFrameDecryptor? = nil
if aesKey != nil {
let encryptor = RTCFrameEncryptor.init(sizeChange: Int32(WebRTCClient.ivTagBytes))
encryptor.delegate = self
frameEncryptor = encryptor
connection.senders.forEach { $0.setRtcFrameEncryptor(encryptor) }
let decryptor = RTCFrameDecryptor.init(sizeChange: -Int32(WebRTCClient.ivTagBytes))
decryptor.delegate = self
frameDecryptor = decryptor
// Has no video receiver in outgoing call if applied here, see [peerConnection(_ connection: RTCPeerConnection, didChange newState]
// connection.receivers.forEach { $0.setRtcFrameDecryptor(decryptor) }
}
return Call(
connection: connection,
iceCandidates: IceCandidates(),
localMedia: mediaType,
localCamera: localCamera,
localVideoSource: localVideoSource,
localStream: localStream,
remoteStream: remoteStream,
localAudioTrack: localAudioTrack,
localVideoTrack: localVideoTrack,
device: device,
aesKey: aesKey,
frameEncryptor: frameEncryptor,
frameDecryptor: frameDecryptor
frameDecryptor: frameDecryptor,
peerHasOldVersion: false
)
}
@@ -151,18 +169,24 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
func sendCallCommand(command: WCallCommand) async {
var resp: WCallResponse? = nil
let pc = activeCall.wrappedValue?.connection
let pc = activeCall?.connection
switch command {
case .capabilities:
case let .capabilities(media): // outgoing
let localCameraAndTrack: (RTCVideoCapturer, RTCVideoTrack)? = media == .video
? createVideoTrackAndStartCapture(.front)
: nil
notConnectedCall = NotConnectedCall(audioTrack: createAudioTrack(), localCameraAndTrack: localCameraAndTrack, device: .front)
resp = .capabilities(capabilities: CallCapabilities(encryption: WebRTCClient.enableEncryption))
case let .start(media: media, aesKey, iceServers, relay):
case let .start(media: media, aesKey, iceServers, relay): // incoming
logger.debug("starting incoming call - create webrtc session")
if activeCall.wrappedValue != nil { endCall() }
if activeCall != nil { endCall() }
let encryption = WebRTCClient.enableEncryption
let call = initializeCall(iceServers?.toWebRTCIceServers(), media, encryption ? aesKey : nil, relay)
activeCall.wrappedValue = call
activeCall = call
setupLocalTracks(true, call)
let (offer, error) = await call.connection.offer()
if let offer = offer {
setupEncryptionForLocalTracks(call)
resp = .offer(
offer: compressToBase64(input: encodeJSON(CustomRTCSessionDescription(type: offer.type.toSdpType(), sdp: offer.sdp))),
iceCandidates: compressToBase64(input: encodeJSON(await self.getInitialIceCandidates())),
@@ -172,18 +196,24 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
} else {
resp = .error(message: "offer error: \(error?.localizedDescription ?? "unknown error")")
}
case let .offer(offer, iceCandidates, media, aesKey, iceServers, relay):
if activeCall.wrappedValue != nil {
case let .offer(offer, iceCandidates, media, aesKey, iceServers, relay): // outgoing
if activeCall != nil {
resp = .error(message: "accept: call already started")
} else if !WebRTCClient.enableEncryption && aesKey != nil {
resp = .error(message: "accept: encryption is not supported")
} else if let offer: CustomRTCSessionDescription = decodeJSON(decompressFromBase64(input: offer)),
let remoteIceCandidates: [RTCIceCandidate] = decodeJSON(decompressFromBase64(input: iceCandidates)) {
let call = initializeCall(iceServers?.toWebRTCIceServers(), media, WebRTCClient.enableEncryption ? aesKey : nil, relay)
activeCall.wrappedValue = call
activeCall = call
let pc = call.connection
if let type = offer.type, let sdp = offer.sdp {
if (try? await pc.setRemoteDescription(RTCSessionDescription(type: type.toWebRTCSdpType(), sdp: sdp))) != nil {
setupLocalTracks(false, call)
setupEncryptionForLocalTracks(call)
pc.transceivers.forEach { transceiver in
transceiver.setDirection(.sendRecv, error: nil)
}
await adaptToOldVersion(pc.transceivers.count <= 2)
let (answer, error) = await pc.answer()
if let answer = answer {
self.addIceCandidates(pc, remoteIceCandidates)
@@ -200,7 +230,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
}
}
}
case let .answer(answer, iceCandidates):
case let .answer(answer, iceCandidates): // incoming
if pc == nil {
resp = .error(message: "answer: call not started")
} else if pc?.localDescription == nil {
@@ -212,6 +242,9 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
let type = answer.type, let sdp = answer.sdp,
let pc = pc {
if (try? await pc.setRemoteDescription(RTCSessionDescription(type: type.toWebRTCSdpType(), sdp: sdp))) != nil {
var currentDirection: RTCRtpTransceiverDirection = .sendOnly
pc.transceivers[2].currentDirection(&currentDirection)
await adaptToOldVersion(currentDirection == .sendOnly)
addIceCandidates(pc, remoteIceCandidates)
resp = .ok
} else {
@@ -226,13 +259,11 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
} else {
resp = .error(message: "ice: call not started")
}
case let .media(media, enable):
if activeCall.wrappedValue == nil {
case let .media(source, enable):
if activeCall == nil {
resp = .error(message: "media: call not started")
} else if activeCall.wrappedValue?.localMedia == .audio && media == .video {
resp = .error(message: "media: no video")
} else {
enableMedia(media, enable)
await enableMedia(source, enable)
resp = .ok
}
case .end:
@@ -247,7 +278,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
func getInitialIceCandidates() async -> [RTCIceCandidate] {
await untilIceComplete(timeoutMs: 750, stepMs: 150) {}
let candidates = await activeCall.wrappedValue?.iceCandidates.getAndClear() ?? []
let candidates = await activeCall?.iceCandidates.getAndClear() ?? []
logger.debug("WebRTCClient: sending initial ice candidates: \(candidates.count)")
return candidates
}
@@ -255,7 +286,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
func waitForMoreIceCandidates() {
Task {
await untilIceComplete(timeoutMs: 12000, stepMs: 1500) {
let candidates = await self.activeCall.wrappedValue?.iceCandidates.getAndClear() ?? []
let candidates = await self.activeCall?.iceCandidates.getAndClear() ?? []
if candidates.count > 0 {
logger.debug("WebRTCClient: sending more ice candidates: \(candidates.count)")
await self.sendIceCandidates(candidates)
@@ -272,25 +303,203 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
)
}
func enableMedia(_ media: CallMediaType, _ enable: Bool) {
logger.debug("WebRTCClient: enabling media \(media.rawValue) \(enable)")
media == .video ? setVideoEnabled(enable) : setAudioEnabled(enable)
func setupMuteUnmuteListener(_ transceiver: RTCRtpTransceiver, _ track: RTCMediaStreamTrack) {
// logger.log("Setting up mute/unmute listener in the call without encryption for mid = \(transceiver.mid)")
Task {
// for some reason even for disabled tracks one packet arrives (seeing this on screenVideo track)
var lastPacketsReceived = 1
// muted initially
var mutedSeconds = 4
while let call = self.activeCall, transceiver.receiver.track?.readyState == .live {
let stats: RTCStatisticsReport = await call.connection.statistics(for: transceiver.receiver)
let stat = stats.statistics.values.first(where: { stat in stat.type == "inbound-rtp"})
if let stat {
//logger.debug("Stat \(stat.debugDescription)")
let packets = stat.values["packetsReceived"] as! Int
if packets <= lastPacketsReceived {
mutedSeconds += 1
if mutedSeconds == 3 {
await MainActor.run {
self.onMediaMuteUnmute(transceiver.mid, true)
}
}
} else {
if mutedSeconds >= 3 {
await MainActor.run {
self.onMediaMuteUnmute(transceiver.mid, false)
}
}
lastPacketsReceived = packets
mutedSeconds = 0
}
}
try? await Task.sleep(nanoseconds: 1000_000000)
}
}
}
func addLocalRenderer(_ activeCall: Call, _ renderer: RTCEAGLVideoView) {
activeCall.localStream?.add(renderer)
@MainActor
func onMediaMuteUnmute(_ transceiverMid: String?, _ mute: Bool) {
guard let activeCall = ChatModel.shared.activeCall else { return }
let source = mediaSourceFromTransceiverMid(transceiverMid)
logger.log("Mute/unmute \(source.rawValue) track = \(mute) with mid = \(transceiverMid ?? "nil")")
if source == .mic && activeCall.peerMediaSources.mic == mute {
activeCall.peerMediaSources.mic = !mute
} else if (source == .camera && activeCall.peerMediaSources.camera == mute) {
activeCall.peerMediaSources.camera = !mute
} else if (source == .screenAudio && activeCall.peerMediaSources.screenAudio == mute) {
activeCall.peerMediaSources.screenAudio = !mute
} else if (source == .screenVideo && activeCall.peerMediaSources.screenVideo == mute) {
activeCall.peerMediaSources.screenVideo = !mute
}
}
@MainActor
func enableMedia(_ source: CallMediaSource, _ enable: Bool) {
logger.debug("WebRTCClient: enabling media \(source.rawValue) \(enable)")
source == .camera ? setCameraEnabled(enable) : setAudioEnabled(enable)
}
@MainActor
func adaptToOldVersion(_ peerHasOldVersion: Bool) {
activeCall?.peerHasOldVersion = peerHasOldVersion
if peerHasOldVersion {
logger.debug("The peer has an old version. Remote audio track is nil = \(self.activeCall?.remoteAudioTrack == nil), video = \(self.activeCall?.remoteVideoTrack == nil)")
onMediaMuteUnmute("0", false)
if activeCall?.remoteVideoTrack != nil {
onMediaMuteUnmute("1", false)
}
if ChatModel.shared.activeCall?.localMediaSources.camera == true && ChatModel.shared.activeCall?.peerMediaSources.camera == false {
logger.debug("Stopping video track for the old version")
activeCall?.connection.senders[1].track = nil
ChatModel.shared.activeCall?.localMediaSources.camera = false
(activeCall?.localCamera as? RTCCameraVideoCapturer)?.stopCapture()
activeCall?.localCamera = nil
activeCall?.localVideoTrack = nil
}
}
}
func addLocalRenderer(_ renderer: RTCEAGLVideoView) {
if let activeCall {
if let track = activeCall.localVideoTrack {
track.add(renderer)
}
} else if let notConnectedCall {
if let track = notConnectedCall.localCameraAndTrack?.1 {
track.add(renderer)
}
}
// To get width and height of a frame, see videoView(videoView:, didChangeVideoSize)
renderer.delegate = self
}
func removeLocalRenderer(_ renderer: RTCEAGLVideoView) {
if let activeCall {
if let track = activeCall.localVideoTrack {
track.remove(renderer)
}
} else if let notConnectedCall {
if let track = notConnectedCall.localCameraAndTrack?.1 {
track.remove(renderer)
}
}
renderer.delegate = nil
}
func videoView(_ videoView: RTCVideoRenderer, didChangeVideoSize size: CGSize) {
guard size.height > 0 else { return }
localRendererAspectRatio.wrappedValue = size.width / size.height
}
func setupLocalTracks(_ incomingCall: Bool, _ call: Call) {
let pc = call.connection
let transceivers = call.connection.transceivers
let audioTrack = call.localAudioTrack
let videoTrack = call.localVideoTrack
if incomingCall {
let micCameraInit = RTCRtpTransceiverInit()
// streamIds required for old versions which adds tracks from stream, not from track property
micCameraInit.streamIds = ["micCamera"]
let screenAudioVideoInit = RTCRtpTransceiverInit()
screenAudioVideoInit.streamIds = ["screenAudioVideo"]
// incoming call, no transceivers yet. But they should be added in order: mic, camera, screen audio, screen video
// mid = 0, mic
if let audioTrack {
pc.addTransceiver(with: audioTrack, init: micCameraInit)
} else {
pc.addTransceiver(of: .audio, init: micCameraInit)
}
// mid = 1, camera
if let videoTrack {
pc.addTransceiver(with: videoTrack, init: micCameraInit)
} else {
pc.addTransceiver(of: .video, init: micCameraInit)
}
// mid = 2, screenAudio
pc.addTransceiver(of: .audio, init: screenAudioVideoInit)
// mid = 3, screenVideo
pc.addTransceiver(of: .video, init: screenAudioVideoInit)
} else {
// new version
if transceivers.count > 2 {
// Outgoing call. All transceivers are ready. Don't addTrack() because it will create new transceivers, replace existing (nil) tracks
transceivers
.first(where: { elem in mediaSourceFromTransceiverMid(elem.mid) == .mic })?
.sender.track = audioTrack
transceivers
.first(where: { elem in mediaSourceFromTransceiverMid(elem.mid) == .camera })?
.sender.track = videoTrack
} else {
// old version, only two transceivers
if let audioTrack {
pc.add(audioTrack, streamIds: ["micCamera"])
} else {
// it's important to have any track in order to be able to turn it on again (currently it's off)
let sender = pc.add(createAudioTrack(), streamIds: ["micCamera"])
sender?.track = nil
}
if let videoTrack {
pc.add(videoTrack, streamIds: ["micCamera"])
} else {
// it's important to have any track in order to be able to turn it on again (currently it's off)
let localVideoSource = WebRTCClient.factory.videoSource()
let localVideoTrack = WebRTCClient.factory.videoTrack(with: localVideoSource, trackId: "video0")
let sender = pc.add(localVideoTrack, streamIds: ["micCamera"])
sender?.track = nil
}
}
}
}
func mediaSourceFromTransceiverMid(_ mid: String?) -> CallMediaSource {
switch mid {
case "0":
return .mic
case "1":
return .camera
case "2":
return .screenAudio
case "3":
return .screenVideo
default:
return .unknown
}
}
// Should be called after local description set
func setupEncryptionForLocalTracks(_ call: Call) {
if let encryptor = call.frameEncryptor {
call.connection.senders.forEach { $0.setRtcFrameEncryptor(encryptor) }
}
}
func frameDecryptor(_ decryptor: RTCFrameDecryptor, mediaType: RTCRtpMediaType, withFrame encrypted: Data) -> Data? {
guard encrypted.count > 0 else { return nil }
if var key: [CChar] = activeCall.wrappedValue?.aesKey?.cString(using: .utf8),
if var key: [CChar] = activeCall?.aesKey?.cString(using: .utf8),
let pointer: UnsafeMutableRawPointer = malloc(encrypted.count) {
memcpy(pointer, (encrypted as NSData).bytes, encrypted.count)
let isKeyFrame = encrypted[0] & 1 == 0
@@ -304,7 +513,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
func frameEncryptor(_ encryptor: RTCFrameEncryptor, mediaType: RTCRtpMediaType, withFrame unencrypted: Data) -> Data? {
guard unencrypted.count > 0 else { return nil }
if var key: [CChar] = activeCall.wrappedValue?.aesKey?.cString(using: .utf8),
if var key: [CChar] = activeCall?.aesKey?.cString(using: .utf8),
let pointer: UnsafeMutableRawPointer = malloc(unencrypted.count + WebRTCClient.ivTagBytes) {
memcpy(pointer, (unencrypted as NSData).bytes, unencrypted.count)
let isKeyFrame = unencrypted[0] & 1 == 0
@@ -327,18 +536,42 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
}
}
func addRemoteRenderer(_ activeCall: Call, _ renderer: RTCVideoRenderer) {
activeCall.remoteStream?.add(renderer)
func addRemoteCameraRenderer(_ renderer: RTCVideoRenderer) {
if activeCall?.remoteVideoTrack != nil {
activeCall?.remoteVideoTrack?.add(renderer)
} else {
cameraRenderers.append(renderer)
}
}
func removeRemoteRenderer(_ activeCall: Call, _ renderer: RTCVideoRenderer) {
activeCall.remoteStream?.remove(renderer)
func removeRemoteCameraRenderer(_ renderer: RTCVideoRenderer) {
if activeCall?.remoteVideoTrack != nil {
activeCall?.remoteVideoTrack?.remove(renderer)
} else {
cameraRenderers.removeAll(where: { $0.isEqual(renderer) })
}
}
func startCaptureLocalVideo(_ activeCall: Call) {
func addRemoteScreenRenderer(_ renderer: RTCVideoRenderer) {
if activeCall?.remoteScreenVideoTrack != nil {
activeCall?.remoteScreenVideoTrack?.add(renderer)
} else {
screenRenderers.append(renderer)
}
}
func removeRemoteScreenRenderer(_ renderer: RTCVideoRenderer) {
if activeCall?.remoteScreenVideoTrack != nil {
activeCall?.remoteScreenVideoTrack?.remove(renderer)
} else {
screenRenderers.removeAll(where: { $0.isEqual(renderer) })
}
}
func startCaptureLocalVideo(_ device: AVCaptureDevice.Position?, _ capturer: RTCVideoCapturer?) {
#if targetEnvironment(simulator)
guard
let capturer = activeCall.localCamera as? RTCFileVideoCapturer
let capturer = (activeCall?.localCamera ?? notConnectedCall?.localCameraAndTrack?.0) as? RTCFileVideoCapturer
else {
logger.error("Unable to work with a file capturer")
return
@@ -348,10 +581,10 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
capturer.startCapturing(fromFileNamed: "sounds/video.mp4")
#else
guard
let capturer = activeCall.localCamera as? RTCCameraVideoCapturer,
let camera = (RTCCameraVideoCapturer.captureDevices().first { $0.position == activeCall.device })
let capturer = capturer as? RTCCameraVideoCapturer,
let camera = (RTCCameraVideoCapturer.captureDevices().first { $0.position == device })
else {
logger.error("Unable to find a camera")
logger.error("Unable to find a camera or local track")
return
}
@@ -377,19 +610,6 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
#endif
}
private func createAudioSender(_ connection: RTCPeerConnection) {
let streamId = "stream"
let audioTrack = createAudioTrack()
connection.add(audioTrack, streamIds: [streamId])
}
private func createVideoSender(_ connection: RTCPeerConnection) -> (RTCVideoTrack?, RTCVideoTrack?, RTCVideoCapturer?, RTCVideoSource?) {
let streamId = "stream"
let (localVideoTrack, localCamera, localVideoSource) = createVideoTrack()
connection.add(localVideoTrack, streamIds: [streamId])
return (localVideoTrack, connection.transceivers.first { $0.mediaType == .video }?.receiver.track as? RTCVideoTrack, localCamera, localVideoSource)
}
private func createAudioTrack() -> RTCAudioTrack {
let audioConstrains = RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil)
let audioSource = WebRTCClient.factory.audioSource(with: audioConstrains)
@@ -397,7 +617,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
return audioTrack
}
private func createVideoTrack() -> (RTCVideoTrack, RTCVideoCapturer, RTCVideoSource) {
private func createVideoTrackAndStartCapture(_ device: AVCaptureDevice.Position) -> (RTCVideoCapturer, RTCVideoTrack) {
let localVideoSource = WebRTCClient.factory.videoSource()
#if targetEnvironment(simulator)
@@ -407,7 +627,8 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
#endif
let localVideoTrack = WebRTCClient.factory.videoTrack(with: localVideoSource, trackId: "video0")
return (localVideoTrack, localCamera, localVideoSource)
startCaptureLocalVideo(device, localCamera)
return (localCamera, localVideoTrack)
}
func endCall() {
@@ -420,15 +641,16 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
}
private func _endCall() {
guard let call = activeCall.wrappedValue else { return }
(notConnectedCall?.localCameraAndTrack?.0 as? RTCCameraVideoCapturer)?.stopCapture()
guard let call = activeCall else { return }
logger.debug("WebRTCClient: ending the call")
activeCall.wrappedValue = nil
(call.localCamera as? RTCCameraVideoCapturer)?.stopCapture()
call.connection.close()
call.connection.delegate = nil
call.frameEncryptor?.delegate = nil
call.frameDecryptor?.delegate = nil
(call.localCamera as? RTCCameraVideoCapturer)?.stopCapture()
audioSessionToDefaults()
activeCall = nil
}
func untilIceComplete(timeoutMs: UInt64, stepMs: UInt64, action: @escaping () async -> Void) async {
@@ -437,7 +659,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
_ = try? await Task.sleep(nanoseconds: stepMs * 1000000)
t += stepMs
await action()
} while t < timeoutMs && activeCall.wrappedValue?.connection.iceGatheringState != .complete
} while t < timeoutMs && activeCall?.connection.iceGatheringState != .complete
}
}
@@ -498,11 +720,40 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
logger.debug("Connection should negotiate")
}
func peerConnection(_ peerConnection: RTCPeerConnection, didStartReceivingOn transceiver: RTCRtpTransceiver) {
if let track = transceiver.receiver.track {
DispatchQueue.main.async {
// Doesn't work for outgoing video call (audio in video call works ok still, same as incoming call)
// if let decryptor = self.activeCall?.frameDecryptor {
// transceiver.receiver.setRtcFrameDecryptor(decryptor)
// }
let source = self.mediaSourceFromTransceiverMid(transceiver.mid)
switch source {
case .mic: self.activeCall?.remoteAudioTrack = track as? RTCAudioTrack
case .camera:
self.activeCall?.remoteVideoTrack = track as? RTCVideoTrack
self.cameraRenderers.forEach({ renderer in
self.activeCall?.remoteVideoTrack?.add(renderer)
})
self.cameraRenderers.removeAll()
case .screenAudio: self.activeCall?.remoteScreenAudioTrack = track as? RTCAudioTrack
case .screenVideo:
self.activeCall?.remoteScreenVideoTrack = track as? RTCVideoTrack
self.screenRenderers.forEach({ renderer in
self.activeCall?.remoteScreenVideoTrack?.add(renderer)
})
self.screenRenderers.removeAll()
case .unknown: ()
}
}
self.setupMuteUnmuteListener(transceiver, track)
}
}
func peerConnection(_ connection: RTCPeerConnection, didChange newState: RTCIceConnectionState) {
debugPrint("Connection new connection state: \(newState.toString() ?? "" + newState.rawValue.description) \(connection.receivers)")
guard let call = activeCall.wrappedValue,
let connectionStateString = newState.toString(),
guard let connectionStateString = newState.toString(),
let iceConnectionStateString = connection.iceConnectionState.toString(),
let iceGatheringStateString = connection.iceGatheringState.toString(),
let signalingStateString = connection.signalingState.toString()
@@ -523,18 +774,14 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
switch newState {
case .checking:
if let frameDecryptor = activeCall.wrappedValue?.frameDecryptor {
if let frameDecryptor = activeCall?.frameDecryptor {
connection.receivers.forEach { $0.setRtcFrameDecryptor(frameDecryptor) }
}
let enableSpeaker: Bool
switch call.localMedia {
case .video: enableSpeaker = true
default: enableSpeaker = false
}
let enableSpeaker: Bool = ChatModel.shared.activeCall?.localMediaSources.hasVideo == true
setSpeakerEnabledAndConfigureSession(enableSpeaker)
case .connected: sendConnectedEvent(connection)
case .disconnected, .failed: endCall()
default: do {}
default: ()
}
}
}
@@ -546,7 +793,7 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
func peerConnection(_ connection: RTCPeerConnection, didGenerate candidate: WebRTC.RTCIceCandidate) {
// logger.debug("Connection generated candidate \(candidate.debugDescription)")
Task {
await self.activeCall.wrappedValue?.iceCandidates.append(candidate.toCandidate(nil, nil))
await self.activeCall?.iceCandidates.append(candidate.toCandidate(nil, nil))
}
}
@@ -601,11 +848,42 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
}
extension WebRTCClient {
func setAudioEnabled(_ enabled: Bool) {
setTrackEnabled(RTCAudioTrack.self, enabled)
static func isAuthorized(for type: AVMediaType) async -> Bool {
let status = AVCaptureDevice.authorizationStatus(for: type)
var isAuthorized = status == .authorized
if status == .notDetermined {
isAuthorized = await AVCaptureDevice.requestAccess(for: type)
}
return isAuthorized
}
func setSpeakerEnabledAndConfigureSession( _ enabled: Bool) {
static func showUnauthorizedAlert(for type: AVMediaType) {
if type == .audio {
AlertManager.shared.showAlert(Alert(
title: Text("No permission to record speech"),
message: Text("To record speech please grant permission to use Microphone."),
primaryButton: .default(Text("Open Settings")) {
DispatchQueue.main.async {
UIApplication.shared.open(URL(string: UIApplication.openSettingsURLString)!, options: [:], completionHandler: nil)
}
},
secondaryButton: .cancel()
))
} else if type == .video {
AlertManager.shared.showAlert(Alert(
title: Text("No permission to record video"),
message: Text("To record video please grant permission to use Camera."),
primaryButton: .default(Text("Open Settings")) {
DispatchQueue.main.async {
UIApplication.shared.open(URL(string: UIApplication.openSettingsURLString)!, options: [:], completionHandler: nil)
}
},
secondaryButton: .cancel()
))
}
}
func setSpeakerEnabledAndConfigureSession( _ enabled: Bool, skipExternalDevice: Bool = false) {
logger.debug("WebRTCClient: configuring session with speaker enabled \(enabled)")
audioQueue.async { [weak self] in
guard let self = self else { return }
@@ -618,7 +896,7 @@ extension WebRTCClient {
if enabled {
try self.rtcAudioSession.setCategory(AVAudioSession.Category.playAndRecord.rawValue, with: [.defaultToSpeaker, .allowBluetooth, .allowAirPlay, .allowBluetoothA2DP])
try self.rtcAudioSession.setMode(AVAudioSession.Mode.videoChat.rawValue)
if hasExternalAudioDevice, let preferred = self.rtcAudioSession.session.preferredInputDevice() {
if hasExternalAudioDevice && !skipExternalDevice, let preferred = self.rtcAudioSession.session.preferredInputDevice() {
try self.rtcAudioSession.setPreferredInput(preferred)
} else {
try self.rtcAudioSession.overrideOutputAudioPort(.speaker)
@@ -628,7 +906,7 @@ extension WebRTCClient {
try self.rtcAudioSession.setMode(AVAudioSession.Mode.voiceChat.rawValue)
try self.rtcAudioSession.overrideOutputAudioPort(.none)
}
if hasExternalAudioDevice {
if hasExternalAudioDevice && !skipExternalDevice {
logger.debug("WebRTCClient: configuring session with external device available, skip configuring speaker")
}
try self.rtcAudioSession.setActive(true)
@@ -659,25 +937,59 @@ extension WebRTCClient {
}
}
func setVideoEnabled(_ enabled: Bool) {
setTrackEnabled(RTCVideoTrack.self, enabled)
@MainActor
func setAudioEnabled(_ enabled: Bool) {
if activeCall != nil {
activeCall?.localAudioTrack = enabled ? createAudioTrack() : nil
activeCall?.connection.transceivers.first(where: { t in mediaSourceFromTransceiverMid(t.mid) == .mic })?.sender.track = activeCall?.localAudioTrack
} else if notConnectedCall != nil {
notConnectedCall?.audioTrack = enabled ? createAudioTrack() : nil
}
ChatModel.shared.activeCall?.localMediaSources.mic = enabled
}
@MainActor
func setCameraEnabled(_ enabled: Bool) {
if let call = activeCall {
if enabled {
if call.localVideoTrack == nil {
let device = activeCall?.device ?? notConnectedCall?.device ?? .front
let (camera, track) = createVideoTrackAndStartCapture(device)
activeCall?.localCamera = camera
activeCall?.localVideoTrack = track
}
} else {
(call.localCamera as? RTCCameraVideoCapturer)?.stopCapture()
activeCall?.localCamera = nil
activeCall?.localVideoTrack = nil
}
call.connection.transceivers
.first(where: { t in mediaSourceFromTransceiverMid(t.mid) == .camera })?
.sender.track = activeCall?.localVideoTrack
ChatModel.shared.activeCall?.localMediaSources.camera = activeCall?.localVideoTrack != nil
} else if let call = notConnectedCall {
if enabled {
let device = activeCall?.device ?? notConnectedCall?.device ?? .front
notConnectedCall?.localCameraAndTrack = createVideoTrackAndStartCapture(device)
} else {
(call.localCameraAndTrack?.0 as? RTCCameraVideoCapturer)?.stopCapture()
notConnectedCall?.localCameraAndTrack = nil
}
ChatModel.shared.activeCall?.localMediaSources.camera = notConnectedCall?.localCameraAndTrack != nil
}
}
func flipCamera() {
switch activeCall.wrappedValue?.device {
case .front: activeCall.wrappedValue?.device = .back
case .back: activeCall.wrappedValue?.device = .front
default: ()
let device = activeCall?.device ?? notConnectedCall?.device
if activeCall != nil {
activeCall?.device = device == .front ? .back : .front
} else {
notConnectedCall?.device = device == .front ? .back : .front
}
if let call = activeCall.wrappedValue {
startCaptureLocalVideo(call)
}
}
private func setTrackEnabled<T: RTCMediaStreamTrack>(_ type: T.Type, _ enabled: Bool) {
activeCall.wrappedValue?.connection.transceivers
.compactMap { $0.sender.track as? T }
.forEach { $0.isEnabled = enabled }
startCaptureLocalVideo(
activeCall?.device ?? notConnectedCall?.device,
(activeCall?.localCamera ?? notConnectedCall?.localCameraAndTrack?.0) as? RTCCameraVideoCapturer
)
}
}
+8
View File
@@ -80,6 +80,14 @@ public enum CallMediaType: String, Codable, Equatable {
case audio = "audio"
}
public enum CallMediaSource: String, Codable, Equatable {
case mic = "mic"
case camera = "camera"
case screenAudio = "screenAudio"
case screenVideo = "screenVideo"
case unknown = "unknown"
}
public enum VideoCamera: String, Codable, Equatable {
case user = "user"
case environment = "environment"
@@ -71,6 +71,9 @@ object ChatModel {
val groupMembers = mutableStateListOf<GroupMember>()
val groupMembersIndexes = mutableStateMapOf<Long, Int>()
// false: default placement, true: floating window.
// Used for deciding to add terminal items on main thread or not. Floating means appPrefs.terminalAlwaysVisible
var terminalsVisible = setOf<Boolean>()
val terminalItems = mutableStateOf<List<TerminalItem>>(listOf())
val userAddress = mutableStateOf<UserContactLinkRec?>(null)
val chatItemTTL = mutableStateOf<ChatItemTTL>(ChatItemTTL.None)
@@ -772,6 +775,16 @@ object ChatModel {
fun addTerminalItem(item: TerminalItem) {
val maxItems = if (appPreferences.developerTools.get()) 500 else 200
if (terminalsVisible.isNotEmpty()) {
withApi {
addTerminalItem(item, maxItems)
}
} else {
addTerminalItem(item, maxItems)
}
}
private fun addTerminalItem(item: TerminalItem, maxItems: Int) {
if (terminalItems.value.size >= maxItems) {
terminalItems.value = terminalItems.value.subList(1, terminalItems.value.size)
}
@@ -20,11 +20,12 @@ import chat.simplex.common.views.chat.*
import chat.simplex.common.views.helpers.*
import chat.simplex.common.model.ChatModel
import chat.simplex.common.platform.*
import chat.simplex.res.MR
import dev.icerock.moko.resources.compose.stringResource
import kotlinx.coroutines.flow.collect
import kotlinx.coroutines.flow.filter
import kotlinx.coroutines.launch
@Composable
fun TerminalView(chatModel: ChatModel, close: () -> Unit) {
fun TerminalView(floating: Boolean = false, close: () -> Unit) {
val composeState = remember { mutableStateOf(ComposeState(useLinkPreviews = false)) }
val close = {
close()
@@ -37,6 +38,7 @@ fun TerminalView(chatModel: ChatModel, close: () -> Unit) {
})
TerminalLayout(
composeState,
floating,
sendCommand = { sendCommand(chatModel, composeState) },
close
)
@@ -65,6 +67,7 @@ private fun sendCommand(chatModel: ChatModel, composeState: MutableState<Compose
@Composable
fun TerminalLayout(
composeState: MutableState<ComposeState>,
floating: Boolean,
sendCommand: () -> Unit,
close: () -> Unit
) {
@@ -118,19 +121,40 @@ fun TerminalLayout(
color = MaterialTheme.colors.background,
contentColor = LocalContentColor.current
) {
TerminalLog()
TerminalLog(floating)
}
}
}
}
@Composable
fun TerminalLog() {
fun TerminalLog(floating: Boolean) {
val reversedTerminalItems by remember {
derivedStateOf { chatModel.terminalItems.value.asReversed() }
}
val clipboard = LocalClipboardManager.current
LazyColumnWithScrollBar(reverseLayout = true) {
val listState = LocalAppBarHandler.current?.listState ?: rememberLazyListState()
LaunchedEffect(Unit) {
var autoScrollToBottom = true
launch {
snapshotFlow { listState.layoutInfo.totalItemsCount }
.filter { autoScrollToBottom }
.collect {
try {
listState.scrollToItem(0)
} catch (e: Exception) {
Log.e(TAG, e.stackTraceToString())
}
}
}
launch {
snapshotFlow { listState.firstVisibleItemIndex }
.collect {
autoScrollToBottom = listState.firstVisibleItemIndex == 0
}
}
}
LazyColumnWithScrollBar(reverseLayout = true, state = listState) {
items(reversedTerminalItems, key = { item -> item.id to item.createdAtNanos }) { item ->
val rhId = item.remoteHostId
val rhIdStr = if (rhId == null) "" else "$rhId "
@@ -142,7 +166,12 @@ fun TerminalLog() {
modifier = Modifier
.fillMaxWidth()
.clickable {
ModalManager.start.showModal(endButtons = { ShareButton { clipboard.shareText(item.details) } }) {
val modalPlace = if (floating) {
ModalManager.floatingTerminal
} else {
ModalManager.start
}
modalPlace.showModal(endButtons = { ShareButton { clipboard.shareText(item.details) } }) {
SelectionContainer(modifier = Modifier.verticalScroll(rememberScrollState())) {
val details = item.details
.let {
@@ -156,6 +185,16 @@ fun TerminalLog() {
)
}
}
DisposableEffect(Unit) {
val terminals = chatModel.terminalsVisible.toMutableSet()
terminals += floating
chatModel.terminalsVisible = terminals
onDispose {
val terminals = chatModel.terminalsVisible.toMutableSet()
terminals -= floating
chatModel.terminalsVisible = terminals
}
}
}
@Preview/*(
@@ -169,6 +208,7 @@ fun PreviewTerminalLayout() {
TerminalLayout(
composeState = remember { mutableStateOf(ComposeState(useLinkPreviews = false)) },
sendCommand = {},
floating = false,
close = {}
)
}
@@ -2,10 +2,8 @@ package chat.simplex.common.views.helpers
import androidx.compose.animation.*
import androidx.compose.animation.core.*
import androidx.compose.foundation.ScrollState
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.*
import androidx.compose.foundation.lazy.LazyListState
import androidx.compose.material.*
import androidx.compose.runtime.*
import androidx.compose.ui.Modifier
@@ -209,11 +207,14 @@ class ModalManager(private val placement: ModalPlacement? = null) {
val end = if (appPlatform.isAndroid) shared else ModalManager(ModalPlacement.END)
val fullscreen = if (appPlatform.isAndroid) shared else ModalManager(ModalPlacement.FULLSCREEN)
val floatingTerminal = if (appPlatform.isAndroid) shared else ModalManager(ModalPlacement.START)
fun closeAllModalsEverywhere() {
start.closeModals()
center.closeModals()
end.closeModals()
fullscreen.closeModals()
floatingTerminal.closeModals()
}
@OptIn(ExperimentalAnimationApi::class)
@@ -35,7 +35,7 @@ fun DeveloperView(
val unchangedHints = mutableStateOf(unchangedHintPreferences())
SectionView {
InstallTerminalAppItem(uriHandler)
ChatConsoleItem { withAuth(generalGetString(MR.strings.auth_open_chat_console), generalGetString(MR.strings.auth_log_in_using_credential), showCustomModal { it, close -> TerminalView(it, close) }) }
ChatConsoleItem { withAuth(generalGetString(MR.strings.auth_open_chat_console), generalGetString(MR.strings.auth_log_in_using_credential), showCustomModal { it, close -> TerminalView(false, close) }) }
ResetHintsItem(unchangedHints)
SettingsPreferenceItem(painterResource(MR.images.ic_code), stringResource(MR.strings.show_developer_options), developerTools)
SectionTextFooter(
@@ -199,7 +199,13 @@ private fun ApplicationScope.AppWindow(closedByError: MutableState<Boolean>) {
768.dp)
Window(state = cWindowState, onCloseRequest = { hiddenUntilRestart = true }, title = stringResource(MR.strings.chat_console)) {
SimpleXTheme {
TerminalView(ChatModel) { hiddenUntilRestart = true }
TerminalView(true) { hiddenUntilRestart = true }
ModalManager.floatingTerminal.showInView()
DisposableEffect(Unit) {
onDispose {
ModalManager.floatingTerminal.closeModals()
}
}
}
}
}
+1 -1
View File
@@ -1,5 +1,5 @@
name: simplex-chat
version: 6.1.0.6
version: 6.1.0.7
#synopsis:
#description:
homepage: https://github.com/simplex-chat/simplex-chat#readme
+2 -1
View File
@@ -5,7 +5,7 @@ cabal-version: 1.12
-- see: https://github.com/sol/hpack
name: simplex-chat
version: 6.1.0.6
version: 6.1.0.7
category: Web, System, Services, Cryptography
homepage: https://github.com/simplex-chat/simplex-chat#readme
author: simplex.chat
@@ -148,6 +148,7 @@ library
Simplex.Chat.Migrations.M20240528_quota_err_counter
Simplex.Chat.Migrations.M20240827_calls_uuid
Simplex.Chat.Migrations.M20240920_user_order
Simplex.Chat.Migrations.M20241008_indexes
Simplex.Chat.Mobile
Simplex.Chat.Mobile.File
Simplex.Chat.Mobile.Shared
@@ -0,0 +1,18 @@
{-# LANGUAGE QuasiQuotes #-}
module Simplex.Chat.Migrations.M20241008_indexes where
import Database.SQLite.Simple (Query)
import Database.SQLite.Simple.QQ (sql)
m20241008_indexes :: Query
m20241008_indexes =
[sql|
CREATE INDEX idx_received_probes_group_member_id on received_probes(group_member_id);
|]
down_m20241008_indexes :: Query
down_m20241008_indexes =
[sql|
DROP INDEX idx_received_probes_group_member_id;
|]
@@ -885,3 +885,6 @@ CREATE INDEX idx_chat_items_fwd_from_group_id ON chat_items(fwd_from_group_id);
CREATE INDEX idx_chat_items_fwd_from_chat_item_id ON chat_items(
fwd_from_chat_item_id
);
CREATE INDEX idx_received_probes_group_member_id on received_probes(
group_member_id
);
+3 -1
View File
@@ -112,6 +112,7 @@ import Simplex.Chat.Migrations.M20240515_rcv_files_user_approved_relays
import Simplex.Chat.Migrations.M20240528_quota_err_counter
import Simplex.Chat.Migrations.M20240827_calls_uuid
import Simplex.Chat.Migrations.M20240920_user_order
import Simplex.Chat.Migrations.M20241008_indexes
import Simplex.Messaging.Agent.Store.SQLite.Migrations (Migration (..))
schemaMigrations :: [(String, Query, Maybe Query)]
@@ -223,7 +224,8 @@ schemaMigrations =
("20240515_rcv_files_user_approved_relays", m20240515_rcv_files_user_approved_relays, Just down_m20240515_rcv_files_user_approved_relays),
("20240528_quota_err_counter", m20240528_quota_err_counter, Just down_m20240528_quota_err_counter),
("20240827_calls_uuid", m20240827_calls_uuid, Just down_m20240827_calls_uuid),
("20240920_user_order", m20240920_user_order, Just down_m20240920_user_order)
("20240920_user_order", m20240920_user_order, Just down_m20240920_user_order),
("20241008_indexes", m20241008_indexes, Just down_m20241008_indexes)
]
-- | The list of migrations in ascending order by date