Merge branch 'master' into av/multiplatform-call-logs

apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/call/CallView.android.kt

@@ -115,7 +115,7 @@ actual fun ActiveCallView() {
       Log.d(TAG, "received from WebRTCView: $apiMsg")
       val call = chatModel.activeCall.value
       if (call != null) {
-        Log.d(TAG, "has active call $call")
+        Log.d(TAG, "has active call callState: ${call.callState}, localMedia: ${call.localMedia}, peerMedia: ${call.peerMedia}, audioEnabled: ${call.audioEnabled}, connectionInfo: ${call.connectionInfo}")
         val callRh = call.remoteHostId
         when (val r = apiMsg.resp) {
           is WCallResponse.Capabilities -> withBGApi {

apps/multiplatform/common/src/commonMain/resources/assets/www/call.js

@@ -93,8 +93,7 @@ const processCommand = (function () {
                 if (delay)
                     clearTimeout(delay);
                 resolved = true;
-                // console.log("resolveIceCandidates", JSON.stringify(candidates))
-                console.log("resolveIceCandidates");
+                console.log("resolveIceCandidates", JSON.stringify(candidates));
                 const iceCandidates = serialize(candidates);
                 candidates = [];
                 resolve(iceCandidates);
@@ -102,8 +101,7 @@ const processCommand = (function () {
             function sendIceCandidates() {
                 if (candidates.length === 0)
                     return;
-                // console.log("sendIceCandidates", JSON.stringify(candidates))
-                console.log("sendIceCandidates");
+                console.log("sendIceCandidates", JSON.stringify(candidates));
                 const iceCandidates = serialize(candidates);
                 candidates = [];
                 sendMessageToNative({ resp: { type: "ice", iceCandidates } });
@@ -112,6 +110,7 @@ const processCommand = (function () {
     }
     async function initializeCall(config, mediaType, aesKey) {
         var _a;
+        console.log("Initializing call");
         let pc;
         try {
             pc = new RTCPeerConnection(config.peerConnectionConfig);
@@ -154,6 +153,7 @@ const processCommand = (function () {
                 connectionHandler();
         }
         async function connectionHandler() {
+            console.log(`Connection state change: ${pc.connectionState}, ${pc.iceConnectionState}, ${pc.iceGatheringState}, ${pc.signalingState}`);
             sendMessageToNative({
                 resp: {
                     type: "connection",
@@ -209,6 +209,7 @@ const processCommand = (function () {
     }
     async function processCommand(body) {
         const { corrId, command } = body;
+        console.log(`process command: ${command.type}`);
         const pc = activeCall === null || activeCall === void 0 ? void 0 : activeCall.connection;
         let resp;
         try {
@@ -220,6 +221,7 @@ const processCommand = (function () {
                     // This request for local media stream is made to prompt for camera/mic permissions on call start
                     if (command.media)
                         await getLocalMediaStream(command.media, VideoCamera.User);
+                    console.log("Asked video camera permission");
                     const encryption = supportsInsertableStreams(useWorker);
                     resp = { type: "capabilities", capabilities: { encryption } };
                     break;
@@ -251,7 +253,7 @@ const processCommand = (function () {
                         iceCandidates: await activeCall.iceCandidates,
                         capabilities: { encryption },
                     };
-                    // console.log("offer response", JSON.stringify(resp))
+                    console.log("offer response", JSON.stringify(resp));
                     break;
                 }
                 case "offer":
@@ -267,7 +269,7 @@ const processCommand = (function () {
                         const { media, aesKey, iceServers, relay } = command;
                         activeCall = await initializeCall(getCallConfig(!!aesKey, iceServers, relay), media, aesKey);
                         const pc = activeCall.connection;
-                        // console.log("offer remoteIceCandidates", JSON.stringify(remoteIceCandidates))
+                        console.log("offer remoteIceCandidates", JSON.stringify(remoteIceCandidates));
                         await pc.setRemoteDescription(new RTCSessionDescription(offer));
                         const answer = await pc.createAnswer();
                         await pc.setLocalDescription(answer);
@@ -279,7 +281,7 @@ const processCommand = (function () {
                             iceCandidates: await activeCall.iceCandidates,
                         };
                     }
-                    // console.log("answer response", JSON.stringify(resp))
+                    console.log("answer response", JSON.stringify(resp));
                     break;
                 case "answer":
                     if (!pc) {
@@ -294,7 +296,7 @@ const processCommand = (function () {
                     else {
                         const answer = parse(command.answer);
                         const remoteIceCandidates = parse(command.iceCandidates);
-                        // console.log("answer remoteIceCandidates", JSON.stringify(remoteIceCandidates))
+                        console.log("answer remoteIceCandidates", JSON.stringify(remoteIceCandidates));
                         await pc.setRemoteDescription(new RTCSessionDescription(answer));
                         addIceCandidates(pc, remoteIceCandidates);
                         resp = { type: "ok" };
@@ -371,10 +373,11 @@ const processCommand = (function () {
     function addIceCandidates(conn, iceCandidates) {
         for (const c of iceCandidates) {
             conn.addIceCandidate(new RTCIceCandidate(c));
-            // console.log("addIceCandidates", JSON.stringify(c))
+            console.log("addIceCandidates", JSON.stringify(c));
         }
     }
     async function setupMediaStreams(call) {
+        console.log("Setup media streams start");
         const videos = getVideoElements();
         if (!videos)
             throw Error("no video elements");
@@ -389,6 +392,7 @@ const processCommand = (function () {
         // Without doing it manually Firefox shows black screen but video can be played in Picture-in-Picture
         videos.local.play();
         videos.remote.play();
+        console.log("Setup media streams end");
     }
     async function setupEncryptionWorker(call) {
         if (call.aesKey) {
@@ -409,6 +413,7 @@ const processCommand = (function () {
         const pc = call.connection;
         let { localStream } = call;
         for (const track of localStream.getTracks()) {
+            console.log(`Adding local track: ${track.kind}`);
             pc.addTrack(track, localStream);
         }
         if (call.aesKey && call.key) {
@@ -429,6 +434,7 @@ const processCommand = (function () {
                 }
                 for (const stream of event.streams) {
                     for (const track of stream.getTracks()) {
+                        console.log(`Adding remote track: ${track.kind}`);
                         call.remoteStream.addTrack(track);
                     }
                 }

apps/multiplatform/common/src/commonMain/resources/assets/www/desktop/call.html

@@ -11,6 +11,7 @@
       id="remote-video-stream"
       class="inline"
       playsinline
+      autoplay
       poster="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAQAAAAnOwc2AAAAEUlEQVR42mNk+M+AARiHsiAAcCIKAYwFoQ8AAAAASUVORK5CYII="
       onclick="javascript:toggleRemoteVideoFitFill()"
     ></video>
@@ -19,6 +20,7 @@
       class="inline"
       muted
       playsinline
+      autoplay
       poster="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAQAAAAnOwc2AAAAEUlEQVR42mNk+M+AARiHsiAAcCIKAYwFoQ8AAAAASUVORK5CYII="
     ></video>
     <div id="progress"></div>

apps/multiplatform/common/src/commonMain/resources/assets/www/desktop/ui.js

@@ -7,15 +7,17 @@ const socket = new WebSocket(`ws://${location.host}`);
 socket.addEventListener("open", (_event) => {
     console.log("Opened socket");
     sendMessageToNative = (msg) => {
-        console.log("Message to server");
+        var _a;
+        console.log(`Message to server will be sent: ${(_a = msg.command) === null || _a === void 0 ? void 0 : _a.type}`);
         socket.send(JSON.stringify(msg));
     };
 });
 socket.addEventListener("message", (event) => {
+    var _a;
     const parsed = JSON.parse(event.data);
     reactOnMessageFromServer(parsed);
     processCommand(parsed);
-    console.log("Message from server");
+    console.log(`Message from server finished processing: ${(_a = parsed.command) === null || _a === void 0 ? void 0 : _a.type}`);
 });
 socket.addEventListener("close", (_event) => {
     console.log("Closed socket");

apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/call/CallView.desktop.kt

@@ -30,7 +30,7 @@ actual fun ActiveCallView() {
     Log.d(TAG, "received from WebRTCController: $apiMsg")
     val call = chatModel.activeCall.value
     if (call != null) {
-      Log.d(TAG, "has active call $call")
+      Log.d(TAG, "has active call callState: ${call.callState}, localMedia: ${call.localMedia}, peerMedia: ${call.peerMedia}, audioEnabled: ${call.audioEnabled}, connectionInfo: ${call.connectionInfo}")
       val callRh = call.remoteHostId
       when (val r = apiMsg.resp) {
         is WCallResponse.Capabilities -> withBGApi {

packages/simplex-chat-webrtc/src/call.ts

@@ -302,8 +302,7 @@ const processCommand = (function () {
       function resolveIceCandidates() {
         if (delay) clearTimeout(delay)
         resolved = true
-        // console.log("resolveIceCandidates", JSON.stringify(candidates))
-        console.log("resolveIceCandidates")
+        console.log("resolveIceCandidates", JSON.stringify(candidates))
         const iceCandidates = serialize(candidates)
         candidates = []
         resolve(iceCandidates)
@@ -311,8 +310,7 @@ const processCommand = (function () {
 
       function sendIceCandidates() {
         if (candidates.length === 0) return
-        // console.log("sendIceCandidates", JSON.stringify(candidates))
-        console.log("sendIceCandidates")
+        console.log("sendIceCandidates", JSON.stringify(candidates))
         const iceCandidates = serialize(candidates)
         candidates = []
         sendMessageToNative({resp: {type: "ice", iceCandidates}})
@@ -321,6 +319,7 @@ const processCommand = (function () {
   }
 
   async function initializeCall(config: CallConfig, mediaType: CallMediaType, aesKey?: string): Promise<Call> {
+    console.log("Initializing call")
     let pc: RTCPeerConnection
     try {
       pc = new RTCPeerConnection(config.peerConnectionConfig)
@@ -365,6 +364,7 @@ const processCommand = (function () {
     }
 
     async function connectionHandler() {
+      console.log(`Connection state change: ${pc.connectionState}, ${pc.iceConnectionState}, ${pc.iceGatheringState}, ${pc.signalingState}`)
       sendMessageToNative({
         resp: {
           type: "connection",
@@ -423,6 +423,7 @@ const processCommand = (function () {
 
   async function processCommand(body: WVAPICall): Promise<WVApiMessage> {
     const {corrId, command} = body
+    console.log(`process command: ${command.type}`)
     const pc = activeCall?.connection
     let resp: WCallResponse
     try {
@@ -432,6 +433,7 @@ const processCommand = (function () {
           if (activeCall) endCall()
           // This request for local media stream is made to prompt for camera/mic permissions on call start
           if (command.media) await getLocalMediaStream(command.media, VideoCamera.User)
+          console.log("Asked video camera permission")
           const encryption = supportsInsertableStreams(useWorker)
           resp = {type: "capabilities", capabilities: {encryption}}
           break
@@ -462,7 +464,7 @@ const processCommand = (function () {
             iceCandidates: await activeCall.iceCandidates,
             capabilities: {encryption},
           }
-          // console.log("offer response", JSON.stringify(resp))
+          console.log("offer response", JSON.stringify(resp))
           break
         }
         case "offer":
@@ -476,7 +478,7 @@ const processCommand = (function () {
             const {media, aesKey, iceServers, relay} = command
             activeCall = await initializeCall(getCallConfig(!!aesKey, iceServers, relay), media, aesKey)
             const pc = activeCall.connection
-            // console.log("offer remoteIceCandidates", JSON.stringify(remoteIceCandidates))
+            console.log("offer remoteIceCandidates", JSON.stringify(remoteIceCandidates))
             await pc.setRemoteDescription(new RTCSessionDescription(offer))
             const answer = await pc.createAnswer()
             await pc.setLocalDescription(answer)
@@ -488,7 +490,7 @@ const processCommand = (function () {
               iceCandidates: await activeCall.iceCandidates,
             }
           }
-          // console.log("answer response", JSON.stringify(resp))
+          console.log("answer response", JSON.stringify(resp))
           break
         case "answer":
           if (!pc) {
@@ -500,7 +502,7 @@ const processCommand = (function () {
           } else {
             const answer: RTCSessionDescriptionInit = parse(command.answer)
             const remoteIceCandidates: RTCIceCandidateInit[] = parse(command.iceCandidates)
-            // console.log("answer remoteIceCandidates", JSON.stringify(remoteIceCandidates))
+            console.log("answer remoteIceCandidates", JSON.stringify(remoteIceCandidates))
             await pc.setRemoteDescription(new RTCSessionDescription(answer))
             addIceCandidates(pc, remoteIceCandidates)
             resp = {type: "ok"}
@@ -572,11 +574,12 @@ const processCommand = (function () {
   function addIceCandidates(conn: RTCPeerConnection, iceCandidates: RTCIceCandidateInit[]) {
     for (const c of iceCandidates) {
       conn.addIceCandidate(new RTCIceCandidate(c))
-      // console.log("addIceCandidates", JSON.stringify(c))
+      console.log("addIceCandidates", JSON.stringify(c))
     }
   }
 
   async function setupMediaStreams(call: Call): Promise<void> {
+    console.log("Setup media streams start")
     const videos = getVideoElements()
     if (!videos) throw Error("no video elements")
     await setupEncryptionWorker(call)
@@ -590,6 +593,7 @@ const processCommand = (function () {
     // Without doing it manually Firefox shows black screen but video can be played in Picture-in-Picture
     videos.local.play()
     videos.remote.play()
+    console.log("Setup media streams end")
   }
 
   async function setupEncryptionWorker(call: Call) {
@@ -611,6 +615,7 @@ const processCommand = (function () {
     let {localStream} = call
 
     for (const track of localStream.getTracks()) {
+      console.log(`Adding local track: ${track.kind}`)
       pc.addTrack(track, localStream)
     }
 
@@ -633,6 +638,7 @@ const processCommand = (function () {
         }
         for (const stream of event.streams) {
           for (const track of stream.getTracks()) {
+            console.log(`Adding remote track: ${track.kind}`)
             call.remoteStream.addTrack(track)
           }
         }

packages/simplex-chat-webrtc/src/desktop/call.html

@@ -11,6 +11,7 @@
       id="remote-video-stream"
       class="inline"
       playsinline
+      autoplay
       poster="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAQAAAAnOwc2AAAAEUlEQVR42mNk+M+AARiHsiAAcCIKAYwFoQ8AAAAASUVORK5CYII="
       onclick="javascript:toggleRemoteVideoFitFill()"
     ></video>
@@ -19,6 +20,7 @@
       class="inline"
       muted
       playsinline
+      autoplay
       poster="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAQAAAAnOwc2AAAAEUlEQVR42mNk+M+AARiHsiAAcCIKAYwFoQ8AAAAASUVORK5CYII="
     ></video>
     <div id="progress"></div>

packages/simplex-chat-webrtc/src/desktop/ui.ts

@@ -8,7 +8,7 @@ const socket = new WebSocket(`ws://${location.host}`)
 socket.addEventListener("open", (_event) => {
   console.log("Opened socket")
   sendMessageToNative = (msg: WVApiMessage) => {
-    console.log("Message to server")
+    console.log(`Message to server will be sent: ${msg.command?.type}`)
     socket.send(JSON.stringify(msg))
   }
 })
@@ -17,7 +17,7 @@ socket.addEventListener("message", (event) => {
   const parsed = JSON.parse(event.data)
   reactOnMessageFromServer(parsed)
   processCommand(parsed)
-  console.log("Message from server")
+  console.log(`Message from server finished processing: ${parsed.command?.type}`)
 })
 
 socket.addEventListener("close", (_event) => {