android, desktop: calls switching from audio to video and back

2024-12-17 17:20:21 +01:00 · 2024-08-30 23:32:28 +09:00
parent 564b137f95
commit 4ca07338ea
11 changed files with 726 additions and 115 deletions
@@ -162,6 +162,16 @@ actual fun ActiveCallView() {
          is WCallResponse.Connected -> {
            updateActiveCall(call) { it.copy(callState = CallState.Connected, connectionInfo = r.connectionInfo) }
          }
+          is WCallResponse.PeerMedia -> {
+            updateActiveCall(call) {
+              val sources = it.peerMediaSources
+              when (r.source) {
+                CallMediaSource.Mic -> it.copy(peerMediaSources = sources.copy(mic = r.enabled))
+                CallMediaSource.Camera -> it.copy(peerMediaSources = sources.copy(camera = r.enabled))
+                CallMediaSource.Screen -> it.copy(peerMediaSources = sources.copy(screen = r.enabled))
+              }
+            }
+          }
          is WCallResponse.End -> {
            withBGApi { chatModel.callManager.endCall(call) }
          }
@@ -175,7 +185,7 @@ actual fun ActiveCallView() {
            is WCallCommand.Media -> {
              updateActiveCall(call) {
                when (cmd.media) {
-                  CallMediaType.Video -> it.copy(videoEnabled = cmd.enable)
+                  CallMediaType.Video -> it.copy(videoEnabled = cmd.enable, localMedia = if (cmd.enable) CallMediaType.Video else CallMediaType.Audio)
                  CallMediaType.Audio -> it.copy(audioEnabled = cmd.enable)
                }
              }
@@ -293,9 +303,9 @@ private fun ActiveCallOverlayLayout(
  flipCamera: () -> Unit
 ) {
  Column {
-    val media = call.peerMedia ?: call.localMedia
+    val supportsVideo = call.supportsVideo()
    CloseSheetBar({ chatModel.activeCallViewIsCollapsed.value = true }, true, tintColor = Color(0xFFFFFFD8)) {
-      if (media == CallMediaType.Video) {
+      if (supportsVideo) {
        Text(call.contact.chatViewName, Modifier.fillMaxWidth().padding(end = DEFAULT_PADDING), color = Color(0xFFFFFFD8), style = MaterialTheme.typography.h2, overflow = TextOverflow.Ellipsis, maxLines = 1)
      }
    }
@@ -327,29 +337,12 @@ private fun ActiveCallOverlayLayout(
        }
      }

-      when (media) {
-        CallMediaType.Video -> {
+      when (supportsVideo) {
+        true -> {
          VideoCallInfoView(call)
-          Box(Modifier.fillMaxWidth().fillMaxHeight().weight(1f), contentAlignment = Alignment.BottomCenter) {
-            DisabledBackgroundCallsButton()
-          }
-          Row(Modifier.fillMaxWidth().padding(horizontal = 6.dp), horizontalArrangement = Arrangement.SpaceBetween, verticalAlignment = Alignment.CenterVertically) {
-            ToggleAudioButton(call, enabled, toggleAudio)
-            SelectSoundDevice()
-            IconButton(onClick = dismiss, enabled = enabled) {
-              Icon(painterResource(MR.images.ic_call_end_filled), stringResource(MR.strings.icon_descr_hang_up), tint = if (enabled) Color.Red else MaterialTheme.colors.secondary, modifier = Modifier.size(64.dp))
-            }
-            if (call.videoEnabled) {
-              ControlButton(call, painterResource(MR.images.ic_flip_camera_android_filled), MR.strings.icon_descr_flip_camera, enabled, flipCamera)
-              ControlButton(call, painterResource(MR.images.ic_videocam_filled), MR.strings.icon_descr_video_off, enabled, toggleVideo)
-            } else {
-              Spacer(Modifier.size(48.dp))
-              ControlButton(call, painterResource(MR.images.ic_videocam_off), MR.strings.icon_descr_video_on, enabled, toggleVideo)
-            }
-          }
        }

-        CallMediaType.Audio -> {
+        false -> {
          Spacer(Modifier.fillMaxHeight().weight(1f))
          Column(
            Modifier.fillMaxWidth(),
@@ -359,24 +352,24 @@ private fun ActiveCallOverlayLayout(
            ProfileImage(size = 192.dp, image = call.contact.profile.image)
            AudioCallInfoView(call)
          }
-          Box(Modifier.fillMaxWidth().fillMaxHeight().weight(1f), contentAlignment = Alignment.BottomCenter) {
-            DisabledBackgroundCallsButton()
-          }
-          Box(Modifier.fillMaxWidth().padding(bottom = DEFAULT_BOTTOM_PADDING), contentAlignment = Alignment.CenterStart) {
-            Box(Modifier.fillMaxWidth(), contentAlignment = Alignment.Center) {
-              IconButton(onClick = dismiss, enabled = enabled) {
-                Icon(painterResource(MR.images.ic_call_end_filled), stringResource(MR.strings.icon_descr_hang_up), tint = if (enabled) Color.Red else MaterialTheme.colors.secondary, modifier = Modifier.size(64.dp))
-              }
-            }
-            Box(Modifier.padding(start = 32.dp)) {
-              ToggleAudioButton(call, enabled, toggleAudio)
-            }
-            Box(Modifier.fillMaxWidth(), contentAlignment = Alignment.CenterEnd) {
-              Box(Modifier.padding(end = 32.dp)) {
-                SelectSoundDevice()
-              }
-            }
-          }
+        }
+      }
+      Box(Modifier.fillMaxWidth().fillMaxHeight().weight(1f), contentAlignment = Alignment.BottomCenter) {
+        DisabledBackgroundCallsButton()
+      }
+
+      Row(Modifier.fillMaxWidth().padding(horizontal = 6.dp), horizontalArrangement = Arrangement.SpaceBetween, verticalAlignment = Alignment.CenterVertically) {
+        ToggleAudioButton(call, enabled, toggleAudio)
+        SelectSoundDevice()
+        IconButton(onClick = dismiss, enabled = enabled) {
+          Icon(painterResource(MR.images.ic_call_end_filled), stringResource(MR.strings.icon_descr_hang_up), tint = if (enabled) Color.Red else MaterialTheme.colors.secondary, modifier = Modifier.size(64.dp))
+        }
+        if (call.videoEnabled) {
+          ControlButton(call, painterResource(MR.images.ic_flip_camera_android_filled), MR.strings.icon_descr_flip_camera, enabled, flipCamera)
+          ControlButton(call, painterResource(MR.images.ic_videocam_filled), MR.strings.icon_descr_video_off, enabled, toggleVideo)
+        } else {
+          Spacer(Modifier.size(48.dp))
+          ControlButton(call, painterResource(MR.images.ic_videocam_off), MR.strings.icon_descr_video_on, enabled, toggleVideo)
        }
      }
    }
@@ -769,7 +762,7 @@ fun PreviewActiveCallOverlayVideo() {
        contact = Contact.sampleData,
        callState = CallState.Negotiated,
        localMedia = CallMediaType.Video,
-        peerMedia = CallMediaType.Video,
+        peerMediaSources = CallMediaSources(),
        callUUID = "",
        connectionInfo = ConnectionInfo(
          RTCIceCandidate(RTCIceCandidateType.Host, "tcp"),
@@ -799,7 +792,7 @@ fun PreviewActiveCallOverlayAudio() {
        contact = Contact.sampleData,
        callState = CallState.Negotiated,
        localMedia = CallMediaType.Audio,
-        peerMedia = CallMediaType.Audio,
+        peerMediaSources = CallMediaSources(),
        callUUID = "",
        connectionInfo = ConnectionInfo(
          RTCIceCandidate(RTCIceCandidateType.Host, "udp"),
@@ -54,7 +54,7 @@ actual fun ActiveCallInteractiveArea(call: Call) {
        .align(Alignment.BottomCenter),
      contentAlignment = Alignment.Center
    ) {
-      val media = call.peerMedia ?: call.localMedia
+      val media = call.peerMediaSources ?: call.localMedia
      if (media == CallMediaType.Video) {
        Icon(painterResource(MR.images.ic_videocam_filled), null, Modifier.size(27.dp).offset(x = 2.5.dp, y = 2.dp), tint = Color.White)
      } else {
@@ -2409,7 +2409,7 @@ object ChatController {
        // TODO askConfirmation?
        // TODO check encryption is compatible
        withCall(r, r.contact) { call ->
-          chatModel.activeCall.value = call.copy(callState = CallState.OfferReceived, peerMedia = r.callType.media, sharedKey = r.sharedKey)
+          chatModel.activeCall.value = call.copy(callState = CallState.OfferReceived, sharedKey = r.sharedKey)
          val useRelay = appPrefs.webrtcPolicyRelay.get()
          val iceServers = getIceServers()
          Log.d(TAG, ".callOffer iceServers $iceServers")
@@ -17,7 +17,7 @@ data class Call(
  val callState: CallState,
  val localMedia: CallMediaType,
  val localCapabilities: CallCapabilities? = null,
-  val peerMedia: CallMediaType? = null,
+  val peerMediaSources: CallMediaSources = CallMediaSources(),
  val sharedKey: String? = null,
  val audioEnabled: Boolean = true,
  val videoEnabled: Boolean = localMedia == CallMediaType.Video,
@@ -37,7 +37,7 @@ data class Call(

  val hasMedia: Boolean get() = callState == CallState.OfferSent || callState == CallState.Negotiated || callState == CallState.Connected

-  fun supportsVideo(): Boolean = peerMedia == CallMediaType.Video || localMedia == CallMediaType.Video
+  fun supportsVideo(): Boolean = peerMediaSources.hasVideo() || localMedia == CallMediaType.Video

 }

@@ -68,6 +68,14 @@ enum class CallState {
@Serializable data class WVAPICall(val corrId: Int? = null, val command: WCallCommand)
@Serializable data class WVAPIMessage(val corrId: Int? = null, val resp: WCallResponse, val command: WCallCommand? = null)

+@Serializable data class CallMediaSources(
+  val mic: Boolean = false,
+  val camera: Boolean = false,
+  val screen: Boolean = false
+) {
+  fun hasVideo() = camera || screen
+}
+
@Serializable
 sealed class WCallCommand {
  @Serializable @SerialName("capabilities") data class Capabilities(val media: CallMediaType): WCallCommand()
@@ -90,6 +98,7 @@ sealed class WCallResponse {
  @Serializable @SerialName("ice") data class Ice(val iceCandidates: String): WCallResponse()
  @Serializable @SerialName("connection") data class Connection(val state: ConnectionState): WCallResponse()
  @Serializable @SerialName("connected") data class Connected(val connectionInfo: ConnectionInfo): WCallResponse()
+  @Serializable @SerialName("peerMedia") data class PeerMedia(val media: CallMediaType, val source: CallMediaSource, val enabled: Boolean): WCallResponse()
  @Serializable @SerialName("end") object End: WCallResponse()
  @Serializable @SerialName("ended") object Ended: WCallResponse()
  @Serializable @SerialName("ok") object Ok: WCallResponse()
@@ -165,6 +174,13 @@ enum class CallMediaType {
  @SerialName("audio") Audio
 }

+@Serializable
+enum class CallMediaSource {
+  @SerialName("mic") Mic,
+  @SerialName("camera") Camera,
+  @SerialName("screen") Screen
+}
+
@Serializable
 enum class VideoCamera {
  @SerialName("user") User,
@@ -6,6 +6,13 @@ var CallMediaType;
    CallMediaType["Audio"] = "audio";
    CallMediaType["Video"] = "video";
 })(CallMediaType || (CallMediaType = {}));
+var CallMediaSource;
+(function (CallMediaSource) {
+    CallMediaSource["Mic"] = "mic";
+    CallMediaSource["Camera"] = "camera";
+    CallMediaSource["Screen"] = "screen";
+    CallMediaSource["Unknown"] = "unknown";
+})(CallMediaSource || (CallMediaSource = {}));
 var VideoCamera;
 (function (VideoCamera) {
    VideoCamera["User"] = "user";
@@ -131,6 +138,7 @@ const processCommand = (function () {
                .filter((elem) => elem.kind == "video")
                .forEach((elem) => (elem.enabled = false));
        }
+        // Will become video when any video tracks will be added
        const iceCandidates = getIceCandidates(pc, config);
        const call = {
            connection: pc,
@@ -139,9 +147,14 @@ const processCommand = (function () {
            localCamera,
            localStream,
            remoteStream,
+            peerMediaSources: {
+                mic: false,
+                camera: false,
+                screen: false,
+            },
            aesKey,
            screenShareEnabled: false,
-            cameraEnabled: true,
+            cameraEnabled: !isDesktop,
        };
        await setupMediaStreams(call);
        let connectionTimeout = setTimeout(connectionHandler, answerTimeout);
@@ -232,6 +245,13 @@ const processCommand = (function () {
                    const aesKey = encryption ? command.aesKey : undefined;
                    activeCall = await initializeCall(getCallConfig(encryption && !!aesKey, iceServers, relay), media, aesKey);
                    const pc = activeCall.connection;
+                    if (media == CallMediaType.Audio) {
+                        console.log("LALAL ADDING TRANSCEIVER for video");
+                        // For camera. So the first video in the list is for camera
+                        pc.addTransceiver("video", { streams: [activeCall.localStream] });
+                    }
+                    // For screenshare. So the second video in the list is for screenshare
+                    pc.addTransceiver("video", { streams: [activeCall.localStream] });
                    const offer = await pc.createOffer();
                    await pc.setLocalDescription(offer);
                    // for debugging, returning the command for callee to use
@@ -269,7 +289,11 @@ const processCommand = (function () {
                        const pc = activeCall.connection;
                        // console.log("offer remoteIceCandidates", JSON.stringify(remoteIceCandidates))
                        await pc.setRemoteDescription(new RTCSessionDescription(offer));
-                        const answer = await pc.createAnswer();
+                        pc.getTransceivers().forEach((elem) => (elem.direction = "sendrecv"));
+                        console.log("LALAL TRANSCE", pc.getTransceivers());
+                        let answer = await pc.createAnswer();
+                        console.log("LALAL SDP", answer, answer.sdp);
+                        // answer!.sdp = answer.sdp?.replace("a=recvonly", "a=sendrecv")
                        await pc.setLocalDescription(answer);
                        addIceCandidates(pc, remoteIceCandidates);
                        // same as command for caller to use
@@ -295,6 +319,7 @@ const processCommand = (function () {
                        const answer = parse(command.answer);
                        const remoteIceCandidates = parse(command.iceCandidates);
                        // console.log("answer remoteIceCandidates", JSON.stringify(remoteIceCandidates))
+                        console.log("LALAL SDP2", answer, answer.sdp);
                        await pc.setRemoteDescription(new RTCSessionDescription(answer));
                        addIceCandidates(pc, remoteIceCandidates);
                        resp = { type: "ok" };
@@ -314,8 +339,9 @@ const processCommand = (function () {
                    if (!activeCall) {
                        resp = { type: "error", message: "media: call not started" };
                    }
-                    else if (activeCall.localMedia == CallMediaType.Audio && command.media == CallMediaType.Video) {
-                        resp = { type: "error", message: "media: no video" };
+                    else if (activeCall.localMedia == CallMediaType.Audio && command.media == CallMediaType.Video && command.enable) {
+                        await startSendingVideo(activeCall, activeCall.localCamera);
+                        resp = { type: "ok" };
                    }
                    else {
                        enableMedia(activeCall.localStream, command.media, command.enable);
@@ -399,6 +425,12 @@ const processCommand = (function () {
                call.worker = new Worker(URL.createObjectURL(new Blob([workerCode], { type: "text/javascript" })));
                call.worker.onerror = ({ error, filename, lineno, message }) => console.log({ error, filename, lineno, message });
                // call.worker.onmessage = ({data}) => console.log(JSON.stringify({message: data}))
+                call.worker.onmessage = ({ data }) => {
+                    console.log(JSON.stringify({ message: data }));
+                    const transceiverMid = data.transceiverMid;
+                    const mute = data.mute;
+                    onMediaMuteUnmute(transceiverMid, mute);
+                };
            }
        }
    }
@@ -413,8 +445,9 @@ const processCommand = (function () {
        }
        if (call.aesKey && call.key) {
            console.log("set up encryption for sending");
-            for (const sender of pc.getSenders()) {
-                setupPeerTransform(TransformOperation.Encrypt, sender, call.worker, call.aesKey, call.key);
+            for (const transceiver of pc.getTransceivers()) {
+                const sender = transceiver.sender;
+                setupPeerTransform(TransformOperation.Encrypt, sender, call.worker, call.aesKey, call.key, transceiver.sender.track.kind == "video" ? CallMediaType.Video : CallMediaType.Audio, transceiver.mid);
            }
        }
    }
@@ -422,16 +455,49 @@ const processCommand = (function () {
        // Pull tracks from remote stream as they arrive add them to remoteStream video
        const pc = call.connection;
        pc.ontrack = (event) => {
+            console.log("LALAL ON TRACK ", event);
            try {
                if (call.aesKey && call.key) {
                    console.log("set up decryption for receiving");
-                    setupPeerTransform(TransformOperation.Decrypt, event.receiver, call.worker, call.aesKey, call.key);
+                    setupPeerTransform(TransformOperation.Decrypt, event.receiver, call.worker, call.aesKey, call.key, event.receiver.track.kind == "video" ? CallMediaType.Video : CallMediaType.Audio, event.transceiver.mid);
                }
-                for (const stream of event.streams) {
-                    for (const track of stream.getTracks()) {
-                        call.remoteStream.addTrack(track);
+                // const source = mediaSourceFromTransceiverMid(event.transceiver.mid)
+                // const sources = call.peerMediaSources
+                // if (source == CallMediaSource.Mic) {
+                //   sources.mic = true
+                // } else if (source == CallMediaSource.Camera) {
+                //   sources.camera = true
+                // } else if (source == CallMediaSource.Screen) {
+                //   sources.screen = true
+                // }
+                // call.peerMediaSources = sources
+                if (event.streams.length > 0) {
+                    for (const stream of event.streams) {
+                        for (const track of stream.getTracks()) {
+                            call.remoteStream.addTrack(track);
+                            // const resp: WRPeerMedia = {
+                            //   type: "peerMedia",
+                            //   media: track.kind == "audio" ? CallMediaType.Audio : CallMediaType.Video,
+                            //   source: source,
+                            //   enabled: track.enabled,
+                            // }
+                            // console.log("LALAL ADDED REMOTE", track, track.kind)
+                            // sendMessageToNative({resp: resp})
+                        }
                    }
                }
+                else {
+                    const track = event.track;
+                    call.remoteStream.addTrack(track);
+                    // const resp: WRPeerMedia = {
+                    //   type: "peerMedia",
+                    //   media: track.kind == "audio" ? CallMediaType.Audio : CallMediaType.Video,
+                    //   source: source,
+                    //   enabled: track.enabled,
+                    // }
+                    // console.log("LALAL ADDED REMOTE", track, track.kind)
+                    // sendMessageToNative({resp: resp})
+                }
                console.log(`ontrack success`);
            }
            catch (e) {
@@ -479,6 +545,42 @@ const processCommand = (function () {
            }
        }
    }
+    async function startSendingVideo(call, camera) {
+        console.log("LALAL STARTING SENDING VIDEO");
+        const videos = getVideoElements();
+        if (!videos)
+            throw Error("no video elements");
+        const pc = call.connection;
+        // Taking the first video transceiver and use it for sending video from camera. Following tracks are for other purposes
+        const tc = pc.getTransceivers().find((tc) => tc.receiver.track.kind == "video" && tc.direction == "sendrecv");
+        console.log(pc.getTransceivers().map((elem) => { var _a, _b; return "" + ((_a = elem.sender.track) === null || _a === void 0 ? void 0 : _a.kind) + " " + ((_b = elem.receiver.track) === null || _b === void 0 ? void 0 : _b.kind) + " " + elem.direction; }));
+        let localStream;
+        try {
+            localStream = await getLocalMediaStream(CallMediaType.Video, camera);
+            for (const t of localStream.getVideoTracks()) {
+                console.log("LALAL TC", tc, pc.getTransceivers());
+                call.localStream.addTrack(t);
+                tc === null || tc === void 0 ? void 0 : tc.sender.replaceTrack(t);
+                localStream.removeTrack(t);
+                // when adding track a `sender` will be created on that track automatically
+                //pc.addTrack(t, call.localStream)
+                console.log("LALAL ADDED VIDEO TRACK " + t);
+            }
+            call.localMedia = CallMediaType.Video;
+            call.cameraEnabled = true;
+        }
+        catch (e) {
+            return;
+        }
+        const sender = tc === null || tc === void 0 ? void 0 : tc.sender;
+        console.log("LALAL SENDER " + sender + " " + (sender === null || sender === void 0 ? void 0 : sender.getParameters()));
+        if (call.aesKey && call.key && sender) {
+            setupPeerTransform(TransformOperation.Encrypt, sender, call.worker, call.aesKey, call.key, CallMediaType.Video, tc.mid);
+        }
+        // Without doing it manually Firefox shows black screen but video can be played in Picture-in-Picture
+        videos.local.play();
+        console.log("LALAL SENDING VIDEO");
+    }
    async function replaceMedia(call, camera) {
        const videos = getVideoElements();
        if (!videos)
@@ -535,20 +637,39 @@ const processCommand = (function () {
            for (const t of tracks)
                sender.replaceTrack(t);
    }
-    function setupPeerTransform(operation, peer, worker, aesKey, key) {
+    function mediaSourceFromTransceiverMid(mid) {
+        switch (mid) {
+            case "0":
+                return CallMediaSource.Mic;
+            case "1":
+                return CallMediaSource.Camera;
+            case "2":
+                return CallMediaSource.Screen;
+            default:
+                return CallMediaSource.Unknown;
+        }
+    }
+    function setupPeerTransform(operation, peer, worker, aesKey, key, media, transceiverMid) {
+        console.log("LALAL MEDIA " + media + " " + transceiverMid);
        if (worker && "RTCRtpScriptTransform" in window) {
            console.log(`${operation} with worker & RTCRtpScriptTransform`);
-            peer.transform = new RTCRtpScriptTransform(worker, { operation, aesKey });
+            peer.transform = new RTCRtpScriptTransform(worker, { operation, aesKey, media, transceiverMid });
        }
        else if ("createEncodedStreams" in peer) {
            const { readable, writable } = peer.createEncodedStreams();
            if (worker) {
                console.log(`${operation} with worker`);
-                worker.postMessage({ operation, readable, writable, aesKey }, [readable, writable]);
+                worker.postMessage({ operation, readable, writable, aesKey, media, transceiverMid }, [
+                    readable,
+                    writable,
+                ]);
            }
            else {
                console.log(`${operation} without worker`);
-                const transform = callCrypto.transformFrame[operation](key);
+                const onMediaMuteUnmuteConst = (mute) => {
+                    onMediaMuteUnmute(transceiverMid, mute);
+                };
+                const transform = callCrypto.transformFrame[operation](key, onMediaMuteUnmuteConst);
                readable.pipeThrough(new TransformStream({ transform })).pipeTo(writable);
            }
        }
@@ -556,6 +677,46 @@ const processCommand = (function () {
            console.log(`no ${operation}`);
        }
    }
+    function onMediaMuteUnmute(transceiverMid, mute) {
+        if (activeCall) {
+            const source = mediaSourceFromTransceiverMid(transceiverMid);
+            console.log("LALAL ON MUTE/UNMUTE", mute, source, transceiverMid);
+            const sources = activeCall.peerMediaSources;
+            if (source == CallMediaSource.Mic && activeCall.peerMediaSources.mic == mute) {
+                const resp = {
+                    type: "peerMedia",
+                    media: CallMediaType.Audio,
+                    source: source,
+                    enabled: !mute,
+                };
+                sources.mic = !mute;
+                activeCall.peerMediaSources = sources;
+                sendMessageToNative({ resp: resp });
+            }
+            else if (source == CallMediaSource.Camera && activeCall.peerMediaSources.camera == mute) {
+                const resp = {
+                    type: "peerMedia",
+                    media: CallMediaType.Video,
+                    source: source,
+                    enabled: !mute,
+                };
+                sources.camera = !mute;
+                activeCall.peerMediaSources = sources;
+                sendMessageToNative({ resp: resp });
+            }
+            else if (source == CallMediaSource.Screen && activeCall.peerMediaSources.screen == mute) {
+                const resp = {
+                    type: "peerMedia",
+                    media: CallMediaType.Video,
+                    source: source,
+                    enabled: !mute,
+                };
+                sources.screen = !mute;
+                activeCall.peerMediaSources = sources;
+                sendMessageToNative({ resp: resp });
+            }
+        }
+    }
    function getLocalMediaStream(mediaType, facingMode) {
        const constraints = callMediaConstraints(mediaType, facingMode);
        return navigator.mediaDevices.getUserMedia(constraints);
@@ -703,6 +864,7 @@ function callCryptoFunction() {
                    : new Uint8Array(0);
                frame.data = concatN(initial, ciphertext, iv).buffer;
                controller.enqueue(frame);
+                // console.log("LALAL ENCRYPT", frame.data.byteLength)
            }
            catch (e) {
                console.log(`encryption error ${e}`);
@@ -710,7 +872,9 @@ function callCryptoFunction() {
            }
        };
    }
-    function decryptFrame(key) {
+    function decryptFrame(key, onMediaMuteUnmute) {
+        let wasMuted = true;
+        let lastBytes = [];
        return async (frame, controller) => {
            const data = new Uint8Array(frame.data);
            const n = initialPlainTextRequired[frame.type] || 1;
@@ -723,6 +887,23 @@ function callCryptoFunction() {
                    : new Uint8Array(0);
                frame.data = concatN(initial, plaintext).buffer;
                controller.enqueue(frame);
+                lastBytes.push(frame.data.byteLength);
+                const sliced = lastBytes.slice(-20, lastBytes.length);
+                const average = sliced.reduce((prev, value) => value + prev, 0) / Math.max(1, sliced.length);
+                if (lastBytes.length > 20) {
+                    console.log("LALAL REPLACED", lastBytes.length, sliced.length);
+                    lastBytes = sliced;
+                }
+                console.log("LALAL DECRYPT", frame.type, frame.data.byteLength, average);
+                // frame.type is undefined for audio stream, but defined for video
+                if (frame.type && wasMuted && average > 200) {
+                    wasMuted = false;
+                    onMediaMuteUnmute(false);
+                }
+                else if (frame.type && !wasMuted && average < 200) {
+                    wasMuted = true;
+                    onMediaMuteUnmute(true);
+                }
            }
            catch (e) {
                console.log(`decryption error ${e}`);
@@ -828,9 +1009,9 @@ function workerFunction() {
    if ("RTCTransformEvent" in self) {
        self.addEventListener("rtctransform", async ({ transformer }) => {
            try {
-                const { operation, aesKey } = transformer.options;
+                const { operation, aesKey, transceiverMid } = transformer.options;
                const { readable, writable } = transformer;
-                await setupTransform({ operation, aesKey, readable, writable });
+                await setupTransform({ operation, aesKey, transceiverMid, readable, writable });
                self.postMessage({ result: "setupTransform success" });
            }
            catch (e) {
@@ -838,9 +1019,12 @@ function workerFunction() {
            }
        });
    }
-    async function setupTransform({ operation, aesKey, readable, writable }) {
+    async function setupTransform({ operation, aesKey, transceiverMid, readable, writable }) {
        const key = await callCrypto.decodeAesKey(aesKey);
-        const transform = callCrypto.transformFrame[operation](key);
+        const onMediaMuteUnmute = (mute) => {
+            self.postMessage({ transceiverMid: transceiverMid, mute: mute });
+        };
+        const transform = callCrypto.transformFrame[operation](key, onMediaMuteUnmute);
        readable.pipeThrough(new TransformStream({ transform })).pipeTo(writable);
    }
 }
@@ -9,6 +9,7 @@ socket.addEventListener("open", (_event) => {
    sendMessageToNative = (msg) => {
        console.log("Message to server");
        socket.send(JSON.stringify(msg));
+        reactOnMessageToServer(msg);
    };
 });
 socket.addEventListener("message", (event) => {
@@ -43,17 +44,19 @@ function toggleSpeakerManually() {
 }
 function toggleVideoManually() {
    if (activeCall === null || activeCall === void 0 ? void 0 : activeCall.localMedia) {
-        let res;
        if (activeCall === null || activeCall === void 0 ? void 0 : activeCall.screenShareEnabled) {
            activeCall.cameraEnabled = !activeCall.cameraEnabled;
-            res = activeCall.cameraEnabled;
+            enableVideoIcon(activeCall.cameraEnabled);
+            // } else if (activeCall.localMedia == CallMediaType.Video) {
+            //   enableVideoIcon(toggleMedia(activeCall.localStream, CallMediaType.Video))
        }
        else {
-            res = toggleMedia(activeCall.localStream, CallMediaType.Video);
+            const apiCall = { command: { type: "media", media: CallMediaType.Video, enable: activeCall.cameraEnabled != true } };
+            reactOnMessageFromServer(apiCall);
+            processCommand(apiCall).then(() => {
+                enableVideoIcon((activeCall === null || activeCall === void 0 ? void 0 : activeCall.cameraEnabled) == true);
+            });
        }
-        document.getElementById("toggle-video").innerHTML = res
-            ? '<img src="/desktop/images/ic_videocam_filled.svg" />'
-            : '<img src="/desktop/images/ic_videocam_off.svg" />';
    }
 }
 async function toggleScreenManually() {
@@ -65,6 +68,11 @@ async function toggleScreenManually() {
            : '<img src="/desktop/images/ic_screen_share.svg" />';
    }
 }
+function enableVideoIcon(enabled) {
+    document.getElementById("toggle-video").innerHTML = enabled
+        ? '<img src="/desktop/images/ic_videocam_filled.svg" />'
+        : '<img src="/desktop/images/ic_videocam_off.svg" />';
+}
 function reactOnMessageFromServer(msg) {
    var _a;
    switch ((_a = msg.command) === null || _a === void 0 ? void 0 : _a.type) {
@@ -75,23 +83,41 @@ function reactOnMessageFromServer(msg) {
        case "start":
            document.getElementById("toggle-audio").style.display = "inline-block";
            document.getElementById("toggle-speaker").style.display = "inline-block";
-            if (msg.command.media == CallMediaType.Video) {
-                document.getElementById("toggle-video").style.display = "inline-block";
-                document.getElementById("toggle-screen").style.display = "inline-block";
-            }
+            document.getElementById("toggle-video").style.display = "inline-block";
+            document.getElementById("toggle-screen").style.display = "inline-block";
            document.getElementById("info-block").className = msg.command.media;
            break;
+        case "media":
+            const className = (msg.command.media == CallMediaType.Video && msg.command.enable) ||
+                (activeCall === null || activeCall === void 0 ? void 0 : activeCall.peerMediaSources.camera) ||
+                (activeCall === null || activeCall === void 0 ? void 0 : activeCall.peerMediaSources.screen)
+                ? "video"
+                : "audio";
+            document.getElementById("info-block").className = className;
+            document.getElementById("audio-call-icon").style.display = className == CallMediaType.Audio ? "block" : "none";
+            break;
        case "description":
            updateCallInfoView(msg.command.state, msg.command.description);
            if ((activeCall === null || activeCall === void 0 ? void 0 : activeCall.connection.connectionState) == "connected") {
                document.getElementById("progress").style.display = "none";
-                if (document.getElementById("info-block").className == CallMediaType.Audio) {
-                    document.getElementById("audio-call-icon").style.display = "block";
-                }
+                document.getElementById("audio-call-icon").style.display =
+                    document.getElementById("info-block").className == CallMediaType.Audio ? "block" : "none";
            }
            break;
    }
 }
+function reactOnMessageToServer(msg) {
+    var _a;
+    switch ((_a = msg.resp) === null || _a === void 0 ? void 0 : _a.type) {
+        case "peerMedia":
+            const className = (activeCall === null || activeCall === void 0 ? void 0 : activeCall.localMedia) == CallMediaType.Video || (activeCall === null || activeCall === void 0 ? void 0 : activeCall.peerMediaSources.camera) || (activeCall === null || activeCall === void 0 ? void 0 : activeCall.peerMediaSources.screen)
+                ? "video"
+                : "audio";
+            document.getElementById("info-block").className = className;
+            document.getElementById("audio-call-icon").style.display = className == CallMediaType.Audio ? "block" : "none";
+            break;
+    }
+}
 function updateCallInfoView(state, description) {
    document.getElementById("state").innerText = state;
    document.getElementById("description").innerText = description;
@@ -65,6 +65,14 @@ actual fun ActiveCallView() {
        is WCallResponse.Connected -> {
          chatModel.activeCall.value = call.copy(callState = CallState.Connected, connectionInfo = r.connectionInfo)
        }
+        is WCallResponse.PeerMedia -> {
+          val sources = call.peerMediaSources
+          chatModel.activeCall.value = when (r.source) {
+            CallMediaSource.Mic -> call.copy(peerMediaSources = sources.copy(mic = r.enabled))
+            CallMediaSource.Camera -> call.copy(peerMediaSources = sources.copy(camera = r.enabled))
+            CallMediaSource.Screen -> call.copy(peerMediaSources = sources.copy(screen = r.enabled))
+          }
+        }
        is WCallResponse.End -> {
          withBGApi { chatModel.callManager.endCall(call) }
        }
@@ -24,7 +24,7 @@ import kotlinx.coroutines.flow.MutableStateFlow
@Composable
 actual fun ActiveCallInteractiveArea(call: Call) {
  val showMenu = remember { mutableStateOf(false) }
-  val media = call.peerMedia ?: call.localMedia
+  val media = call.peerMediaSources ?: call.localMedia
  CompositionLocalProvider(
    LocalIndication provides NoIndication
  ) {