android, desktop: calls switching from audio to video and back (#4814)

* android, desktop: calls switching from audio to video and back * refactor * working all 4 streams with mute handling differently * changes * changes * wrong file * changes * padding * android camera service type * icons, sizes, clickable * refactor * Revert "android camera service type" This reverts commit 9878ff38e9. * late init camera permissions * enabling camera sooner than call establishes (not fully done) * changes * alpha * fixes for Safari * enhancements * fix Safari sound * padding between buttons on desktop * android default values for padding * changes * calls without encryption are supported and flipping camera on some devices works * unused param * logs * background color * play local video in Safari * no line height * removed one listener from per frame processing * enhancements --------- Co-authored-by: Evgeny Poberezkin <evgeny@poberezkin.com>
2024-12-17 17:20:21 +01:00 · 2024-09-27 02:18:05 +07:00
parent 4a39b481b1
commit 95c1d8d798
30 changed files with 2319 additions and 538 deletions
@@ -6,6 +6,15 @@
    <script src="../lz-string.min.js"></script>
  </head>
  <body>
+    <video
+      id="remote-screen-video-stream"
+      class="inline"
+      playsinline
+      poster="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAQAAAAnOwc2AAAAEUlEQVR42mNk+M+AARiHsiAAcCIKAYwFoQ8AAAAASUVORK5CYII="
+      style="visibility: hidden"
+      onclick="javascript:toggleRemoteScreenVideoFitFill()"
+    ></video>
+
    <video
      id="remote-video-stream"
      class="inline"
@@ -14,6 +23,7 @@
      poster="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAQAAAAnOwc2AAAAEUlEQVR42mNk+M+AARiHsiAAcCIKAYwFoQ8AAAAASUVORK5CYII="
      onclick="javascript:toggleRemoteVideoFitFill()"
    ></video>
+
    <video
      id="local-video-stream"
      class="inline"
@@ -22,6 +32,15 @@
      playsinline
      poster="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAQAAAAnOwc2AAAAEUlEQVR42mNk+M+AARiHsiAAcCIKAYwFoQ8AAAAASUVORK5CYII="
    ></video>
+
+    <video
+      id="local-screen-video-stream"
+      class="inline"
+      muted
+      playsinline
+      poster="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAQAAAAnOwc2AAAAEUlEQVR42mNk+M+AARiHsiAAcCIKAYwFoQ8AAAAASUVORK5CYII="
+      style="visibility: hidden"
+    ></video>
  </body>
  <footer>
    <script src="../call.js"></script>
@@ -12,6 +12,35 @@ body {
  object-fit: cover;
 }

+#remote-video-stream.collapsed {
+  position: absolute;
+  max-width: 30%;
+  max-height: 30%;
+  object-fit: cover;
+  margin: 16px;
+  border-radius: 16px;
+  bottom: 80px;
+  right: 0;
+}
+
+#remote-video-stream.collapsed-pip {
+  position: absolute;
+  max-width: 50%;
+  max-height: 50%;
+  object-fit: cover;
+  margin: 8px;
+  border-radius: 8px;
+  bottom: 0;
+  right: 0;
+}
+
+#remote-screen-video-stream.inline {
+  position: absolute;
+  width: 100%;
+  height: 100%;
+  object-fit: cover;
+}
+
 #local-video-stream.inline {
  position: absolute;
  width: 30%;
@@ -23,6 +52,17 @@ body {
  right: 0;
 }

+#local-screen-video-stream.inline {
+  position: absolute;
+  width: 30%;
+  max-width: 30%;
+  object-fit: cover;
+  margin: 16px;
+  border-radius: 16px;
+  top: 30%;
+  right: 0;
+}
+
 #remote-video-stream.fullscreen {
  position: absolute;
  height: 100%;
@@ -30,6 +70,13 @@ body {
  object-fit: cover;
 }

+#remote-screen-video-stream.fullscreen {
+  position: absolute;
+  height: 100%;
+  width: 100%;
+  object-fit: cover;
+}
+
 #local-video-stream.fullscreen {
  position: absolute;
  height: 100%;
@@ -37,6 +84,13 @@ body {
  object-fit: cover;
 }

+#local-screen-video-stream.fullscreen {
+  position: absolute;
+  height: 100%;
+  width: 100%;
+  object-fit: cover;
+}
+
 *::-webkit-media-controls {
  display: none !important;
  -webkit-appearance: none !important;
@@ -7,6 +7,15 @@
    <script src="/lz-string.min.js"></script>
  </head>
  <body>
+    <video
+      id="remote-screen-video-stream"
+      class="inline"
+      playsinline
+      poster="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAQAAAAnOwc2AAAAEUlEQVR42mNk+M+AARiHsiAAcCIKAYwFoQ8AAAAASUVORK5CYII="
+      style="visibility: hidden"
+      onclick="javascript:toggleRemoteScreenVideoFitFill()"
+    ></video>
+
    <video
      id="remote-video-stream"
      class="inline"
@@ -14,6 +23,7 @@
      poster="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAQAAAAnOwc2AAAAEUlEQVR42mNk+M+AARiHsiAAcCIKAYwFoQ8AAAAASUVORK5CYII="
      onclick="javascript:toggleRemoteVideoFitFill()"
    ></video>
+
    <video
      id="local-video-stream"
      class="inline"
@@ -21,28 +31,39 @@
      playsinline
      poster="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAQAAAAnOwc2AAAAEUlEQVR42mNk+M+AARiHsiAAcCIKAYwFoQ8AAAAASUVORK5CYII="
    ></video>
+
+    <video
+      id="local-screen-video-stream"
+      class="inline"
+      muted
+      playsinline
+      poster="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAQAAAAnOwc2AAAAEUlEQVR42mNk+M+AARiHsiAAcCIKAYwFoQ8AAAAASUVORK5CYII="
+      style="visibility: hidden"
+    ></video>
+
    <div id="progress"></div>
    <div id="info-block">
      <p id="state"></p>
      <p id="description"></p>
+      <b id="media-sources" style="color: #fff"></b>
    </div>
    <div id="audio-call-icon">
      <img src="/desktop/images/ic_phone_in_talk.svg" />
    </div>
    <p id="manage-call">
-      <button id="toggle-screen" style="display: none" onclick="javascript:toggleScreenManually()">
+      <button id="toggle-screen" onclick="javascript:toggleScreenManually()">
        <img src="/desktop/images/ic_screen_share.svg" />
      </button>
-      <button id="toggle-audio" style="display: none" onclick="javascript:toggleAudioManually()">
+      <button id="toggle-mic" onclick="javascript:toggleMicManually()">
        <img src="/desktop/images/ic_mic.svg" />
      </button>
-      <button id="end-call" onclick="javascript:endCallManually()">
+      <button id="end-call" style="background: red" onclick="javascript:endCallManually()">
        <img src="/desktop/images/ic_call_end_filled.svg" />
      </button>
-      <button id="toggle-speaker" style="display: none" onclick="javascript:toggleSpeakerManually()">
+      <button id="toggle-speaker" onclick="javascript:toggleSpeakerManually()">
        <img src="/desktop/images/ic_volume_up.svg" />
      </button>
-      <button id="toggle-video" style="display: none" onclick="javascript:toggleVideoManually()">
+      <button id="toggle-camera" onclick="javascript:toggleCameraManually()">
        <img src="/desktop/images/ic_videocam_off.svg" />
      </button>
    </p>
@@ -1 +1 @@
-<svg xmlns="http://www.w3.org/2000/svg" height="44" viewBox="0 96 960 960" width="44"><path fill="red" d="M480 418q125 0 238.75 50.25T914 613.5q8 9.5 8.25 21t-8.25 20L821 748q-8 8-22.5 8.75t-23-5.75l-113-84.5q-6-4.5-8.75-10.25T651 643.5v-139q-42-16-85.5-22.5t-85-6.5q-42 0-85.5 6.5t-85.5 22.5v139q0 6.5-2.75 12.5T298 666.5L184.5 751q-11.5 8.5-23.5 7.5T139.5 748L46 654.5q-8.5-8.5-8.25-20t8.25-21q81.5-95 195.25-145.25T480 418Z"/></svg>
+<svg xmlns="http://www.w3.org/2000/svg" height="44" viewBox="0 96 960 960" width="44"><path fill="white" d="M480 418q125 0 238.75 50.25T914 613.5q8 9.5 8.25 21t-8.25 20L821 748q-8 8-22.5 8.75t-23-5.75l-113-84.5q-6-4.5-8.75-10.25T651 643.5v-139q-42-16-85.5-22.5t-85-6.5q-42 0-85.5 6.5t-85.5 22.5v139q0 6.5-2.75 12.5T298 666.5L184.5 751q-11.5 8.5-23.5 7.5T139.5 748L46 654.5q-8.5-8.5-8.25-20t8.25-21q81.5-95 195.25-145.25T480 418Z"/></svg>
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="44" viewBox="0 -960 960 960" width="44" fill="white"><path d="m809.5-61.5-133-133q-27 19-58.25 33.25T553.5-139.5V-199q21.5-6.5 42.5-14.5t39.5-22L476-396v229L280-363H122.5v-234H274L54.5-816.5 96-858l755 754-41.5 42.5ZM770-291l-41.5-41.5q20-33 29.75-70.67Q768-440.85 768-481q0-100.82-58.75-180.41T553.5-763v-59.5q120 28 196 123.25t76 218.25q0 50.5-14 98.75T770-291ZM642.5-418.5l-89-89v-132q46.5 21.5 73.75 64.75T654.5-480q0 16-3 31.5t-9 30ZM476-585 372-689l104-104v208Zm-57.5 278v-145.5l-87-87H180v119h124.5l114 113.5ZM375-496Z"/></svg>
@@ -12,7 +12,37 @@ body {
  object-fit: cover;
 }

+#remote-video-stream.collapsed {
+  position: absolute;
+  max-width: 20%;
+  max-height: 20%;
+  object-fit: cover;
+  margin: 16px;
+  border-radius: 16px;
+  bottom: 80px;
+  right: 0;
+}
+
+#remote-video-stream.collapsed-pip {
+  position: absolute;
+  max-width: 30%;
+  max-height: 30%;
+  object-fit: cover;
+  margin: 8px;
+  border-radius: 8px;
+  bottom: 0;
+  right: 0;
+}
+
+#remote-screen-video-stream.inline {
+  position: absolute;
+  width: 100%;
+  height: 100%;
+  object-fit: cover;
+}
+
 #local-video-stream.inline {
+  background-color: black;
  position: absolute;
  width: 20%;
  max-width: 20%;
@@ -23,6 +53,17 @@ body {
  right: 0;
 }

+#local-screen-video-stream.inline {
+  position: absolute;
+  width: 20%;
+  max-width: 20%;
+  object-fit: cover;
+  margin: 16px;
+  border-radius: 16px;
+  top: 33%;
+  right: 0;
+}
+
 #remote-video-stream.fullscreen {
  position: absolute;
  height: 100%;
@@ -30,6 +71,13 @@ body {
  object-fit: cover;
 }

+#remote-screen-video-stream.fullscreen {
+  position: absolute;
+  height: 100%;
+  width: 100%;
+  object-fit: cover;
+}
+
 #local-video-stream.fullscreen {
  position: absolute;
  height: 100%;
@@ -37,6 +85,13 @@ body {
  object-fit: cover;
 }

+#local-screen-video-stream.fullscreen {
+  position: absolute;
+  height: 100%;
+  width: 100%;
+  object-fit: cover;
+}
+
 *::-webkit-media-controls {
  display: none !important;
  -webkit-appearance: none !important;
@@ -57,19 +112,32 @@ body {
 #manage-call {
  position: absolute;
  width: fit-content;
-  top: 90%;
+  bottom: 15px;
  left: 50%;
  transform: translate(-50%, 0);
  display: grid;
  grid-auto-flow: column;
-  grid-column-gap: 30px;
+  grid-column-gap: 38px;
 }

 #manage-call button {
  border: none;
  cursor: pointer;
  appearance: none;
-  background-color: inherit;
+  background-color: #ffffff33;
+  border-radius: 50%;
+  padding: 13px;
+  width: 61px;
+  height: 61px;
+}
+
+#manage-call img {
+  width: 35px;
+  height: 35px;
+}
+
+#manage-call button .video {
+  background: #00000033;
 }

 #progress {
@@ -110,7 +178,6 @@ body {
 #info-block {
  position: absolute;
  color: white;
-  line-height: 10px;
  opacity: 0.8;
  width: 200px;
  font-family: Arial, Helvetica, sans-serif;
@@ -10,6 +10,7 @@ socket.addEventListener("open", (_event) => {
  sendMessageToNative = (msg: WVApiMessage) => {
    console.log("Message to server")
    socket.send(JSON.stringify(msg))
+    reactOnMessageToServer(msg)
  }
 })

@@ -32,74 +33,165 @@ function endCallManually() {
  sendMessageToNative({resp: {type: "end"}})
 }

-function toggleAudioManually() {
-  if (activeCall?.localMedia) {
-    document.getElementById("toggle-audio")!!.innerHTML = toggleMedia(activeCall.localStream, CallMediaType.Audio)
-      ? '<img src="/desktop/images/ic_mic.svg" />'
-      : '<img src="/desktop/images/ic_mic_off.svg" />'
+function toggleMicManually() {
+  const enable = activeCall ? !activeCall.localMediaSources.mic : !inactiveCallMediaSources.mic
+  const apiCall: WVAPICall = {
+    command: {type: "media", source: CallMediaSource.Mic, enable: enable},
  }
+  processCommand(apiCall)
 }

 function toggleSpeakerManually() {
-  if (activeCall?.remoteStream) {
-    document.getElementById("toggle-speaker")!!.innerHTML = toggleMedia(activeCall.remoteStream, CallMediaType.Audio)
-      ? '<img src="/desktop/images/ic_volume_up.svg" />'
-      : '<img src="/desktop/images/ic_volume_down.svg" />'
+  if (activeCall?.remoteStream && activeCall.peerMediaSources.mic) {
+    enableSpeakerIcon(togglePeerMedia(activeCall.remoteStream, CallMediaType.Audio), !activeCall.peerMediaSources.mic)
  }
 }

-function toggleVideoManually() {
-  if (activeCall?.localMedia) {
-    let res: boolean
-    if (activeCall?.screenShareEnabled) {
-      activeCall.cameraEnabled = !activeCall.cameraEnabled
-      res = activeCall.cameraEnabled
-    } else {
-      res = toggleMedia(activeCall.localStream, CallMediaType.Video)
-    }
-    document.getElementById("toggle-video")!!.innerHTML = res
-      ? '<img src="/desktop/images/ic_videocam_filled.svg" />'
-      : '<img src="/desktop/images/ic_videocam_off.svg" />'
+function toggleCameraManually() {
+  const enable = activeCall ? !activeCall.localMediaSources.camera : !inactiveCallMediaSources.camera
+  const apiCall: WVAPICall = {
+    command: {type: "media", source: CallMediaSource.Camera, enable: enable},
  }
+  processCommand(apiCall)
 }

 async function toggleScreenManually() {
-  const was = activeCall?.screenShareEnabled
  await toggleScreenShare()
-  if (was != activeCall?.screenShareEnabled) {
-    document.getElementById("toggle-screen")!!.innerHTML = activeCall?.screenShareEnabled
-      ? '<img src="/desktop/images/ic_stop_screen_share.svg" />'
-      : '<img src="/desktop/images/ic_screen_share.svg" />'
+}
+
+// override function in call.ts to adapt UI to enabled media sources
+localOrPeerMediaSourcesChanged = (call: Call) => {
+  enableMicIcon(call.localMediaSources.mic)
+  enableCameraIcon(call.localMediaSources.camera)
+  enableScreenIcon(call.localMediaSources.screenVideo)
+
+  const className =
+    localMedia(call) == CallMediaType.Video || peerMedia(call) == CallMediaType.Video ? CallMediaType.Video : CallMediaType.Audio
+  document.getElementById("info-block")!.className = className
+
+  if (call.connection.iceConnectionState == "connected") {
+    document.getElementById("audio-call-icon")!.style.display = className == CallMediaType.Audio ? "block" : "none"
  }
+
+  // document.getElementById("media-sources")!.innerText = mediaSourcesStatus(call)
+  document.getElementById("manage-call")!.className = localMedia(call) == CallMediaType.Video ? CallMediaType.Video : ""
+}
+
+// override function in call.ts to adapt UI to enabled media sources
+inactiveCallMediaSourcesChanged = (inactiveCallMediaSources: CallMediaSources) => {
+  const mic = inactiveCallMediaSources.mic
+  const camera = inactiveCallMediaSources.camera
+  const screenVideo = inactiveCallMediaSources.screenVideo
+  enableMicIcon(mic)
+  enableCameraIcon(camera)
+  enableScreenIcon(screenVideo)
+
+  const className = camera ? CallMediaType.Video : CallMediaType.Audio
+  document.getElementById("info-block")!.className = className
+  // document.getElementById("media-sources")!.innerText = inactiveCallMediaSourcesStatus(inactiveCallMediaSources)
+}
+
+function enableMicIcon(enabled: boolean) {
+  document.getElementById("toggle-mic")!.innerHTML = enabled
+    ? '<img src="/desktop/images/ic_mic.svg" />'
+    : '<img src="/desktop/images/ic_mic_off.svg" />'
+}
+
+function enableCameraIcon(enabled: boolean) {
+  document.getElementById("toggle-camera")!.innerHTML = enabled
+    ? '<img src="/desktop/images/ic_videocam_filled.svg" />'
+    : '<img src="/desktop/images/ic_videocam_off.svg" />'
+}
+
+function enableScreenIcon(enabled: boolean) {
+  document.getElementById("toggle-screen")!.innerHTML = enabled
+    ? '<img src="/desktop/images/ic_stop_screen_share.svg" />'
+    : '<img src="/desktop/images/ic_screen_share.svg" />'
+}
+
+function enableSpeakerIcon(enabled: boolean, muted: boolean) {
+  document.getElementById("toggle-speaker")!!.innerHTML = muted
+    ? '<img src="/desktop/images/ic_volume_off.svg" />'
+    : enabled
+    ? '<img src="/desktop/images/ic_volume_up.svg" />'
+    : '<img src="/desktop/images/ic_volume_down.svg" />'
+  document.getElementById("toggle-speaker")!!.style.opacity = muted ? "0.7" : "1"
+}
+
+function mediaSourcesStatus(call: Call): string {
+  let status = "local"
+  if (call.localMediaSources.mic) status += " mic"
+  if (call.localMediaSources.camera) status += " cam"
+  if (call.localMediaSources.screenAudio) status += " scrA"
+  if (call.localMediaSources.screenVideo) status += " scrV"
+
+  status += " | peer"
+
+  if (call.peerMediaSources.mic) status += " mic"
+  if (call.peerMediaSources.camera) status += " cam"
+  if (call.peerMediaSources.screenAudio) status += " scrA"
+  if (call.peerMediaSources.screenVideo) status += " scrV"
+  return status
+}
+
+function inactiveCallMediaSourcesStatus(inactiveCallMediaSources: CallMediaSources): string {
+  let status = "local"
+  const mic = inactiveCallMediaSources.mic
+  const camera = inactiveCallMediaSources.camera
+  const screenAudio = inactiveCallMediaSources.screenAudio
+  const screenVideo = inactiveCallMediaSources.screenVideo
+  if (mic) status += " mic"
+  if (camera) status += " cam"
+  if (screenAudio) status += " scrA"
+  if (screenVideo) status += " scrV"
+  return status
 }

 function reactOnMessageFromServer(msg: WVApiMessage) {
+  // screen is not allowed to be enabled before connection estabilished
+  if (msg.command?.type == "capabilities" || msg.command?.type == "offer") {
+    document.getElementById("toggle-screen")!!.style.opacity = "0.7"
+  } else if (activeCall) {
+    document.getElementById("toggle-screen")!!.style.opacity = "1"
+  }
  switch (msg.command?.type) {
    case "capabilities":
-      document.getElementById("info-block")!!.className = msg.command.media
-      break
    case "offer":
    case "start":
-      document.getElementById("toggle-audio")!!.style.display = "inline-block"
-      document.getElementById("toggle-speaker")!!.style.display = "inline-block"
-      if (msg.command.media == CallMediaType.Video) {
-        document.getElementById("toggle-video")!!.style.display = "inline-block"
-        document.getElementById("toggle-screen")!!.style.display = "inline-block"
-      }
      document.getElementById("info-block")!!.className = msg.command.media
+      document.getElementById("toggle-mic")!!.style.display = "inline-block"
+      document.getElementById("toggle-speaker")!!.style.display = "inline-block"
+      document.getElementById("toggle-camera")!!.style.display = "inline-block"
+      document.getElementById("toggle-screen")!!.style.display = "inline-block"
+      enableSpeakerIcon(true, true)
      break
    case "description":
      updateCallInfoView(msg.command.state, msg.command.description)
      if (activeCall?.connection.connectionState == "connected") {
        document.getElementById("progress")!.style.display = "none"
-        if (document.getElementById("info-block")!!.className == CallMediaType.Audio) {
-          document.getElementById("audio-call-icon")!.style.display = "block"
-        }
+        document.getElementById("audio-call-icon")!.style.display =
+          document.getElementById("info-block")!!.className == CallMediaType.Audio ? "block" : "none"
      }
      break
  }
 }

+function reactOnMessageToServer(msg: WVApiMessage) {
+  if (!activeCall) return
+
+  switch (msg.resp?.type) {
+    case "peerMedia":
+      const className = localMedia(activeCall) == CallMediaType.Video || peerMedia(activeCall) == CallMediaType.Video ? "video" : "audio"
+      document.getElementById("info-block")!!.className = className
+      document.getElementById("audio-call-icon")!.style.display = className == CallMediaType.Audio ? "block" : "none"
+      enableSpeakerIcon(
+        activeCall.remoteStream.getAudioTracks().every((elem) => elem.enabled),
+        !activeCall.peerMediaSources.mic
+      )
+      break
+  }
+}
+
 function updateCallInfoView(state: string, description: string) {
  document.getElementById("state")!!.innerText = state
  document.getElementById("description")!!.innerText = description
				`@@ -0,0 +1 @@`
				<svg xmlns="http://www.w3.org/2000/svg" height="44" viewBox="0 -960 960 960" width="44" fill="white"><path d="m809.5-61.5-133-133q-27 19-58.25 33.25T553.5-139.5V-199q21.5-6.5 42.5-14.5t39.5-22L476-396v229L280-363H122.5v-234H274L54.5-816.5 96-858l755 754-41.5 42.5ZM770-291l-41.5-41.5q20-33 29.75-70.67Q768-440.85 768-481q0-100.82-58.75-180.41T553.5-763v-59.5q120 28 196 123.25t76 218.25q0 50.5-14 98.75T770-291ZM642.5-418.5l-89-89v-132q46.5 21.5 73.75 64.75T654.5-480q0 16-3 31.5t-9 30ZM476-585 372-689l104-104v208Zm-57.5 278v-145.5l-87-87H180v119h124.5l114 113.5ZM375-496Z"/></svg>