Improvements to Transcript and Captions

materialious/src/lib/api/model.ts

@@ -98,6 +98,8 @@ export interface Ytjs {
 	rawApiResponse: ApiResponse;
 }
 
+export type FallbackPatches = 'youtubejs' | 'piped';
+
 export interface VideoPlay extends Video {
 	keywords: string[];
 	likeCount: number;
@@ -120,7 +122,7 @@ export interface VideoPlay extends Video {
 	captions: Captions[];
 	storyboards?: StoryBoard[];
 	ytjs?: Ytjs;
-	fallbackPatch?: 'youtubejs' | 'piped';
+	fallbackPatch?: FallbackPatches;
 }
 
 export interface StoryBoard {

materialious/src/lib/components/player/ClosedCaptions.svelte

@@ -48,11 +48,12 @@
 
 <script lang="ts">
 	import type { VideoPlay } from '$lib/api/model';
-	import { decodeHtmlCharCodes, findElementForTime, getPublicEnv } from '$lib/misc';
+	import { decodeHtmlCharCodes, findElementForTime, getPublicEnv, isYTBackend } from '$lib/misc';
 	import { invidiousInstanceStore } from '$lib/store';
 	import { onDestroy, onMount } from 'svelte';
 	import { addToast } from '../Toast.svelte';
-	import { parseText, type VTTCue } from 'media-captions';
+	import { parseText, renderVTTCueString, type VTTCue } from 'media-captions';
+	import { getCaptionUrl } from '$lib/player/captions';
 
 	let {
 		video,
@@ -91,14 +92,9 @@
 
 		if (video.captions) {
 			for (const caption of video.captions) {
-				let captionUrl: string;
-				if (!getPublicEnv('DEFAULT_COMPANION_INSTANCE') && $invidiousInstanceStore) {
-					captionUrl = caption.url.startsWith('http')
-						? caption.url
-						: `${new URL($invidiousInstanceStore).origin}${caption.url}`;
-				} else {
-					captionUrl = `${getPublicEnv('DEFAULT_COMPANION_INSTANCE')}${caption.url}`;
-				}
+				const captionUrl = getCaptionUrl(caption, video.fallbackPatch);
+
+				if (!captionUrl) continue;
 
 				captionTracks[caption.language_code] = captionUrl;
 			}
@@ -111,13 +107,13 @@
 </script>
 
 {#if trackVisible && captionsCues.length > 0}
-	{#if currentCaption && currentTime <= currentCaption.endTime && currentTime >= currentCaption.startTime}
+	{#if currentCaption}
 		<div
 			class="caption-container"
 			bind:this={captionElement}
 			style:top={`calc(${showControls ? 'var(--video-player-height) * 0.85' : 'var(--video-player-height) * 0.98'} - ${captionContainerHeight}px)`}
 		>
-			{#if currentCaption?.text}
+			{#if currentCaption}
 				<p
 					style="
 					font-size: {captionFontSize};
@@ -129,7 +125,8 @@
 					user-select: none;
 				"
 				>
-					{decodeHtmlCharCodes(currentCaption.text)}
+					<!-- eslint-disable-next-line svelte/no-at-html-tags -->
+					{@html renderVTTCueString(currentCaption, currentTime)}
 				</p>
 			{/if}
 		</div>

materialious/src/lib/components/watch/Transcript.svelte

@@ -1,31 +1,32 @@
 <script lang="ts">
 	import { videoLength } from '$lib/numbers';
 	import Fuse from 'fuse.js';
-	import { type VTTCue, parseText, type ParsedCaptionsResult } from 'media-captions';
+	import {
+		type VTTCue,
+		parseText,
+		type ParsedCaptionsResult,
+		renderVTTCueString
+	} from 'media-captions';
 	import { _ } from '$lib/i18n';
-	import type { VideoPlay } from '$lib/api/model';
-	import { decodeHtmlCharCodes } from '$lib/misc';
-	import { invidiousInstanceStore } from '$lib/store';
+	import type { Captions, VideoPlay } from '$lib/api/model';
+	import { getCaptionUrl } from '$lib/player/captions';
 
 	interface Props {
 		video: VideoPlay;
-		playerElement: HTMLMediaElement;
+		currentTime: number;
 	}
 
-	let { video, playerElement = $bindable() }: Props = $props();
+	let { video, currentTime = $bindable() }: Props = $props();
 
-	let url: string | null = $state(null);
+	let selectedCaption: Captions | undefined = $state();
 	let autoScroll: boolean = $state(true);
 
 	let transcript: ParsedCaptionsResult | null = $state(null);
 	let transcriptCues: VTTCue[] = $state([]);
 	let isLoading = $state(false);
-	let currentTime = $state(0);
 	let search: string = $state('');
 
-	playerElement.addEventListener('timeupdate', () => {
-		currentTime = playerElement.currentTime;
-
+	$effect(() => {
 		if (autoScroll) {
 			const currentTranscriptLine = document.querySelector(
 				'.transcript-line.secondary-container'
@@ -40,28 +41,21 @@
 	});
 
 	async function loadTranscript() {
-		if (!url) {
+		if (!selectedCaption) {
 			transcript = null;
 			return;
 		}
-
-		let urlConstructed = '';
-
-		if (video.fallbackPatch === 'youtubejs') {
-			urlConstructed = url;
-		} else if ($invidiousInstanceStore) {
-			urlConstructed = new URL($invidiousInstanceStore).origin + url;
-		} else {
-			return;
-		}
-
 		isLoading = true;
 		transcript = null;
 
-		const resp = await fetch(urlConstructed);
-		if (!resp.ok) return;
-		transcript = await parseText(await resp.text(), { strict: false });
+		const captionUrl = getCaptionUrl(selectedCaption, video.fallbackPatch);
 
+		if (!captionUrl) return;
+
+		const resp = await fetch(captionUrl);
+		if (!resp.ok) return;
+
+		transcript = await parseText(await resp.text(), { strict: false });
 		transcriptCues = transcript.cues;
 
 		isLoading = false;
@@ -87,10 +81,10 @@
 	<article class="no-elevate padding" style="position: sticky; top: 0; z-index: 3;">
 		<h6>{$_('transcript')}</h6>
 		<div class="field label suffix surface-container-highest">
-			<select bind:value={url} onchange={loadTranscript} name="captions">
+			<select bind:value={selectedCaption} onchange={loadTranscript} name="captions">
 				<option selected={true} value={null}>{$_('selectLang')}</option>
 				{#each video.captions as caption (caption)}
-					<option value={caption.url}>{caption.label}</option>
+					<option value={caption}>{caption.label}</option>
 				{/each}
 			</select>
 			<label for="captions">{$_('language')}</label>
@@ -132,7 +126,10 @@
 							currentTime <= cue.endTime}
 					>
 						<p class="chip no-margin">{videoLength(cue.startTime)}</p>
-						<p class="transcript-text">{decodeHtmlCharCodes(cue.text.replace(/<[^>]+>/g, ''))}</p>
+						<p class="transcript-text">
+							<!-- eslint-disable-next-line svelte/no-at-html-tags -->
+							{@html renderVTTCueString(cue, currentTime)}
+						</p>
 					</div>
 				{/each}
 			{:else}

materialious/src/lib/player/captions.ts

@@ -0,0 +1,25 @@
+import type { Captions, FallbackPatches } from '$lib/api/model';
+import { getPublicEnv } from '$lib/misc';
+import { invidiousInstanceStore } from '$lib/store';
+import { get } from 'svelte/store';
+
+export function getCaptionUrl(
+	caption: Captions,
+	fallbackPath: FallbackPatches | undefined = undefined
+) {
+	let captionUrl: string | undefined;
+
+	const invidiousInstance = get(invidiousInstanceStore);
+
+	if (getPublicEnv('DEFAULT_COMPANION_INSTANCE')) {
+		captionUrl = `${getPublicEnv('DEFAULT_COMPANION_INSTANCE')}${caption.url}`;
+	} else if (fallbackPath === 'youtubejs') {
+		captionUrl = caption.url;
+	} else if (invidiousInstance) {
+		captionUrl = caption.url.startsWith('http')
+			? caption.url
+			: `${new URL(invidiousInstance).origin}${caption.url}`;
+	}
+
+	return captionUrl;
+}

materialious/src/routes/(app)/watch/[slug]/+page.svelte

@@ -595,8 +595,8 @@
 	</div>
 	{#if !$playertheatreModeIsActive}
 		<div class="s12 m12 l3 recommended">
-			{#if showTranscript && playerElement}
-				<Transcript video={data.video} bind:playerElement />
+			{#if showTranscript}
+				<Transcript video={data.video} bind:currentTime={playerCurrentTime} />
 			{/if}
 			{#if data.playlistId && data.playlistId in $playlistCacheStore}
 				<Playlist video={data.video} playlist={$playlistCacheStore[data.playlistId]} />