blade/materialious
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved timeline thumbnails

Browse Source
This commit is contained in:
WardPearce
2026-02-11 18:08:47 +13:00
parent 5fccf1aac8
commit d86c865fd5
13 changed files with 290 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
materialious/src/lib/api/model.ts
+2
View File
@@ -125,6 +125,8 @@ export interface StoryBoard {
storyboardWidth: number;
storyboardHeight: number;
storyboardCount: number;
columns?: number;
rows?: number;
}
export interface ReturnYTDislikes {
materialious/src/lib/api/youtubejs.ts
+4 -2
View File
@@ -8,8 +8,8 @@ import type {
VideoBase,
VideoPlay
} from '$lib/api/model';
import { convertToSeconds } from '$lib/numbers';
import { interfaceRegionStore, poTokenCacheStore } from '$lib/store';
import { convertToSeconds } from '$lib/time';
import { Capacitor } from '@capacitor/core';
import { USER_AGENT } from 'bgutils-js';
import { get } from 'svelte/store';
@@ -221,7 +221,9 @@ export async function getVideoTYjs(videoId: string): Promise<VideoPlay> {
interval: board.interval,
storyboardCount: board.storyboard_count,
storyboardHeight: board.thumbnail_height,
storyboardWidth: board.thumbnail_width
storyboardWidth: board.thumbnail_width,
columns: board.columns,
rows: board.rows
});
});
}
materialious/src/lib/components/Player.svelte
+35 -23
View File
@@ -1,8 +1,8 @@
<script lang="ts">
import { page } from '$app/stores';
import { getBestThumbnail, ImageCache } from '$lib/images';
import { padTime, videoLength } from '$lib/numbers';
import { type PhasedDescription, type Timestamp } from '$lib/timestamps';
import { videoLength } from '$lib/numbers';
import { generateChapterWebVTT, type PhasedDescription, type Timestamp } from '$lib/description';
import { SystemBars, SystemBarsStyle, SystemBarType } from '@capacitor/core';
import { Capacitor } from '@capacitor/core';
import { ScreenOrientation, type ScreenOrientationResult } from '@capacitor/screen-orientation';
@@ -60,10 +60,12 @@
import { addToast } from './Toast.svelte';
import { isMobile, truncate } from '$lib/misc';
import {
generateThumbnailWebVTT,
drawTimelineThumbnail,
storyboardThumbnails,
type TimelineThumbnail
} from '$lib/timelineThumbnails';
import { padTime } from '$lib/time';
interface Props {
data: { video: VideoPlay; content: PhasedDescription; playlistId: string | null };
@@ -526,12 +528,36 @@
}
if (data.video.storyboards && data.video.storyboards.length > 2) {
try {
storyboardThumbnails(data.video).then((thumbnails) => {
playerTimelineThumbnails = thumbnails;
});
} catch {
// Continue regardless of error.
let thumbnailVTT: string | undefined;
const selectedStoryboard = data.video.storyboards[2];
if (
data.video.fallbackPatch === 'youtubejs' &&
typeof selectedStoryboard.rows !== 'undefined' &&
typeof selectedStoryboard.columns !== 'undefined'
) {
thumbnailVTT = generateThumbnailWebVTT(
{
...selectedStoryboard,
rows: selectedStoryboard.rows,
columns: selectedStoryboard.columns
},
playerMaxKnownTime
);
} else if (!data.video.fallbackPatch) {
const thumbnailVTTResp = await fetch(`${$instanceStore}${selectedStoryboard.url}`);
if (thumbnailVTTResp.ok) thumbnailVTT = await thumbnailVTTResp.text();
}
if (thumbnailVTT) {
try {
storyboardThumbnails(thumbnailVTT).then((thumbnails) => {
playerTimelineThumbnails = thumbnails;
});
} catch {
// Continue regardless of error.
}
}
}
@@ -569,22 +595,9 @@
}
if (data.content.timestamps) {
let chapterWebVTT = 'WEBVTT\n\n';
data.content.timestamps.forEach((timestamp, timestampIndex) => {
let endTime: string;
if (timestampIndex === data.content.timestamps.length - 1) {
endTime = videoLength(data.video.lengthSeconds);
} else {
endTime = data.content.timestamps[timestampIndex + 1].timePretty;
}
chapterWebVTT += `${padTime(timestamp.timePretty)}.000 --> ${padTime(endTime)}.000\n${timestamp.title.replaceAll('-', '').trim()}\n\n`;
});
try {
player.addChaptersTrack(
`data:text/vtt;base64,${btoa(chapterWebVTT)}`,
`data:text/vtt;base64,${btoa(generateChapterWebVTT(data.content.timestamps, playerMaxKnownTime))}`,
get(playerDefaultLanguage)
);
} catch {
@@ -1146,7 +1159,6 @@
canvasContext,
playerTimelineThumbnailsCache,
playerTimelineThumbnails,
data.video,
time
);
}
materialious/src/lib/timestamps.ts → materialious/src/lib/description.ts
+19 -1
View File
@@ -1,5 +1,6 @@
import { decodeHtmlCharCodes } from './misc';
import { convertToSeconds } from './numbers';
import { videoLength } from './numbers';
import { convertToSeconds, padTime } from './time';
export type Timestamp = { title: string; time: number; timePretty: string; endTime: number };
export type Timestamps = Timestamp[];
@@ -118,3 +119,20 @@ export function phaseDescription(
return { description: filteredContent, timestamps };
}
export function generateChapterWebVTT(timestamps: Timestamp[], videoLengthSeconds: number) {
let chapterWebVTT = 'WEBVTT\n\n';
timestamps.forEach((timestamp, timestampIndex) => {
let endTime: string;
if (timestampIndex === timestamps.length - 1) {
endTime = videoLength(videoLengthSeconds);
} else {
endTime = timestamps[timestampIndex + 1].timePretty;
}
chapterWebVTT += `${padTime(timestamp.timePretty)}.000 --> ${padTime(endTime)}.000\n${timestamp.title.replaceAll('-', '').trim()}\n\n`;
});
return chapterWebVTT;
}
materialious/src/lib/misc.ts
+33
View File
@@ -189,3 +189,36 @@ export function createVideoUrl(videoId: string, playlistId: string): URL {
export function timeout(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export function findElementForTime<T>(
elements: T[],
currentTime: number,
getStartTime: (element: T) => number,
getEndTime: (element: T) => number
): T | null {
let left = 0;
let right = elements.length - 1;
while (left <= right) {
const mid = Math.floor((left + right) / 2);
const element = elements[mid];
const startTime = getStartTime(element);
const endTime = getEndTime(element);
// Check if currentTime is within the time range of the element
if (currentTime >= startTime && currentTime <= endTime) {
return element;
}
// If currentTime is earlier, search the left half
if (currentTime < startTime) {
right = mid - 1;
}
// If currentTime is later, search the right half
else {
left = mid + 1;
}
}
return null;
}
materialious/src/lib/numbers.ts
+19 -79
View File
@@ -1,92 +1,32 @@
import humanNumber from 'human-number';
export function numberWithCommas(number: number) {
if (typeof number === 'undefined') return;
return number.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ',');
if (typeof number === 'undefined') return;
return number.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ',');
}
export function cleanNumber(number: number): string {
return humanNumber(number, (number: number) =>
Number.parseFloat(number.toString()).toFixed(1)
).replace('.0', '');
return humanNumber(number, (number: number) =>
Number.parseFloat(number.toString()).toFixed(1)
).replace('.0', '');
}
export function videoLength(lengthSeconds: number): string {
const hours = Math.floor(lengthSeconds / 3600);
let minutes: number | string = Math.floor((lengthSeconds % 3600) / 60);
let seconds: number | string = Math.round(lengthSeconds % 60);
const hours = Math.floor(lengthSeconds / 3600);
let minutes: number | string = Math.floor((lengthSeconds % 3600) / 60);
let seconds: number | string = Math.round(lengthSeconds % 60);
if (minutes < 10) {
minutes = `0${minutes}`;
}
if (minutes < 10) {
minutes = `0${minutes}`;
}
if (seconds < 10) {
seconds = `0${seconds}`;
}
if (seconds < 10) {
seconds = `0${seconds}`;
}
if (hours !== 0) {
return `${hours}:${minutes}:${seconds}`;
} else {
return `${minutes}:${seconds}`;
}
if (hours !== 0) {
return `${hours}:${minutes}:${seconds}`;
} else {
return `${minutes}:${seconds}`;
}
}
export function padTime(time: string): string {
let timeParts = time.split(':');
if (timeParts.length < 3) {
timeParts = ('00:' + time).split(':');
}
const hours = (timeParts[0] || '0').padStart(2, '0');
const minutes = (timeParts[1] || '0').padStart(2, '0');
let seconds = timeParts[2] || '0';
let milliseconds = '';
if (seconds.includes('.')) {
const [sec, ms] = seconds.split('.');
seconds = sec.padStart(2, '0');
milliseconds = `.${ms.padStart(3, '0')}`;
} else {
seconds = seconds.padStart(2, '0');
}
return `${hours}:${minutes}:${seconds}${milliseconds}`;
}
export function convertToSeconds(time: string): number {
const parts = time.split(':').map((part) => parseInt(part));
let seconds = 0;
if (parts.length === 3) {
// hh:mm:ss
seconds = parts[0] * 3600 + parts[1] * 60 + parts[2];
} else if (parts.length === 2) {
// hh:ss or m:ss
seconds = parts[0] * 60 + parts[1];
} else if (parts.length === 1) {
// s
seconds = parts[0];
}
return seconds;
}
export function humanizeSeconds(totalSeconds: number): string {
const secondsInMinute = 60;
const secondsInHour = 3600;
const hours = Math.floor(totalSeconds / secondsInHour);
const minutes = Math.floor((totalSeconds % secondsInHour) / secondsInMinute);
const parts: string[] = [];
if (hours > 0) {
parts.push(`${hours} hour${hours > 1 ? 's' : ''}`);
}
if (minutes > 0) {
parts.push(`${minutes} minute${minutes > 1 ? 's' : ''}`);
}
return parts.join(', ');
}
materialious/src/lib/store.ts
+1 -1
View File
@@ -25,7 +25,7 @@ import type {
VideoPlay
} from './api/model';
import { ensureNoTrailingSlash } from './misc';
import type { PhasedDescription } from './timestamps';
import type { PhasedDescription } from './description';
function createListenerFunctions(): {
callListeners: (eventKey: string, newValue: any) => void;
materialious/src/lib/time.ts
+59
View File
@@ -34,3 +34,62 @@ export function relativeTimestamp(epochTime: number): string {
return timestamp.format('MMMM Do YYYY h:mm A');
}
}
export function padTime(time: string): string {
let timeParts = time.split(':');
if (timeParts.length < 3) {
timeParts = ('00:' + time).split(':');
}
const hours = (timeParts[0] || '0').padStart(2, '0');
const minutes = (timeParts[1] || '0').padStart(2, '0');
let seconds = timeParts[2] || '0';
let milliseconds = '';
if (seconds.includes('.')) {
const [sec, ms] = seconds.split('.');
seconds = sec.padStart(2, '0');
milliseconds = `.${ms.padStart(3, '0')}`;
} else {
seconds = seconds.padStart(2, '0');
}
return `${hours}:${minutes}:${seconds}${milliseconds}`;
}
export function convertToSeconds(time: string): number {
const parts = time.split(':').map((part) => parseInt(part));
let seconds = 0;
if (parts.length === 3) {
// hh:mm:ss
seconds = parts[0] * 3600 + parts[1] * 60 + parts[2];
} else if (parts.length === 2) {
// hh:ss or m:ss
seconds = parts[0] * 60 + parts[1];
} else if (parts.length === 1) {
// s
seconds = parts[0];
}
return seconds;
}
export function humanizeSeconds(totalSeconds: number): string {
const secondsInMinute = 60;
const secondsInHour = 3600;
const hours = Math.floor(totalSeconds / secondsInHour);
const minutes = Math.floor((totalSeconds % secondsInHour) / secondsInMinute);
const parts: string[] = [];
if (hours > 0) {
parts.push(`${hours} hour${hours > 1 ? 's' : ''}`);
}
if (minutes > 0) {
parts.push(`${minutes} minute${minutes > 1 ? 's' : ''}`);
}
return parts.join(', ');
}
materialious/src/lib/timelineThumbnails.ts
+113 -89
View File
@@ -1,82 +1,114 @@
import { get } from 'svelte/store';
import type { VideoPlay } from './api/model';
import type { StoryBoard } from './api/model';
import { ImageCache } from './images';
import { instanceStore } from './store';
import { parseText } from 'media-captions';
import { findElementForTime } from './misc';
export interface TimelineThumbnail {
url: string;
time: number;
startTime: number;
endTime: number;
width: number;
height: number;
index: number;
yCoord: number;
xCoord: number;
}
export async function storyboardThumbnails(video: VideoPlay): Promise<TimelineThumbnail[]> {
if (!video.storyboards || video.storyboards.length < 2) return [];
// Modified implementation from Freetube
// https://github.com/FreeTubeApp/FreeTube/blob/20a44152fb1465cc01f496f1e22abd6a9a4b8390/src/renderer/helpers/utils.js#L107
export function generateThumbnailWebVTT(
storyboard: StoryBoard & { columns: number; rows: number },
videoLengthSeconds: number
) {
let vttString = 'WEBVTT\n\n';
// Amount of thumbnails per sheet
const numberOfThumbnailsPerSheet = storyboard.columns * storyboard.rows;
// Amount of sheets
const numberOfSheets = storyboard.count;
let intervalInSeconds;
if (storyboard.interval > 0) {
intervalInSeconds = storyboard.interval / 1000;
} else {
intervalInSeconds = videoLengthSeconds / (numberOfSheets * numberOfThumbnailsPerSheet);
}
let startHours = 0;
let startMinutes = 0;
let startSeconds = 0;
let endHours = 0;
let endMinutes = 0;
let endSeconds = intervalInSeconds;
for (let i = 0; i < numberOfSheets; i++) {
const currentUrl = storyboard.templateUrl.replace('$M.jpg', `${i}.jpg`);
let xCoord = 0;
let yCoord = 0;
for (let j = 0; j < numberOfThumbnailsPerSheet; j++) {
// add the timestamp information
const paddedStartHours = startHours.toString().padStart(2, '0');
const paddedStartMinutes = startMinutes.toString().padStart(2, '0');
const paddedStartSeconds = startSeconds.toFixed(3).padStart(6, '0');
const paddedEndHours = endHours.toString().padStart(2, '0');
const paddedEndMinutes = endMinutes.toString().padStart(2, '0');
const paddedEndSeconds = endSeconds.toFixed(3).padStart(6, '0');
vttString += `${paddedStartHours}:${paddedStartMinutes}:${paddedStartSeconds} --> ${paddedEndHours}:${paddedEndMinutes}:${paddedEndSeconds}\n`;
// add the current image url as well as the x, y, width, height information
vttString += `${currentUrl}#xywh=${xCoord},${yCoord},${storyboard.width},${storyboard.height}\n\n`;
// update the variables
startHours = endHours;
startMinutes = endMinutes;
startSeconds = endSeconds;
endSeconds += intervalInSeconds;
if (endSeconds >= 60) {
endSeconds -= 60;
endMinutes += 1;
}
if (endMinutes >= 60) {
endMinutes -= 60;
endHours += 1;
}
// x coordinate can only be smaller than the width of one subimage * the number of subimages per row
xCoord = (xCoord + storyboard.width) % (storyboard.width * storyboard.columns);
// only if the x coordinate is , so in a new row, we have to update the y coordinate
if (xCoord === 0) {
yCoord += storyboard.height;
}
}
}
return vttString;
}
export async function storyboardThumbnails(WebVTT: string): Promise<TimelineThumbnail[]> {
const thumbnails: TimelineThumbnail[] = [];
if (video.fallbackPatch === 'youtubejs') {
const { count, storyboardCount, width, height, templateUrl } = video.storyboards[2];
const thumbnailsSheets = await parseText(WebVTT, {
strict: true,
type: 'vtt'
});
const totalThumbnails = storyboardCount * count;
const videoDurationMs = video.lengthSeconds * 1000;
const timeInterval = videoDurationMs / totalThumbnails;
let index = 0;
thumbnailsSheets.cues.forEach((cue) => {
const urlParts = cue.text.split('#xywh=');
const xywh = urlParts[1];
const xywhValues = xywh.split(',');
for (let sheetIndex = 0; sheetIndex < storyboardCount; sheetIndex++) {
const url = templateUrl.replace('M$M', `M${sheetIndex}`);
const time = Math.floor(sheetIndex * count * timeInterval);
thumbnails.push({
url,
time,
width,
height,
index: sheetIndex * count
});
if (xywhValues.length !== 4) {
return [];
}
} else {
const WebVTTResp = await fetch(`${get(instanceStore)}${video.storyboards[2].url}`);
if (!WebVTTResp.ok) return [];
const thumbnailsSheets = await parseText(await WebVTTResp.text(), {
strict: true,
type: 'vtt'
const [xCoord, yCoord, thumbWidth, thumbHeight] = xywhValues.map(Number);
thumbnails.push({
url: cue.text,
startTime: cue.startTime,
endTime: cue.endTime,
width: thumbWidth,
height: thumbHeight,
index,
xCoord,
yCoord
});
const processedUrls = new Set<string>();
let index = 0;
thumbnailsSheets.cues.forEach((cue) => {
const urlParts = cue.text.split('#');
const cleanedUrl = urlParts[0];
if (!processedUrls.has(cleanedUrl)) {
const xywh = urlParts[1];
const xywhValues = xywh.split(',');
if (xywhValues.length !== 4) {
return [];
}
const [, , thumbWidth, thumbHeight] = xywhValues.map(Number);
processedUrls.add(cleanedUrl);
thumbnails.push({
url: cleanedUrl,
time: cue.startTime * 1000,
width: thumbWidth,
height: thumbHeight,
index
});
index++;
}
});
}
index++;
});
return thumbnails;
}
@@ -85,45 +117,37 @@ export async function drawTimelineThumbnail(
ctx: CanvasRenderingContext2D,
imageCache: ImageCache,
thumbnails: TimelineThumbnail[],
video: VideoPlay,
currentTime: number
): Promise<void> {
if (!thumbnails.length) return;
const timeInMs = currentTime * 1000;
const thumbnail = findElementForTime(
thumbnails,
currentTime,
(thumbnail: TimelineThumbnail) => thumbnail.startTime,
(thumbnail: TimelineThumbnail) => thumbnail.endTime
);
const closest = thumbnails.reduce((prev, curr) => {
return Math.abs(curr.time - timeInMs) < Math.abs(prev.time - timeInMs) ? curr : prev;
});
if (!thumbnail) return;
const nextThumbnail = thumbnails.find((thumb) => thumb.time > closest.time);
const spriteSheetEndTime = nextThumbnail ? nextThumbnail.time : video.lengthSeconds * 1000;
const img = await imageCache.load(closest.url);
const img = await imageCache.load(thumbnail.url);
if (!img) {
return;
}
const sheetWidth = img.width;
const sheetHeight = img.height;
const thumbWidth = closest.width;
const thumbHeight = closest.height;
const cols = Math.floor(sheetWidth / thumbWidth);
const rows = Math.floor(sheetHeight / thumbHeight);
const thumbnailsPerSheet = cols * rows;
const elapsedTime = timeInMs - closest.time;
const totalDuration = spriteSheetEndTime - closest.time;
const elapsedPercentage = Math.min(Math.max(elapsedTime / totalDuration, 0), 1);
const indexInSheet = Math.floor(elapsedPercentage * thumbnailsPerSheet);
const thumbX = (indexInSheet % cols) * thumbWidth;
const thumbY = Math.floor(indexInSheet / cols) * thumbHeight;
const thumbWidth = thumbnail.width;
const thumbHeight = thumbnail.height;
ctx.clearRect(0, 0, thumbWidth, thumbHeight);
ctx.drawImage(img, thumbX, thumbY, thumbWidth, thumbHeight, 0, 0, thumbWidth, thumbHeight);
ctx.drawImage(
img,
thumbnail.xCoord,
thumbnail.yCoord,
thumbWidth,
thumbHeight,
0,
0,
thumbWidth,
thumbHeight
);
}
materialious/src/lib/watch.ts
+1 -1
View File
@@ -13,7 +13,7 @@ import {
returnYTDislikesInstanceStore,
returnYtDislikesStore
} from '$lib/store';
import { phaseDescription } from '$lib/timestamps';
import { phaseDescription } from '$lib/description';
import { error } from '@sveltejs/kit';
import { get } from 'svelte/store';
import { _ } from './i18n';
materialious/src/routes/(app)/watch/[slug]/+page.svelte
+2 -2
View File
@@ -12,7 +12,7 @@
import Transcript from '$lib/components/Transcript.svelte';
import { getBestThumbnail } from '$lib/images';
import { letterCase } from '$lib/letterCasing';
import { cleanNumber, humanizeSeconds, numberWithCommas } from '$lib/numbers';
import { cleanNumber, numberWithCommas } from '$lib/numbers';
import { goToNextVideo, goToPreviousVideo, type PlayerEvents } from '$lib/player';
import {
authStore,
@@ -39,7 +39,7 @@
import LikesDislikes from '$lib/components/watch/LikesDislikes.svelte';
import Comment from '$lib/components/watch/Comment.svelte';
import { expandSummery } from '$lib/misc';
import { relativeTimestamp } from '$lib/time.js';
import { humanizeSeconds, relativeTimestamp } from '$lib/time.js';
import { getWatchDetails } from '$lib/watch.js';
import { page } from '$app/state';
materialious/static/localApi/ghContributors.json
+1 -1
View File
File diff suppressed because one or more lines are too long
update_versions.py
+1 -1
View File
@@ -3,7 +3,7 @@ import os
import re
from datetime import datetime
LATEST_VERSION = "1.13.18"
LATEST_VERSION = "1.13.19"
RELEASE_DATE = datetime.now().strftime("%Y-%-m-%d") # Format: YYYY-M-D
WORKING_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "materialious")