From f4579326d138d1d6653cefb2c9e27b67e2f94182 Mon Sep 17 00:00:00 2001 From: WardPearce Date: Tue, 24 Feb 2026 12:26:56 +1300 Subject: [PATCH] Implemented watch history pool --- materialious/src/lib/api/apiExtended.ts | 50 ---------- materialious/src/lib/api/backend/history.ts | 94 +++++++++++++++++-- .../src/lib/api/backend/historyPool.ts | 51 ++++++++++ materialious/src/lib/api/model.ts | 11 +++ .../src/lib/components/player/Player.svelte | 6 ++ .../thumbnail/VideoThumbnail.svelte | 7 ++ materialious/src/lib/server/database.ts | 20 ++-- .../src/routes/api/user/history/+server.ts | 20 ++-- .../api/user/history/[videoHash]/+server.ts | 12 +-- 9 files changed, 192 insertions(+), 79 deletions(-) delete mode 100644 materialious/src/lib/api/apiExtended.ts create mode 100644 materialious/src/lib/api/backend/historyPool.ts diff --git a/materialious/src/lib/api/apiExtended.ts b/materialious/src/lib/api/apiExtended.ts deleted file mode 100644 index 5d0e1035..00000000 --- a/materialious/src/lib/api/apiExtended.ts +++ /dev/null @@ -1,50 +0,0 @@ -// const videoIds: string[] = []; -// const pendingResolves = new Map void>(); - -// let timeout: ReturnType | null = null; -// const DEBOUNCE_MS = 1000; -// const BATCH_SIZE = 100; - -// async function processBatches(): Promise { -// const batches: string[][] = []; - -// while (videoIds.length > 0) { -// batches.push(videoIds.splice(0, BATCH_SIZE)); -// } - -// const results: ApiExntendedProgressModel[] = []; - -// for (const batch of batches) { -// const res: ApiExntendedProgressModel[] = await getVideoProgress(batch.join(',')); -// results.push(...res); - -// // Resolve pending promises for this batch -// for (const videoId of batch) { -// const match = res.find((item) => item.video_id === videoId); -// const resolve = pendingResolves.get(videoId); -// if (resolve) { -// resolve(match); -// pendingResolves.delete(videoId); -// } -// } -// } -// } - -// export function queueGetWatchProgress( -// videoId: string -// ): Promise { -// videoIds.push(videoId); - -// const promise = new Promise((resolve) => { -// pendingResolves.set(videoId, resolve); -// }); - -// if (timeout) clearTimeout(timeout); -// timeout = setTimeout(() => { -// processBatches().catch((e) => { -// console.error('Failed to process batches:', e); -// }); -// }, DEBOUNCE_MS); - -// return promise; -// } diff --git a/materialious/src/lib/api/backend/history.ts b/materialious/src/lib/api/backend/history.ts index 3aa85758..f917892b 100644 --- a/materialious/src/lib/api/backend/history.ts +++ b/materialious/src/lib/api/backend/history.ts @@ -1,6 +1,6 @@ import sodium from 'libsodium-wrappers-sumo'; -import { encryptWithMasterKey, getRawKey, getSecureHash } from './encryption'; -import type { VideoPlay } from '../model'; +import { decryptWithMasterKey, encryptWithMasterKey, getRawKey, getSecureHash } from './encryption'; +import type { VideoPlay, WatchHistoryItem } from '../model'; import { getBestThumbnail } from '$lib/images'; export async function updateWatchHistory(videoId: string, progress: number) { @@ -20,6 +20,85 @@ export async function updateWatchHistory(videoId: string, progress: number) { }); } +async function decryptWatchHistory( + history: Record +): Promise { + const author = (await decryptWithMasterKey( + history.authorNonce as string, + history.authorCipher as string + )) as string; + const title = (await decryptWithMasterKey( + history.titleNonce as string, + history.titleCipher as string + )) as string; + const thumbnail = (await decryptWithMasterKey( + history.thumbnailNonce as string, + history.thumbnailCipher as string + )) as string; + const videoId = (await decryptWithMasterKey( + history.videoIdNonce as string, + history.videoIdCipher as string + )) as string; + + return { + author, + title, + thumbnail, + videoId, + watched: new Date(history.watched), + duration: history.duration as number, + id: history.id as string, + progress: history.progress as number + }; +} + +export async function getVideoWatchHistory(videoId: string): Promise { + await sodium.ready; + const rawKey = await getRawKey(); + if (!rawKey) return; + + const videoHash = await getSecureHash(videoId, rawKey); + + const resp = await fetch(`/api/user/history/${videoHash}`, { + method: 'GET', + credentials: 'same-origin' + }); + + if (!resp.ok) return; + + return await decryptWatchHistory(await resp.json()); +} + +export async function getWatchHistory( + options: { page?: number; videoIds?: string[] } = { page: 0, videoIds: undefined } +): Promise { + await sodium.ready; + const rawKey = await getRawKey(); + if (!rawKey) return []; + + const videoHashes: string[] = []; + if (options.videoIds) { + for (const videoId of options.videoIds) { + videoHashes.push(await getSecureHash(videoId, rawKey)); + } + } + + const resp = await fetch( + `/api/user/history?page=${options.page}${videoHashes.length > 0 ? '&videoHashes=' + videoHashes.join(',') : ''}` + ); + if (!resp.ok) return []; + + const rawHistory = await resp.json(); + + const history: WatchHistoryItem[] = []; + + for (const item of rawHistory) { + history.push(await decryptWatchHistory(item)); + } + + return history; +} + export async function saveWatchHistory(video: VideoPlay, progress: number = 0) { await sodium.ready; const rawKey = await getRawKey(); @@ -30,7 +109,7 @@ export async function saveWatchHistory(video: VideoPlay, progress: number = 0) { const title = await encryptWithMasterKey(video.title); const author = await encryptWithMasterKey(video.author); const thumbnail = await encryptWithMasterKey(getBestThumbnail(video.videoThumbnails)); - const duration = await encryptWithMasterKey(video.lengthSeconds.toString()); + const videoId = await encryptWithMasterKey(video.videoId); await fetch('/api/user/history', { method: 'POST', @@ -51,10 +130,11 @@ export async function saveWatchHistory(video: VideoPlay, progress: number = 0) { cipher: thumbnail?.cipher, nonce: thumbnail?.nonce }, - duration: { - cipher: duration?.cipher, - nonce: duration?.nonce - } + videoId: { + cipher: videoId?.cipher, + nonce: videoId?.nonce + }, + duration: video.lengthSeconds }) }); } diff --git a/materialious/src/lib/api/backend/historyPool.ts b/materialious/src/lib/api/backend/historyPool.ts new file mode 100644 index 00000000..ab60a4de --- /dev/null +++ b/materialious/src/lib/api/backend/historyPool.ts @@ -0,0 +1,51 @@ +import type { WatchHistoryItem } from '../model'; +import { getWatchHistory } from './history'; + +const videoIds: string[] = []; +const pendingResolves = new Map void>(); + +let timeout: ReturnType | null = null; +const DEBOUNCE_MS = 1000; +const BATCH_SIZE = 100; + +async function processBatches(): Promise { + const batches: string[][] = []; + + while (videoIds.length > 0) { + batches.push(videoIds.splice(0, BATCH_SIZE)); + } + + const results: WatchHistoryItem[] = []; + + for (const batch of batches) { + const res: WatchHistoryItem[] = await getWatchHistory({ videoIds: batch }); + results.push(...res); + + // Resolve pending promises for this batch + for (const videoId of batch) { + const match = res.find((item) => item.videoId === videoId); + const resolve = pendingResolves.get(videoId); + if (resolve) { + resolve(match); + pendingResolves.delete(videoId); + } + } + } +} + +export function queueGetWatchHistory(videoId: string): Promise { + videoIds.push(videoId); + + const promise = new Promise((resolve) => { + pendingResolves.set(videoId, resolve); + }); + + if (timeout) clearTimeout(timeout); + timeout = setTimeout(() => { + processBatches().catch((e) => { + console.error('Failed to process batches:', e); + }); + }, DEBOUNCE_MS); + + return promise; +} diff --git a/materialious/src/lib/api/model.ts b/materialious/src/lib/api/model.ts index 5523be05..d2448cf1 100644 --- a/materialious/src/lib/api/model.ts +++ b/materialious/src/lib/api/model.ts @@ -321,3 +321,14 @@ export type ChannelOptions = { continuation?: string; sortBy?: ChannelSortBy; }; + +export interface WatchHistoryItem { + author: string; + watched: Date; + duration: number; + progress: number; + id: string; + title: string; + thumbnail: string; + videoId: string; +} diff --git a/materialious/src/lib/components/player/Player.svelte b/materialious/src/lib/components/player/Player.svelte index 2493a33f..ef786363 100644 --- a/materialious/src/lib/components/player/Player.svelte +++ b/materialious/src/lib/components/player/Player.svelte @@ -62,6 +62,7 @@ import { Network, type ConnectionStatus } from '@capacitor/network'; import { ScreenOrientation, type ScreenOrientationResult } from '@capacitor/screen-orientation'; import ClosedCaptions from './ClosedCaptions.svelte'; + import { getVideoWatchHistory, updateWatchHistory } from '$lib/api/backend/history'; interface Props { data: { video: VideoPlay; content: ParsedDescription; playlistId: string | null }; @@ -750,6 +751,9 @@ // Continue regardless of error } + const watchHistory = await getVideoWatchHistory(data.video.videoId); + if (watchHistory && watchHistory.progress > toSetTime) toSetTime = watchHistory.progress; + return toSetTime; } @@ -769,6 +773,8 @@ // Continue regardless of error } } + + updateWatchHistory(data.video.videoId, playerElement.currentTime); } onDestroy(async () => { diff --git a/materialious/src/lib/components/thumbnail/VideoThumbnail.svelte b/materialious/src/lib/components/thumbnail/VideoThumbnail.svelte index 7c8c4185..9b5c0f2b 100644 --- a/materialious/src/lib/components/thumbnail/VideoThumbnail.svelte +++ b/materialious/src/lib/components/thumbnail/VideoThumbnail.svelte @@ -20,6 +20,7 @@ syncPartyPeerStore } from '$lib/store'; import { relativeTimestamp } from '$lib/time'; + import { queueGetWatchHistory } from '$lib/api/backend/historyPool'; interface Props { video: VideoBase | Video | Notification | PlaylistPageVideo; @@ -72,6 +73,12 @@ calcThumbnailPlaceholderHeight(); }); + queueGetWatchHistory(video.videoId).then((watchHistory) => { + if (watchHistory) { + progress = watchHistory.progress.toString(); + } + }); + if (get(interfaceLowBandwidthMode)) return; let imageSrc = getBestThumbnail(video.videoThumbnails) as string; diff --git a/materialious/src/lib/server/database.ts b/materialious/src/lib/server/database.ts index b8de0aac..a21486e4 100644 --- a/materialious/src/lib/server/database.ts +++ b/materialious/src/lib/server/database.ts @@ -120,10 +120,22 @@ export function getSequelize(): { type: DataTypes.DATE, allowNull: false }, + duration: { + type: DataTypes.INTEGER, + allowNull: false + }, progress: { type: DataTypes.INTEGER, allowNull: false }, + videoIdCipher: { + type: DataTypes.STRING, + allowNull: false + }, + videoIdNonce: { + type: DataTypes.STRING, + allowNull: false + }, titleCipher: { type: DataTypes.STRING, allowNull: false @@ -147,14 +159,6 @@ export function getSequelize(): { thumbnailNonce: { type: DataTypes.STRING, allowNull: false - }, - durationCipher: { - type: DataTypes.STRING, - allowNull: false - }, - durationNonce: { - type: DataTypes.STRING, - allowNull: false } }); diff --git a/materialious/src/routes/api/user/history/+server.ts b/materialious/src/routes/api/user/history/+server.ts index 68ed2bda..e777b08e 100644 --- a/materialious/src/routes/api/user/history/+server.ts +++ b/materialious/src/routes/api/user/history/+server.ts @@ -4,9 +4,10 @@ import z from 'zod'; import { Op } from 'sequelize'; const zUserHistory = z.object({ - id: z.string(), + id: z.string().max(255), watched: z.coerce.date(), - progress: z.number(), + progress: z.number().max(115200), + duration: z.number().max(115200), title: z.object({ cipher: z.string().max(255), nonce: z.string().max(255) @@ -16,10 +17,10 @@ const zUserHistory = z.object({ nonce: z.string().max(255) }), thumbnail: z.object({ - cipher: z.string().max(255), + cipher: z.string().max(1000), nonce: z.string().max(255) }), - duration: z.object({ + videoId: z.object({ cipher: z.string().max(255), nonce: z.string().max(255) }) @@ -42,14 +43,15 @@ export async function POST({ locals, request }) { const toStore = { progress: data.data.progress, watched: data.data.watched, + duration: data.data.duration, titleCipher: data.data.title.cipher, titleNonce: data.data.title.nonce, authorCipher: data.data.author.cipher, authorNonce: data.data.author.nonce, thumbnailCipher: data.data.thumbnail.cipher, thumbnailNonce: data.data.thumbnail.nonce, - durationCipher: data.data.duration.cipher, - durationNonce: data.data.duration.nonce + videoIdCipher: data.data.videoId.cipher, + videoIdNonce: data.data.videoId.nonce }; if (history) { @@ -78,18 +80,20 @@ export async function GET({ locals, url }) { videoHashesList = videoHashes.split(','); } + console.log(videoHashesList); + const whereClause: any = { UserId: locals.userId }; if (videoHashesList.length > 0) { - whereClause[Op.or] = [{ id: { [Op.in]: videoHashesList } }]; + whereClause.id = { [Op.in]: videoHashesList }; } const history = await getSequelize().UserHistoryTable.findAll({ where: whereClause, limit, - offset: limit * page + offset: page > 0 ? limit * page : undefined }); return json(history); diff --git a/materialious/src/routes/api/user/history/[videoHash]/+server.ts b/materialious/src/routes/api/user/history/[videoHash]/+server.ts index ac2c92fc..2d827dd3 100644 --- a/materialious/src/routes/api/user/history/[videoHash]/+server.ts +++ b/materialious/src/routes/api/user/history/[videoHash]/+server.ts @@ -4,8 +4,8 @@ import z from 'zod'; const zUserHistoryUpdate = z .object({ - watched: z.date().optional(), - progress: z.number().optional() + watched: z.coerce.date().optional(), + progress: z.number().max(115200).optional() }) .transform((data) => { return Object.fromEntries(Object.entries(data).filter(([, v]) => v !== undefined)); @@ -15,10 +15,10 @@ export async function POST({ locals, request, params }) { if (!locals.userId) throw error(401, 'Unauthorized'); const data = zUserHistoryUpdate.safeParse(await request.json()); - if (!data.success) throw error(400); + if (!data.success) throw error(400, data.error.message); await getSequelize().UserHistoryTable.update(data.data, { - where: { UserId: locals.userId, VideoHash: params.videoHash } + where: { UserId: locals.userId, id: params.videoHash } }); return new Response(); @@ -28,7 +28,7 @@ export async function DELETE({ locals, params }) { if (!locals.userId) throw error(401, 'Unauthorized'); await getSequelize().UserHistoryTable.destroy({ - where: { UserId: locals.userId, VideoHash: params.videoHash } + where: { UserId: locals.userId, id: params.videoHash } }); return new Response(); @@ -38,7 +38,7 @@ export async function GET({ locals, params }) { if (!locals.userId) throw error(401, 'Unauthorized'); const history = await getSequelize().UserHistoryTable.findOne({ - where: { UserId: locals.userId, VideoHash: params.videoHash } + where: { UserId: locals.userId, id: params.videoHash } }); if (!history) throw error(404);