Improvements to YT backend feed

materialious/electron/package-lock.json

@@ -1,12 +1,12 @@
 {
 	"name": "Materialious",
-	"version": "1.14.3",
+	"version": "1.14.4",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "Materialious",
-			"version": "1.14.3",
+			"version": "1.14.4",
 			"license": "MIT",
 			"dependencies": {
 				"@capacitor-community/electron": "^5.0.0",

materialious/src/lib/api/index.ts

@@ -252,7 +252,7 @@ export async function getFeed(
 	fetchOptions: RequestInit = {}
 ): Promise<Feed> {
 	if (isYTBackend()) {
-		return getFeedYTjs();
+		return getFeedYTjs(maxResults, page);
 	}
 
 	const path = buildPath('auth/feed');

materialious/src/lib/api/youtubejs/subscriptions.ts

@@ -5,7 +5,11 @@ import { relativeTimestamp } from '$lib/time';
 import { get } from 'svelte/store';
 import type { Feed, Subscription, Thumbnail } from '../model';
 import { getChannelYTjs } from './channel';
-import { engineCooldownYTStore, engineCullYTStore } from '$lib/store';
+import {
+	engineCooldownYTStore,
+	engineCullYTStore,
+	engineMaxConcurrentChannelsStore
+} from '$lib/store';
 
 export async function getSubscriptionsYTjs(): Promise<Subscription[]> {
 	const subscriptions: Subscription[] = [];
@@ -134,19 +138,28 @@ export async function parseChannelRSS(channelId: string): Promise<void> {
 	}
 }
 
-export async function getFeedYTjs(): Promise<Feed> {
+export async function getFeedYTjs(maxResults: number, page: number): Promise<Feed> {
 	const channelSubscriptions = await localDb.channelSubscriptions.toArray();
 
 	const toUpdatePromises: Promise<void>[] = [];
 
 	const now = new Date();
+
+	let totalChannelsToParse = 0;
 	for (const channel of channelSubscriptions) {
 		const lastRSSFetch = new Date(channel.lastRSSFetch);
 		const timeDifference = now.getTime() - lastRSSFetch.getTime();
 		const cooldownTime = get(engineCooldownYTStore) * 60 * 60 * 1000;
+
 		if (timeDifference > cooldownTime) {
-			toUpdatePromises.push(parseChannelRSS(channel.channelId));
+			if (totalChannelsToParse < get(engineMaxConcurrentChannelsStore)) {
+				toUpdatePromises.push(parseChannelRSS(channel.channelId));
+			} else {
+				parseChannelRSS(channel.channelId);
+			}
 		}
+
+		totalChannelsToParse++;
 	}
 
 	if (toUpdatePromises) {
@@ -166,8 +179,11 @@ export async function getFeedYTjs(): Promise<Feed> {
 		videos = videos.slice(0, cullAfter);
 	}
 
+	const start = (page - 1) * maxResults;
+	const end = start + maxResults;
+
 	return {
 		notifications: [],
-		videos: videos
+		videos: videos.slice(start, end)
 	};
 }

materialious/src/lib/components/settings/Engine.svelte

@@ -6,6 +6,7 @@
 		engineCooldownYTStore,
 		engineCullYTStore,
 		engineFallbacksStore,
+		engineMaxConcurrentChannelsStore,
 		instanceStore
 	} from '$lib/store';
 	import { useEngineFallback, type EngineFallback } from '$lib/api/misc';
@@ -115,10 +116,22 @@
 				engineCooldownYTStore.set(Number((event.target as HTMLInputElement).value));
 			}}
 			value={$engineCooldownYTStore}
-			name="cull"
+			name="cooldown"
 			type="number"
 		/>
-		<label for="cull">{$_('layout.backendEngine.cooldown')}</label>
+		<label for="cooldown">{$_('layout.backendEngine.cooldown')}</label>
+	</div>
+	<div class="field label prefix border">
+		<i>pending</i>
+		<input
+			oninput={(event: Event) => {
+				engineMaxConcurrentChannelsStore.set(Number((event.target as HTMLInputElement).value));
+			}}
+			value={$engineMaxConcurrentChannelsStore}
+			name="concurrent"
+			type="number"
+		/>
+		<label for="concurrent">{$_('layout.backendEngine.concurrent')}</label>
 	</div>
 
 	{#if $authStore && $instanceStore}

materialious/src/lib/i18n/locales/en.json

@@ -179,6 +179,7 @@
 			"warning": "Advanced settings! For experienced users only. Do not change unless you know what you’re doing.",
 			"cull": "Max feed items",
 			"cooldown": "Re-fetch channel hour cooldown",
+			"concurrent": "Max amount of channels to parse before feed return",
 			"fallbacks": "Fallbacks",
 			"importExport": "Import/Export subscriptions",
 			"exportToInvidious": "Export to Invidious from Materialious",

materialious/src/lib/store.ts

@@ -324,6 +324,11 @@ export const engineCooldownYTStore: Writable<number> = persist(
 	createStorage(),
 	'engineCooldownYT'
 );
+export const engineMaxConcurrentChannelsStore: Writable<number> = persist(
+	writable(100),
+	createStorage(),
+	'engineMaxConcurrentChannels'
+);
 
 export const engineFallbacksStore: Writable<EngineFallback[]> = persist(
 	writable([]),

materialious/src/routes/(app)/subscriptions/+page.svelte

@@ -6,16 +6,13 @@
 	import ItemsList from '$lib/components/ItemsList.svelte';
 	import { resolve } from '$app/paths';
 	import { _ } from '$lib/i18n';
-	import { isYTBackend } from '$lib/misc';
 
 	let currentPage = 1;
 	let videos: (VideoBase | Video | PlaylistPageVideo)[] = $state($feedCacheStore.subscription);
 
 	async function loadMore(event: InfiniteEvent) {
-		// Not supported or needed on YT backend
-		if (isYTBackend()) return;
-
 		currentPage++;
+
 		const feed = await getFeed(100, currentPage);
 		if (feed.videos.length === 0) {
 			event.detail.complete();