diff --git a/packages/core/src/db/schemas.ts b/packages/core/src/db/schemas.ts index 3e7c282b..0ba7ecbb 100644 --- a/packages/core/src/db/schemas.ts +++ b/packages/core/src/db/schemas.ts @@ -360,14 +360,28 @@ export const UserDataSchema = z.object({ requiredStreamExpressions: z.array(z.string().min(1).max(3000)).optional(), preferredStreamExpressions: z.array(z.string().min(1).max(3000)).optional(), includedStreamExpressions: z.array(z.string().min(1).max(3000)).optional(), - disableGroups: z.boolean().optional(), + // disableGroups: z.boolean().optional(), + // groups: z + // .array( + // z.object({ + // addons: z.array(z.string().min(1)), + // condition: z.string().min(1).max(200), + // }) + // ) + // .optional(), groups: z - .array( - z.object({ - addons: z.array(z.string().min(1)), - condition: z.string().min(1).max(200), - }) - ) + .object({ + enabled: z.boolean().optional(), + groupings: z + .array( + z.object({ + addons: z.array(z.string().min(1)), + condition: z.string().min(1).max(200), + }) + ) + .optional(), + behaviour: z.enum(['sequential', 'parallel']).optional(), + }) .optional(), sortCriteria: z.object({ // global must be defined. diff --git a/packages/core/src/db/users.ts b/packages/core/src/db/users.ts index 09719ceb..a8429461 100644 --- a/packages/core/src/db/users.ts +++ b/packages/core/src/db/users.ts @@ -15,6 +15,7 @@ import { verifyHash, validateConfig, formatZodError, + applyMigrations, } from '../utils'; const APIError = constants.APIError; @@ -193,7 +194,7 @@ export class UserRepository { Env.TRUSTED_UUIDS?.split(',').some((u) => new RegExp(u).test(uuid)) ?? false; logger.info(`Retrieved configuration for user ${uuid}`); - return decryptedConfig; + return applyMigrations(decryptedConfig); } catch (error) { logger.error( `Error retrieving user ${uuid}: ${error instanceof Error ? error.message : String(error)}` diff --git a/packages/core/src/streams/fetcher.ts b/packages/core/src/streams/fetcher.ts index f42d0aba..fe863080 100644 --- a/packages/core/src/streams/fetcher.ts +++ b/packages/core/src/streams/fetcher.ts @@ -168,52 +168,54 @@ class StreamFetcher { // If groups are configured, handle group-based fetching if ( - this.userData.groups && - this.userData.groups.length > 0 && - this.userData.disableGroups !== true + this.userData.groups?.groupings && + this.userData.groups.groupings.length > 0 && + this.userData.groups.enabled !== false ) { // add addons that are not assigned to any group to the first group const unassignedAddons = addons.filter( (addon) => - !this.userData.groups!.some((group) => + !this.userData.groups?.groupings?.some((group) => group.addons.includes(addon.preset.id) ) ); - if (unassignedAddons.length > 0) { - this.userData.groups[0].addons.push( + if (unassignedAddons.length > 0 && this.userData.groups.groupings[0]) { + this.userData.groups.groupings[0].addons.push( ...unassignedAddons.map((addon) => addon.preset.id) ); } - const groupPromises = this.userData.groups.map((group) => { - const groupAddons = addons.filter( - (addon) => addon.preset.id && group.addons.includes(addon.preset.id) - ); - logger.info( - `Queueing fetch for group with ${groupAddons.length} addons.` - ); - return fetchFromGroup(groupAddons); - }); + const behaviour = this.userData.groups.behaviour || 'parallel'; let totalTimeTaken = 0; let previousGroupStreams: ParsedStream[] = []; let previousGroupTimeTaken = 0; - for (let i = 0; i < groupPromises.length; i++) { - const groupResult = await groupPromises[i]; - const group = this.userData.groups[i]; + if (behaviour === 'parallel') { + // Fetch all groups in parallel but still evaluate conditions + const groupPromises = this.userData.groups.groupings.map((group) => { + const groupAddons = addons.filter( + (addon) => addon.preset.id && group.addons.includes(addon.preset.id) + ); + logger.info( + `Queueing parallel fetch for group with ${groupAddons.length} addons.` + ); + return fetchFromGroup(groupAddons); + }); - if (i === 0) { - allStreams.push(...groupResult.streams); - allErrors.push(...groupResult.errors); - allStatisticStreams.push(...groupResult.statistics); - totalTimeTaken = groupResult.totalTime; - previousGroupStreams = groupResult.streams; - previousGroupTimeTaken = groupResult.totalTime; + for (let i = 0; i < this.userData.groups.groupings.length; i++) { + const groupResult = await groupPromises[i]; + const group = this.userData.groups.groupings[i]; - // After the first group, check the condition for the second group - if (groupPromises.length > 1) { - const nextGroup = this.userData.groups[1]; - if (!nextGroup.condition || !nextGroup.addons.length) continue; + if (i === 0) { + allStreams.push(...groupResult.streams); + allErrors.push(...groupResult.errors); + allStatisticStreams.push(...groupResult.statistics); + totalTimeTaken = groupResult.totalTime; + previousGroupStreams = groupResult.streams; + previousGroupTimeTaken = groupResult.totalTime; + } else { + // For groups other than the first, check their condition + if (!group.condition || !group.addons.length) continue; const evaluator = new GroupConditionEvaluator( previousGroupStreams, @@ -222,46 +224,64 @@ class StreamFetcher { totalTimeTaken, queryType ); - const shouldFetchNext = await evaluator.evaluate( - nextGroup.condition - ); + const shouldInclude = await evaluator.evaluate(group.condition); - if (!shouldFetchNext) { + if (shouldInclude) { logger.info( - `Condition not met for group 2 based on group 1 results. Halting further processing.` + `Condition met for parallel group ${i + 1}, including streams.` + ); + allStreams.push(...groupResult.streams); + allErrors.push(...groupResult.errors); + allStatisticStreams.push(...groupResult.statistics); + totalTimeTaken = Math.max(totalTimeTaken, groupResult.totalTime); + previousGroupStreams = groupResult.streams; + previousGroupTimeTaken = groupResult.totalTime; + } else { + logger.info( + `Condition not met for parallel group ${i + 1}, skipping streams.` ); - break; // Exit the loop, returning only group 1 streams } } - } else { - // For groups other than the first, check their condition before processing - if (!group.condition || !group.addons.length) continue; + } + } else { + // Sequential behavior - fetch and evaluate one group at a time + for (let i = 0; i < this.userData.groups.groupings.length; i++) { + const group = this.userData.groups.groupings[i]; - const evaluator = new GroupConditionEvaluator( - previousGroupStreams, - allStreams, - previousGroupTimeTaken, - totalTimeTaken, - queryType - ); - const shouldFetch = await evaluator.evaluate(group.condition); + // For groups after the first, check condition before fetching + if (i > 0 && group.condition) { + const evaluator = new GroupConditionEvaluator( + previousGroupStreams, + allStreams, + previousGroupTimeTaken, + totalTimeTaken, + queryType + ); + const shouldFetch = await evaluator.evaluate(group.condition); - if (shouldFetch) { - logger.info( - `Condition met for group ${i + 1}, processing streams.` - ); - allStreams.push(...groupResult.streams); - allErrors.push(...groupResult.errors); - allStatisticStreams.push(...groupResult.statistics); - totalTimeTaken += groupResult.totalTime; - previousGroupStreams = groupResult.streams; - previousGroupTimeTaken = groupResult.totalTime; - } else { - logger.info( - `Condition not met for group ${i + 1}, skipping remaining groups.` - ); - break; // Stop processing any more groups + if (!shouldFetch) { + logger.info( + `Condition not met for sequential group ${i + 1}, stopping.` + ); + break; + } } + + const groupAddons = addons.filter( + (addon) => addon.preset.id && group.addons.includes(addon.preset.id) + ); + logger.info( + `Fetching from sequential group ${i + 1} with ${groupAddons.length} addons.` + ); + + const groupResult = await fetchFromGroup(groupAddons); + + allStreams.push(...groupResult.streams); + allErrors.push(...groupResult.errors); + allStatisticStreams.push(...groupResult.statistics); + totalTimeTaken += groupResult.totalTime; + previousGroupStreams = groupResult.streams; + previousGroupTimeTaken = groupResult.totalTime; } } } else { diff --git a/packages/core/src/utils/config.ts b/packages/core/src/utils/config.ts index ae81bf6f..c92386fd 100644 --- a/packages/core/src/utils/config.ts +++ b/packages/core/src/utils/config.ts @@ -352,8 +352,8 @@ export async function validateConfig( } } - if (config.groups) { - for (const group of config.groups) { + if (config.groups?.groupings) { + for (const group of config.groups.groupings) { await validateGroup(group); } } @@ -456,8 +456,8 @@ function removeInvalidPresetReferences(config: UserData) { existingPresetIds?.includes(addon) ); } - if (config.groups) { - config.groups = config.groups.map((group) => ({ + if (config.groups?.groupings) { + config.groups.groupings = config.groups.groupings.map((group) => ({ ...group, addons: group.addons?.filter((addon) => existingPresetIds?.includes(addon) @@ -467,7 +467,7 @@ function removeInvalidPresetReferences(config: UserData) { return config; } -export function applyMigrations(config: UserData): UserData { +export function applyMigrations(config: any): UserData { if ( config.deduplicator && typeof config.deduplicator.multiGroupBehaviour === 'string' @@ -495,6 +495,14 @@ export function applyMigrations(config: UserData): UserData { }; delete config.titleMatching.matchYear; } + + if (Array.isArray(config.groups)) { + config.groups = { + enabled: config.disableGroups ? false : true, + groupings: config.groups, + behaviour: 'parallel', + }; + } return config; } diff --git a/packages/frontend/src/components/menu/addons.tsx b/packages/frontend/src/components/menu/addons.tsx index cd9febda..a99ddd0b 100644 --- a/packages/frontend/src/components/menu/addons.tsx +++ b/packages/frontend/src/components/menu/addons.tsx @@ -74,6 +74,7 @@ import { IoExtensionPuzzle } from 'react-icons/io5'; import { NumberInput } from '../ui/number-input'; import { useDisclosure } from '@/hooks/disclosure'; import { useMode } from '@/context/mode'; +import { Select } from '../ui/select'; interface CatalogModification { id: string; @@ -1191,7 +1192,7 @@ function AddonGroupCard() { // Helper function to get presets that are not in any group except the current one const getAvailablePresets = (currentGroupIndex: number) => { const presetsInOtherGroups = new Set( - userData.groups?.flatMap((group, idx) => + userData.groups?.groupings?.flatMap((group, idx) => idx !== currentGroupIndex ? group.addons : [] ) || [] ); @@ -1213,7 +1214,7 @@ function AddonGroupCard() { ) => { setUserData((prev) => { // Initialize groups array if it doesn't exist - const currentGroups = prev.groups || []; + const currentGroups = prev.groups?.groupings || []; // Create a new array with all existing groups const newGroups = [...currentGroups]; @@ -1231,7 +1232,10 @@ function AddonGroupCard() { return { ...prev, - groups: newGroups, + groups: { + ...prev.groups, + groupings: newGroups, + }, }; }); }; @@ -1259,21 +1263,46 @@ function AddonGroupCard() { for a detailed guide to using groups. { - setUserData((prev) => ({ ...prev, disableGroups: value })); + setUserData((prev) => ({ + ...prev, + groups: { ...prev.groups, enabled: value }, + })); }} side="right" - help="If enabled, groups will be ignored and all addons will be used." /> - {(userData.groups || []).map((group, index) => ( +