From 302b4cb5c99fe00f21b5b775ef2187f4088717a9 Mon Sep 17 00:00:00 2001 From: Viren070 Date: Mon, 16 Jun 2025 14:13:41 +0100 Subject: [PATCH] feat: implement advanced stream filtering with excluded conditions closes #57 --- package-lock.json | 11 + packages/core/package.json | 2 + packages/core/src/db/schemas.ts | 4 + packages/core/src/main.ts | 41 ++- packages/core/src/parser/conditions.ts | 318 +++++++++++++++--- packages/core/src/utils/config.ts | 18 +- .../frontend/src/components/menu/filters.tsx | 105 +++++- 7 files changed, 448 insertions(+), 51 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7fc58591..87d1faf9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3644,6 +3644,13 @@ "@types/node": "*" } }, + "node_modules/@types/bytes": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/@types/bytes/-/bytes-3.1.5.tgz", + "integrity": "sha512-VgZkrJckypj85YxEsEavcMmmSOIzkUHqWmM4CCyia5dc54YwsXzJ5uT4fYxBQNEXx+oF1krlhgCbvfubXqZYsQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/connect": { "version": "3.4.38", "dev": true, @@ -5250,6 +5257,8 @@ }, "node_modules/bytes": { "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", "license": "MIT", "engines": { "node": ">= 0.8" @@ -14356,6 +14365,7 @@ "version": "0.0.0", "dependencies": { "bcrypt": "^6.0.0", + "bytes": "^3.1.2", "dotenv": "^16.4.7", "envalid": "^8.0.0", "expr-eval": "^2.0.2", @@ -14371,6 +14381,7 @@ }, "devDependencies": { "@types/bcrypt": "^5.0.2", + "@types/bytes": "^3.1.5", "@types/node": "^20.14.10", "@types/pg": "^8.15.2" } diff --git a/packages/core/package.json b/packages/core/package.json index 0f8bbe7d..7b2b4da9 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -10,6 +10,7 @@ "description": "Combine all your streams into one addon and display them with consistent formatting, sorting, and filtering.", "dependencies": { "bcrypt": "^6.0.0", + "bytes": "^3.1.2", "dotenv": "^16.4.7", "envalid": "^8.0.0", "expr-eval": "^2.0.2", @@ -25,6 +26,7 @@ }, "devDependencies": { "@types/bcrypt": "^5.0.2", + "@types/bytes": "^3.1.5", "@types/node": "^20.14.10", "@types/pg": "^8.15.2" } diff --git a/packages/core/src/db/schemas.ts b/packages/core/src/db/schemas.ts index 338334f3..45f3a85c 100644 --- a/packages/core/src/db/schemas.ts +++ b/packages/core/src/db/schemas.ts @@ -331,6 +331,7 @@ export const UserDataSchema = z.object({ excludeUncachedFromServices: z.array(z.string().min(1)).optional(), excludeUncachedFromStreamTypes: z.array(StreamTypes).optional(), excludeUncachedMode: z.enum(['or', 'and']).optional(), + excludedFilterConditions: z.array(z.string().min(1).max(1000)).optional(), groups: z .array( z.object({ @@ -683,7 +684,10 @@ export const ParsedStreamSchema = z.object({ originalDescription: z.string().optional(), }); +export const ParsedStreams = z.array(ParsedStreamSchema); + export type ParsedStream = z.infer; +export type ParsedStreams = z.infer; export const AIOStream = StreamSchema.extend({ streamData: z.object({ diff --git a/packages/core/src/main.ts b/packages/core/src/main.ts index 172cd930..7662243d 100644 --- a/packages/core/src/main.ts +++ b/packages/core/src/main.ts @@ -42,7 +42,10 @@ import { safeRegexTest, } from './utils/regex'; import { isMatch } from 'super-regex'; -import { ConditionParser } from './parser/conditions'; +import { + GroupConditionParser, + SelectConditionParser, +} from './parser/conditions'; import { RPDB } from './utils/rpdb'; import { FeatureControl } from './utils/feature'; const logger = createLogger('core'); @@ -1092,7 +1095,7 @@ ${errorStreams.length > 0 ? ` ❌ Errors : ${errorStreams.map((s) => ` if (!group.condition || !group.addons.length) continue; try { - const parser = new ConditionParser( + const parser = new GroupConditionParser( previousGroupStreams, parsedStreams, previousGroupTimeTaken, @@ -1207,6 +1210,7 @@ ${errorStreams.length > 0 ? ` ❌ Errors : ${errorStreams.map((s) => ` requiredKeywords: { total: 0, details: {} }, requiredSeeders: { total: 0, details: {} }, excludedSeeders: { total: 0, details: {} }, + excludedFilterCondition: { total: 0, details: {} }, size: { total: 0, details: {} }, }; @@ -2035,7 +2039,38 @@ ${errorStreams.length > 0 ? ` ❌ Errors : ${errorStreams.map((s) => ` }; const filterResults = await Promise.all(streams.map(shouldKeepStream)); - const filteredStreams = streams.filter((_, index) => filterResults[index]); + + let filteredStreams = streams.filter((_, index) => filterResults[index]); + + if (this.userData.excludedFilterConditions) { + const parser = new SelectConditionParser(); + + for (const condition of this.userData.excludedFilterConditions) { + try { + const selectedStreams = await parser.select( + filteredStreams, + condition + ); + + // Remove these streams from filteredStreams + filteredStreams = filteredStreams.filter( + (stream) => !selectedStreams.includes(stream) + ); + + // Update skip reasons for this condition + if (selectedStreams.length > 0) { + skipReasons.excludedFilterCondition.total += selectedStreams.length; + skipReasons.excludedFilterCondition.details[condition] = + selectedStreams.length; + } + } catch (error) { + logger.error( + `Failed to apply excluded filter condition "${condition}": ${error instanceof Error ? error.message : String(error)}` + ); + // Continue with the next condition instead of breaking the entire loop + } + } + } // Log filter summary const totalFiltered = streams.length - filteredStreams.length; diff --git a/packages/core/src/parser/conditions.ts b/packages/core/src/parser/conditions.ts index dccb42a3..2e57fa54 100644 --- a/packages/core/src/parser/conditions.ts +++ b/packages/core/src/parser/conditions.ts @@ -1,25 +1,11 @@ import { Parser } from 'expr-eval'; -import { ParsedStream } from '../db'; +import { ParsedStream, ParsedStreams, ParsedStreamSchema } from '../db'; +import bytes from 'bytes'; -export class ConditionParser { - private parser: Parser; - private previousStreams: ParsedStream[]; - private totalStreams: ParsedStream[]; - private previousGroupTimeTaken: number; - private totalTimeTaken: number; - - constructor( - previousStreams: ParsedStream[], - totalStreams: ParsedStream[], - previousGroupTimeTaken: number, - totalTimeTaken: number, - queryType: string - ) { - this.previousStreams = previousStreams; - this.totalStreams = totalStreams; - this.previousGroupTimeTaken = previousGroupTimeTaken; - this.totalTimeTaken = totalTimeTaken; +export abstract class BaseConditionParser { + protected parser: Parser; + constructor() { // only allow comparison and logical operators this.parser = new Parser({ operators: { @@ -72,12 +58,10 @@ export class ConditionParser { }, }); - this.parser.consts.previousStreams = this.previousStreams; - this.parser.consts.totalStreams = this.totalStreams; - this.parser.consts.queryType = queryType; - this.parser.consts.previousGroupTimeTaken = this.previousGroupTimeTaken; - this.parser.consts.totalTimeTaken = this.totalTimeTaken; + this.setupParserFunctions(); + } + private setupParserFunctions() { this.parser.functions.regexMatched = function ( streams: ParsedStream[], regexName?: string @@ -88,27 +72,25 @@ export class ConditionParser { : stream.regexMatched ); }; + this.parser.functions.indexer = function ( streams: ParsedStream[], indexer: string ) { - if (!Array.isArray(streams)) { - throw new Error( - "Please use one of 'totalStreams' or 'previousStreams' as the first argument" - ); + if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) { + throw new Error('Your streams input must be an array of streams'); } else if (typeof indexer !== 'string') { throw new Error('Indexer must be a string'); } return streams.filter((stream) => stream.indexer === indexer); }; + this.parser.functions.resolution = function ( streams: ParsedStream[], resolution: string ) { - if (!Array.isArray(streams)) { - throw new Error( - "Please use one of 'totalStreams' or 'previousStreams' as the first argument" - ); + if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) { + throw new Error('Your streams input must be an array of streams'); } else if (typeof resolution !== 'string') { throw new Error('Resolution must be a string'); } @@ -116,14 +98,13 @@ export class ConditionParser { (stream) => (stream.parsedFile?.resolution || 'Unknown') === resolution ); }; + this.parser.functions.quality = function ( streams: ParsedStream[], quality: string ) { - if (!Array.isArray(streams)) { - throw new Error( - "Please use one of 'totalStreams' or 'previousStreams' as the first argument" - ); + if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) { + throw new Error('Your streams input must be an array of streams'); } else if (typeof quality !== 'string') { throw new Error('Quality must be a string'); } @@ -131,27 +112,167 @@ export class ConditionParser { (stream) => (stream.parsedFile?.quality || 'Unknown') === quality ); }; + + this.parser.functions.encode = function ( + streams: ParsedStream[], + encode: string + ) { + if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) { + throw new Error('Your streams input must be an array of streams'); + } else if (typeof encode !== 'string') { + throw new Error('Encode must be a string'); + } + return streams.filter((stream) => stream.parsedFile?.encode === encode); + }; + this.parser.functions.type = function ( streams: ParsedStream[], type: string ) { if (!Array.isArray(streams)) { - throw new Error( - "Please use one of 'totalStreams' or 'previousStreams' as the first argument" - ); + throw new Error('Your streams input must be an array of streams'); } else if (typeof type !== 'string') { throw new Error('Type must be a string'); } return streams.filter((stream) => stream.type === type); }; + + this.parser.functions.visualTag = function ( + streams: ParsedStream[], + visualTag: string + ) { + if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) { + throw new Error('Your streams input must be an array of streams'); + } else if (typeof visualTag !== 'string') { + throw new Error('Visual type must be a string'); + } + return streams.filter((stream) => + stream.parsedFile?.visualTags.includes(visualTag) + ); + }; + + this.parser.functions.audioTag = function ( + streams: ParsedStream[], + audioTag: string + ) { + if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) { + throw new Error('Your streams input must be an array of streams'); + } else if (typeof audioTag !== 'string') { + throw new Error('Audio tag must be a string'); + } + return streams.filter((stream) => + stream.parsedFile?.audioTags.includes(audioTag) + ); + }; + + this.parser.functions.audioChannels = function ( + streams: ParsedStream[], + audioChannels: string + ) { + if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) { + throw new Error('Your streams input must be an array of streams'); + } else if (typeof audioChannels !== 'string') { + throw new Error('Audio channels must be a string'); + } + return streams.filter((stream) => + stream.parsedFile?.audioChannels?.includes(audioChannels) + ); + }; + + this.parser.functions.language = function ( + streams: ParsedStream[], + language: string + ) { + if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) { + throw new Error('Your streams input must be an array of streams'); + } else if (typeof language !== 'string') { + throw new Error('Language must be a string'); + } + return streams.filter((stream) => + stream.parsedFile?.languages?.includes(language) + ); + }; + + this.parser.functions.seeders = function ( + streams: ParsedStream[], + minSeeders?: number, + maxSeeders?: number + ) { + if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) { + throw new Error('Your streams input must be an array of streams'); + } else if ( + typeof minSeeders !== 'number' && + typeof maxSeeders !== 'number' + ) { + throw new Error('Min and max seeders must be a number'); + } + // select streams with seeders that lie within the range. + return streams.filter((stream) => { + if ( + minSeeders && + stream.torrent?.seeders && + stream.torrent.seeders < minSeeders + ) { + return false; + } + if ( + maxSeeders && + stream.torrent?.seeders && + stream.torrent.seeders > maxSeeders + ) { + return false; + } + return true; + }); + }; + + this.parser.functions.size = function ( + streams: ParsedStream[], + minSize?: string | number, + maxSize?: string | number + ) { + if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) { + throw new Error('Your streams input must be an array of streams'); + } else if ( + typeof minSize !== 'number' && + typeof maxSize !== 'number' && + typeof minSize !== 'string' && + typeof maxSize !== 'string' + ) { + throw new Error('Min and max size must be a number'); + } + // use the bytes library to ensure we get a number + const minSizeInBytes = + typeof minSize === 'string' ? bytes.parse(minSize) : minSize; + const maxSizeInBytes = + typeof maxSize === 'string' ? bytes.parse(maxSize) : maxSize; + return streams.filter((stream) => { + if ( + minSize && + stream.size && + minSizeInBytes && + stream.size < minSizeInBytes + ) { + return false; + } + if ( + maxSize && + stream.size && + maxSizeInBytes && + stream.size > maxSizeInBytes + ) { + return false; + } + return true; + }); + }; + this.parser.functions.service = function ( streams: ParsedStream[], service: string ) { - if (!Array.isArray(streams)) { - throw new Error( - "Please use one of 'totalStreams' or 'previousStreams' as the first argument" - ); + if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) { + throw new Error('Your streams input must be an array of streams'); } else if ( typeof service !== 'string' || ![ @@ -173,6 +294,7 @@ export class ConditionParser { } return streams.filter((stream) => stream.service?.id === service); }; + this.parser.functions.cached = function (streams: ParsedStream[]) { if (!Array.isArray(streams)) { throw new Error( @@ -181,6 +303,7 @@ export class ConditionParser { } return streams.filter((stream) => stream.service?.cached === true); }; + this.parser.functions.uncached = function (streams: ParsedStream[]) { if (!Array.isArray(streams)) { throw new Error( @@ -189,6 +312,7 @@ export class ConditionParser { } return streams.filter((stream) => stream.service?.cached === false); }; + this.parser.functions.releaseGroup = function ( streams: ParsedStream[], releaseGroup: string @@ -204,6 +328,26 @@ export class ConditionParser { (stream) => stream.parsedFile?.releaseGroup === releaseGroup ); }; + + this.parser.functions.addon = function ( + streams: ParsedStream[], + addon: string + ) { + if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) { + throw new Error('Your streams input must be an array of streams'); + } else if (typeof addon !== 'string') { + throw new Error('Addon must be a string'); + } + return streams.filter((stream) => stream.addon.name === addon); + }; + + this.parser.functions.library = function (streams: ParsedStream[]) { + if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) { + throw new Error('Your streams input must be an array of streams'); + } + return streams.filter((stream) => stream.library); + }; + this.parser.functions.count = function (streams: ParsedStream[]) { if (!Array.isArray(streams)) { throw new Error( @@ -213,7 +357,8 @@ export class ConditionParser { return streams.length; }; } - async parse(condition: string) { + + protected async evaluateCondition(condition: string): Promise { return new Promise((resolve, reject) => { const timeout = setTimeout(() => { reject(new Error('Condition parsing timed out')); @@ -229,9 +374,92 @@ export class ConditionParser { } }); } +} + +export class GroupConditionParser extends BaseConditionParser { + private previousStreams: ParsedStream[]; + private totalStreams: ParsedStream[]; + private previousGroupTimeTaken: number; + private totalTimeTaken: number; + + constructor( + previousStreams: ParsedStream[], + totalStreams: ParsedStream[], + previousGroupTimeTaken: number, + totalTimeTaken: number, + queryType: string + ) { + super(); + + this.previousStreams = previousStreams; + this.totalStreams = totalStreams; + this.previousGroupTimeTaken = previousGroupTimeTaken; + this.totalTimeTaken = totalTimeTaken; + + // Set up constants for this specific parser + this.parser.consts.previousStreams = this.previousStreams; + this.parser.consts.totalStreams = this.totalStreams; + this.parser.consts.queryType = queryType; + this.parser.consts.previousGroupTimeTaken = this.previousGroupTimeTaken; + this.parser.consts.totalTimeTaken = this.totalTimeTaken; + } + + async parse(condition: string) { + return await this.evaluateCondition(condition); + } static async testParse(condition: string) { - const parser = new ConditionParser([], [], 0, 0, 'movie'); + const parser = new GroupConditionParser([], [], 0, 0, 'movie'); return await parser.parse(condition); } } + +export class SelectConditionParser extends BaseConditionParser { + constructor() { + super(); + } + + async select( + streams: ParsedStream[], + condition: string + ): Promise { + // Set the streams constant for this filter operation + this.parser.consts.streams = streams; + let selectedStreams: ParsedStream[] = []; + + try { + selectedStreams = await this.evaluateCondition(condition); + } catch (error) { + throw new Error( + `Filter condition failed: ${error instanceof Error ? error.message : String(error)}` + ); + } + + // attempt to parse the result + try { + selectedStreams = ParsedStreams.parse(selectedStreams); + } catch (error) { + throw new Error( + `Filter condition failed: ${error instanceof Error ? error.message : String(error)}` + ); + } + return selectedStreams; + // // If the result is an array of streams, return those that should be filtered out + // // use ParsedResultSchema to validate + // if (selectedStreams.length > 0) { + // // Filter out the selected streams from the input array + // return streams.filter((stream) => !selectedStreams.includes(stream)); + // } + + // // If the result is not a stream array, return the original streams + // return streams; + } + + static async testSelect( + streams: ParsedStream[], + condition: string + ): Promise { + const parser = new SelectConditionParser(); + return await parser.select(streams, condition); + } +} diff --git a/packages/core/src/utils/config.ts b/packages/core/src/utils/config.ts index ddce3dfe..b4cd62c9 100644 --- a/packages/core/src/utils/config.ts +++ b/packages/core/src/utils/config.ts @@ -15,7 +15,10 @@ import { isEncrypted, decryptString, encryptString } from './crypto'; import { Env } from './env'; import { createLogger, maskSensitiveInfo } from './logger'; import { ZodError } from 'zod'; -import { ConditionParser } from '../parser/conditions'; +import { + GroupConditionParser, + SelectConditionParser, +} from '../parser/conditions'; import { RPDB } from './rpdb'; import { FeatureControl } from './feature'; import { compileRegex } from './regex'; @@ -277,6 +280,17 @@ export async function validateConfig( } } + // validate excluded filter condition + if (config.excludedFilterConditions) { + for (const condition of config.excludedFilterConditions) { + try { + await SelectConditionParser.testSelect([], condition); + } catch (error) { + throw new Error(`Invalid excluded filter condition: ${error}`); + } + } + } + if (config.services) { config.services = config.services.map((service: Service) => validateService(service, decryptValues) @@ -462,7 +476,7 @@ async function validateGroup(group: Group) { // we must be able to parse the condition let result; try { - result = await ConditionParser.testParse(group.condition); + result = await GroupConditionParser.testParse(group.condition); } catch (error: any) { throw new Error( `Your group condition - '${group.condition}' - is invalid: ${error.message}` diff --git a/packages/frontend/src/components/menu/filters.tsx b/packages/frontend/src/components/menu/filters.tsx index a5d6c60e..fc106fee 100644 --- a/packages/frontend/src/components/menu/filters.tsx +++ b/packages/frontend/src/components/menu/filters.tsx @@ -24,6 +24,7 @@ import { MdMovieFilter, MdPerson, MdSurroundSound, + MdTextFields, MdVideoLibrary, } from 'react-icons/md'; import { BiSolidCameraMovie } from 'react-icons/bi'; @@ -76,6 +77,7 @@ import { Modal } from '../ui/modal'; import { useDisclosure } from '@/hooks/disclosure'; import { toast } from 'sonner'; import { Slider } from '../ui/slider/slider'; +import { TbFilterCode } from 'react-icons/tb'; type Resolution = (typeof RESOLUTIONS)[number]; type Quality = (typeof QUALITIES)[number]; @@ -274,9 +276,13 @@ function Content() { Matching - + Keyword + + + Condition + {status?.settings.regexFilterAccess !== 'none' && ( @@ -1308,6 +1314,103 @@ function Content() { + + + +
+

+ Create advanced filters to exclude specific streams from your + results using condition expressions. Write conditions that + evaluate which streams to remove based on properties like + addon type, quality, size, or any other stream attributes. + Multiple conditions can be combined using logical operators + for precise filtering control. +

+
+
+ +
+

+ This filter uses the same expression syntax as the Group + Condition Parser, but operates on a single{' '} + streams constant containing all available + streams. +

+
+

How it works:

+
    +
  • + Your condition should return an array of streams +
    • +
  • + Streams returned by the condition will be{' '} + excluded from results +
    • +
  • + Use functions like addon(),{' '} + type(), quality() to filter + streams +
    • +
  • + Combine conditions together by nesting functions + together, for example: + + addon(type(streams, 'debrid'), 'TorBox') + {' '} + excludes all TorBox debrid streams and keeps its + usenet streams. +
    • +
+
+

+ Example:{' '} + addon(type(streams, 'debrid'), 'TorBox'){' '} + excludes all TorBox debrid streams. +

+

+ For detailed syntax and available functions, see the{' '} + + Wiki page on Groups + +

+
+ + { + setUserData((prev) => ({ + ...prev, + excludedFilterConditions: values, + })); + }} + onValueChange={(value, index) => { + setUserData((prev) => ({ + ...prev, + excludedFilterConditions: [ + ...(prev.excludedFilterConditions || []).slice( + 0, + index + ), + value, + ...(prev.excludedFilterConditions || []).slice( + index + 1 + ), + ], + })); + }} + /> +
+ +