Viren070/AIOStreams
1
0
Fork 0
mirror of https://github.com/Viren070/AIOStreams.git synced 2025-12-01 23:14:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement advanced stream filtering with excluded conditions

Browse Source 
closes #57
This commit is contained in:
Viren070
2025-06-16 14:13:41 +01:00
parent 3779ea09d3
commit 302b4cb5c9
7 changed files with 448 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
+11
View File
@@ -3644,6 +3644,13 @@
"@types/node": "*"
}
},
"node_modules/@types/bytes": {
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/@types/bytes/-/bytes-3.1.5.tgz",
"integrity": "sha512-VgZkrJckypj85YxEsEavcMmmSOIzkUHqWmM4CCyia5dc54YwsXzJ5uT4fYxBQNEXx+oF1krlhgCbvfubXqZYsQ==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/connect": {
"version": "3.4.38",
"dev": true,
@@ -5250,6 +5257,8 @@
},
"node_modules/bytes": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
"integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
@@ -14356,6 +14365,7 @@
"version": "0.0.0",
"dependencies": {
"bcrypt": "^6.0.0",
"bytes": "^3.1.2",
"dotenv": "^16.4.7",
"envalid": "^8.0.0",
"expr-eval": "^2.0.2",
@@ -14371,6 +14381,7 @@
},
"devDependencies": {
"@types/bcrypt": "^5.0.2",
"@types/bytes": "^3.1.5",
"@types/node": "^20.14.10",
"@types/pg": "^8.15.2"
}
packages/core/package.json
+2
View File
@@ -10,6 +10,7 @@
"description": "Combine all your streams into one addon and display them with consistent formatting, sorting, and filtering.",
"dependencies": {
"bcrypt": "^6.0.0",
"bytes": "^3.1.2",
"dotenv": "^16.4.7",
"envalid": "^8.0.0",
"expr-eval": "^2.0.2",
@@ -25,6 +26,7 @@
},
"devDependencies": {
"@types/bcrypt": "^5.0.2",
"@types/bytes": "^3.1.5",
"@types/node": "^20.14.10",
"@types/pg": "^8.15.2"
}
packages/core/src/db/schemas.ts
+4
View File
@@ -331,6 +331,7 @@ export const UserDataSchema = z.object({
excludeUncachedFromServices: z.array(z.string().min(1)).optional(),
excludeUncachedFromStreamTypes: z.array(StreamTypes).optional(),
excludeUncachedMode: z.enum(['or', 'and']).optional(),
excludedFilterConditions: z.array(z.string().min(1).max(1000)).optional(),
groups: z
.array(
z.object({
@@ -683,7 +684,10 @@ export const ParsedStreamSchema = z.object({
originalDescription: z.string().optional(),
});
export const ParsedStreams = z.array(ParsedStreamSchema);
export type ParsedStream = z.infer<typeof ParsedStreamSchema>;
export type ParsedStreams = z.infer<typeof ParsedStreams>;
export const AIOStream = StreamSchema.extend({
streamData: z.object({
packages/core/src/main.ts
+38 -3
View File
@@ -42,7 +42,10 @@ import {
safeRegexTest,
} from './utils/regex';
import { isMatch } from 'super-regex';
import { ConditionParser } from './parser/conditions';
import {
GroupConditionParser,
SelectConditionParser,
} from './parser/conditions';
import { RPDB } from './utils/rpdb';
import { FeatureControl } from './utils/feature';
const logger = createLogger('core');
@@ -1092,7 +1095,7 @@ ${errorStreams.length > 0 ? ` ❌ Errors : ${errorStreams.map((s) => `
if (!group.condition || !group.addons.length) continue;
try {
const parser = new ConditionParser(
const parser = new GroupConditionParser(
previousGroupStreams,
parsedStreams,
previousGroupTimeTaken,
@@ -1207,6 +1210,7 @@ ${errorStreams.length > 0 ? ` ❌ Errors : ${errorStreams.map((s) => `
requiredKeywords: { total: 0, details: {} },
requiredSeeders: { total: 0, details: {} },
excludedSeeders: { total: 0, details: {} },
excludedFilterCondition: { total: 0, details: {} },
size: { total: 0, details: {} },
};
@@ -2035,7 +2039,38 @@ ${errorStreams.length > 0 ? ` ❌ Errors : ${errorStreams.map((s) => `
};
const filterResults = await Promise.all(streams.map(shouldKeepStream));
const filteredStreams = streams.filter((_, index) => filterResults[index]);
let filteredStreams = streams.filter((_, index) => filterResults[index]);
if (this.userData.excludedFilterConditions) {
const parser = new SelectConditionParser();
for (const condition of this.userData.excludedFilterConditions) {
try {
const selectedStreams = await parser.select(
filteredStreams,
condition
);
// Remove these streams from filteredStreams
filteredStreams = filteredStreams.filter(
(stream) => !selectedStreams.includes(stream)
);
// Update skip reasons for this condition
if (selectedStreams.length > 0) {
skipReasons.excludedFilterCondition.total += selectedStreams.length;
skipReasons.excludedFilterCondition.details[condition] =
selectedStreams.length;
}
} catch (error) {
logger.error(
`Failed to apply excluded filter condition "${condition}": ${error instanceof Error ? error.message : String(error)}`
);
// Continue with the next condition instead of breaking the entire loop
}
}
}
// Log filter summary
const totalFiltered = streams.length - filteredStreams.length;
packages/core/src/parser/conditions.ts
+273 -45
View File
@@ -1,25 +1,11 @@
import { Parser } from 'expr-eval';
import { ParsedStream } from '../db';
import { ParsedStream, ParsedStreams, ParsedStreamSchema } from '../db';
import bytes from 'bytes';
export class ConditionParser {
private parser: Parser;
private previousStreams: ParsedStream[];
private totalStreams: ParsedStream[];
private previousGroupTimeTaken: number;
private totalTimeTaken: number;
constructor(
previousStreams: ParsedStream[],
totalStreams: ParsedStream[],
previousGroupTimeTaken: number,
totalTimeTaken: number,
queryType: string
) {
this.previousStreams = previousStreams;
this.totalStreams = totalStreams;
this.previousGroupTimeTaken = previousGroupTimeTaken;
this.totalTimeTaken = totalTimeTaken;
export abstract class BaseConditionParser {
protected parser: Parser;
constructor() {
// only allow comparison and logical operators
this.parser = new Parser({
operators: {
@@ -72,12 +58,10 @@ export class ConditionParser {
},
});
this.parser.consts.previousStreams = this.previousStreams;
this.parser.consts.totalStreams = this.totalStreams;
this.parser.consts.queryType = queryType;
this.parser.consts.previousGroupTimeTaken = this.previousGroupTimeTaken;
this.parser.consts.totalTimeTaken = this.totalTimeTaken;
this.setupParserFunctions();
}
private setupParserFunctions() {
this.parser.functions.regexMatched = function (
streams: ParsedStream[],
regexName?: string
@@ -88,27 +72,25 @@ export class ConditionParser {
: stream.regexMatched
);
};
this.parser.functions.indexer = function (
streams: ParsedStream[],
indexer: string
) {
if (!Array.isArray(streams)) {
throw new Error(
"Please use one of 'totalStreams' or 'previousStreams' as the first argument"
);
if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) {
throw new Error('Your streams input must be an array of streams');
} else if (typeof indexer !== 'string') {
throw new Error('Indexer must be a string');
}
return streams.filter((stream) => stream.indexer === indexer);
};
this.parser.functions.resolution = function (
streams: ParsedStream[],
resolution: string
) {
if (!Array.isArray(streams)) {
throw new Error(
"Please use one of 'totalStreams' or 'previousStreams' as the first argument"
);
if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) {
throw new Error('Your streams input must be an array of streams');
} else if (typeof resolution !== 'string') {
throw new Error('Resolution must be a string');
}
@@ -116,14 +98,13 @@ export class ConditionParser {
(stream) => (stream.parsedFile?.resolution || 'Unknown') === resolution
);
};
this.parser.functions.quality = function (
streams: ParsedStream[],
quality: string
) {
if (!Array.isArray(streams)) {
throw new Error(
"Please use one of 'totalStreams' or 'previousStreams' as the first argument"
);
if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) {
throw new Error('Your streams input must be an array of streams');
} else if (typeof quality !== 'string') {
throw new Error('Quality must be a string');
}
@@ -131,27 +112,167 @@ export class ConditionParser {
(stream) => (stream.parsedFile?.quality || 'Unknown') === quality
);
};
this.parser.functions.encode = function (
streams: ParsedStream[],
encode: string
) {
if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) {
throw new Error('Your streams input must be an array of streams');
} else if (typeof encode !== 'string') {
throw new Error('Encode must be a string');
}
return streams.filter((stream) => stream.parsedFile?.encode === encode);
};
this.parser.functions.type = function (
streams: ParsedStream[],
type: string
) {
if (!Array.isArray(streams)) {
throw new Error(
"Please use one of 'totalStreams' or 'previousStreams' as the first argument"
);
throw new Error('Your streams input must be an array of streams');
} else if (typeof type !== 'string') {
throw new Error('Type must be a string');
}
return streams.filter((stream) => stream.type === type);
};
this.parser.functions.visualTag = function (
streams: ParsedStream[],
visualTag: string
) {
if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) {
throw new Error('Your streams input must be an array of streams');
} else if (typeof visualTag !== 'string') {
throw new Error('Visual type must be a string');
}
return streams.filter((stream) =>
stream.parsedFile?.visualTags.includes(visualTag)
);
};
this.parser.functions.audioTag = function (
streams: ParsedStream[],
audioTag: string
) {
if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) {
throw new Error('Your streams input must be an array of streams');
} else if (typeof audioTag !== 'string') {
throw new Error('Audio tag must be a string');
}
return streams.filter((stream) =>
stream.parsedFile?.audioTags.includes(audioTag)
);
};
this.parser.functions.audioChannels = function (
streams: ParsedStream[],
audioChannels: string
) {
if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) {
throw new Error('Your streams input must be an array of streams');
} else if (typeof audioChannels !== 'string') {
throw new Error('Audio channels must be a string');
}
return streams.filter((stream) =>
stream.parsedFile?.audioChannels?.includes(audioChannels)
);
};
this.parser.functions.language = function (
streams: ParsedStream[],
language: string
) {
if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) {
throw new Error('Your streams input must be an array of streams');
} else if (typeof language !== 'string') {
throw new Error('Language must be a string');
}
return streams.filter((stream) =>
stream.parsedFile?.languages?.includes(language)
);
};
this.parser.functions.seeders = function (
streams: ParsedStream[],
minSeeders?: number,
maxSeeders?: number
) {
if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) {
throw new Error('Your streams input must be an array of streams');
} else if (
typeof minSeeders !== 'number' &&
typeof maxSeeders !== 'number'
) {
throw new Error('Min and max seeders must be a number');
}
// select streams with seeders that lie within the range.
return streams.filter((stream) => {
if (
minSeeders &&
stream.torrent?.seeders &&
stream.torrent.seeders < minSeeders
) {
return false;
}
if (
maxSeeders &&
stream.torrent?.seeders &&
stream.torrent.seeders > maxSeeders
) {
return false;
}
return true;
});
};
this.parser.functions.size = function (
streams: ParsedStream[],
minSize?: string | number,
maxSize?: string | number
) {
if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) {
throw new Error('Your streams input must be an array of streams');
} else if (
typeof minSize !== 'number' &&
typeof maxSize !== 'number' &&
typeof minSize !== 'string' &&
typeof maxSize !== 'string'
) {
throw new Error('Min and max size must be a number');
}
// use the bytes library to ensure we get a number
const minSizeInBytes =
typeof minSize === 'string' ? bytes.parse(minSize) : minSize;
const maxSizeInBytes =
typeof maxSize === 'string' ? bytes.parse(maxSize) : maxSize;
return streams.filter((stream) => {
if (
minSize &&
stream.size &&
minSizeInBytes &&
stream.size < minSizeInBytes
) {
return false;
}
if (
maxSize &&
stream.size &&
maxSizeInBytes &&
stream.size > maxSizeInBytes
) {
return false;
}
return true;
});
};
this.parser.functions.service = function (
streams: ParsedStream[],
service: string
) {
if (!Array.isArray(streams)) {
throw new Error(
"Please use one of 'totalStreams' or 'previousStreams' as the first argument"
);
if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) {
throw new Error('Your streams input must be an array of streams');
} else if (
typeof service !== 'string' ||
![
@@ -173,6 +294,7 @@ export class ConditionParser {
}
return streams.filter((stream) => stream.service?.id === service);
};
this.parser.functions.cached = function (streams: ParsedStream[]) {
if (!Array.isArray(streams)) {
throw new Error(
@@ -181,6 +303,7 @@ export class ConditionParser {
}
return streams.filter((stream) => stream.service?.cached === true);
};
this.parser.functions.uncached = function (streams: ParsedStream[]) {
if (!Array.isArray(streams)) {
throw new Error(
@@ -189,6 +312,7 @@ export class ConditionParser {
}
return streams.filter((stream) => stream.service?.cached === false);
};
this.parser.functions.releaseGroup = function (
streams: ParsedStream[],
releaseGroup: string
@@ -204,6 +328,26 @@ export class ConditionParser {
(stream) => stream.parsedFile?.releaseGroup === releaseGroup
);
};
this.parser.functions.addon = function (
streams: ParsedStream[],
addon: string
) {
if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) {
throw new Error('Your streams input must be an array of streams');
} else if (typeof addon !== 'string') {
throw new Error('Addon must be a string');
}
return streams.filter((stream) => stream.addon.name === addon);
};
this.parser.functions.library = function (streams: ParsedStream[]) {
if (!Array.isArray(streams) || streams.some((stream) => !stream.type)) {
throw new Error('Your streams input must be an array of streams');
}
return streams.filter((stream) => stream.library);
};
this.parser.functions.count = function (streams: ParsedStream[]) {
if (!Array.isArray(streams)) {
throw new Error(
@@ -213,7 +357,8 @@ export class ConditionParser {
return streams.length;
};
}
async parse(condition: string) {
protected async evaluateCondition(condition: string): Promise<any> {
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error('Condition parsing timed out'));
@@ -229,9 +374,92 @@ export class ConditionParser {
}
});
}
}
export class GroupConditionParser extends BaseConditionParser {
private previousStreams: ParsedStream[];
private totalStreams: ParsedStream[];
private previousGroupTimeTaken: number;
private totalTimeTaken: number;
constructor(
previousStreams: ParsedStream[],
totalStreams: ParsedStream[],
previousGroupTimeTaken: number,
totalTimeTaken: number,
queryType: string
) {
super();
this.previousStreams = previousStreams;
this.totalStreams = totalStreams;
this.previousGroupTimeTaken = previousGroupTimeTaken;
this.totalTimeTaken = totalTimeTaken;
// Set up constants for this specific parser
this.parser.consts.previousStreams = this.previousStreams;
this.parser.consts.totalStreams = this.totalStreams;
this.parser.consts.queryType = queryType;
this.parser.consts.previousGroupTimeTaken = this.previousGroupTimeTaken;
this.parser.consts.totalTimeTaken = this.totalTimeTaken;
}
async parse(condition: string) {
return await this.evaluateCondition(condition);
}
static async testParse(condition: string) {
const parser = new ConditionParser([], [], 0, 0, 'movie');
const parser = new GroupConditionParser([], [], 0, 0, 'movie');
return await parser.parse(condition);
}
}
export class SelectConditionParser extends BaseConditionParser {
constructor() {
super();
}
async select(
streams: ParsedStream[],
condition: string
): Promise<ParsedStream[]> {
// Set the streams constant for this filter operation
this.parser.consts.streams = streams;
let selectedStreams: ParsedStream[] = [];
try {
selectedStreams = await this.evaluateCondition(condition);
} catch (error) {
throw new Error(
`Filter condition failed: ${error instanceof Error ? error.message : String(error)}`
);
}
// attempt to parse the result
try {
selectedStreams = ParsedStreams.parse(selectedStreams);
} catch (error) {
throw new Error(
`Filter condition failed: ${error instanceof Error ? error.message : String(error)}`
);
}
return selectedStreams;
// // If the result is an array of streams, return those that should be filtered out
// // use ParsedResultSchema to validate
// if (selectedStreams.length > 0) {
// // Filter out the selected streams from the input array
// return streams.filter((stream) => !selectedStreams.includes(stream));
// }
// // If the result is not a stream array, return the original streams
// return streams;
}
static async testSelect(
streams: ParsedStream[],
condition: string
): Promise<ParsedStream[]> {
const parser = new SelectConditionParser();
return await parser.select(streams, condition);
}
}
packages/core/src/utils/config.ts
+16 -2
View File
@@ -15,7 +15,10 @@ import { isEncrypted, decryptString, encryptString } from './crypto';
import { Env } from './env';
import { createLogger, maskSensitiveInfo } from './logger';
import { ZodError } from 'zod';
import { ConditionParser } from '../parser/conditions';
import {
GroupConditionParser,
SelectConditionParser,
} from '../parser/conditions';
import { RPDB } from './rpdb';
import { FeatureControl } from './feature';
import { compileRegex } from './regex';
@@ -277,6 +280,17 @@ export async function validateConfig(
}
}
// validate excluded filter condition
if (config.excludedFilterConditions) {
for (const condition of config.excludedFilterConditions) {
try {
await SelectConditionParser.testSelect([], condition);
} catch (error) {
throw new Error(`Invalid excluded filter condition: ${error}`);
}
}
}
if (config.services) {
config.services = config.services.map((service: Service) =>
validateService(service, decryptValues)
@@ -462,7 +476,7 @@ async function validateGroup(group: Group) {
// we must be able to parse the condition
let result;
try {
result = await ConditionParser.testParse(group.condition);
result = await GroupConditionParser.testParse(group.condition);
} catch (error: any) {
throw new Error(
`Your group condition - '${group.condition}' - is invalid: ${error.message}`
packages/frontend/src/components/menu/filters.tsx
+104 -1
View File
@@ -24,6 +24,7 @@ import {
MdMovieFilter,
MdPerson,
MdSurroundSound,
MdTextFields,
MdVideoLibrary,
} from 'react-icons/md';
import { BiSolidCameraMovie } from 'react-icons/bi';
@@ -76,6 +77,7 @@ import { Modal } from '../ui/modal';
import { useDisclosure } from '@/hooks/disclosure';
import { toast } from 'sonner';
import { Slider } from '../ui/slider/slider';
import { TbFilterCode } from 'react-icons/tb';
type Resolution = (typeof RESOLUTIONS)[number];
type Quality = (typeof QUALITIES)[number];
@@ -274,9 +276,13 @@ function Content() {
Matching
</TabsTrigger>
<TabsTrigger value="keyword">
<FaTextSlash className="text-lg mr-3" />
<MdTextFields className="text-lg mr-3" />
Keyword
</TabsTrigger>
<TabsTrigger value="condition">
<TbFilterCode className="text-lg mr-3" />
Condition
</TabsTrigger>
{status?.settings.regexFilterAccess !== 'none' && (
<TabsTrigger value="regex">
<BsRegex className="text-lg mr-3" />
@@ -1308,6 +1314,103 @@ function Content() {
</div>
</PageWrapper>
</TabsContent>
<TabsContent value="condition" className="space-y-4">
<PageWrapper>
<HeadingWithPageControls heading="Condition" />
<div className="mb-4">
<p className="text-sm text-[--muted]">
Create advanced filters to exclude specific streams from your
results using condition expressions. Write conditions that
evaluate which streams to remove based on properties like
addon type, quality, size, or any other stream attributes.
Multiple conditions can be combined using logical operators
for precise filtering control.
</p>
</div>
<div className="space-y-4">
<SettingsCard title="Help">
<div className="space-y-3">
<p className="text-sm text-[--muted]">
This filter uses the same expression syntax as the Group
Condition Parser, but operates on a single{' '}
<code>streams</code> constant containing all available
streams.
</p>
<div className="text-sm text-[--muted]">
<p className="font-medium mb-2">How it works:</p>
<ul className="list-disc list-inside space-y-1 ml-2">
<li>
Your condition should return an array of streams
</li>
<li>
Streams returned by the condition will be{' '}
<strong>excluded</strong> from results
</li>
<li>
Use functions like <code>addon()</code>,{' '}
<code>type()</code>, <code>quality()</code> to filter
streams
</li>
<li>
Combine conditions together by nesting functions
together, for example:
<code>
addon(type(streams, 'debrid'), 'TorBox')
</code>{' '}
excludes all TorBox debrid streams and keeps its
usenet streams.
</li>
</ul>
</div>
<p className="text-sm text-[--muted]">
<strong>Example:</strong>{' '}
<code>addon(type(streams, 'debrid'), 'TorBox')</code>{' '}
excludes all TorBox debrid streams.
</p>
<p className="text-sm text-[--muted]">
For detailed syntax and available functions, see the{' '}
<a
href="https://github.com/Viren070/AIOStreams/wiki/Groups"
target="_blank"
rel="noopener noreferrer"
className="text-[--brand] hover:underline"
>
Wiki page on Groups
</a>
</p>
</div>
</SettingsCard>
<TextInputs
label="Excluded Filter Conditions"
itemName="Condition"
help="The conditions to apply to the streams. Streams selected by any of these conditions will be excluded from the results."
placeholder="addon(type(streams, 'debrid'), 'TorBox')"
values={userData.excludedFilterConditions || []}
onValuesChange={(values) => {
setUserData((prev) => ({
...prev,
excludedFilterConditions: values,
}));
}}
onValueChange={(value, index) => {
setUserData((prev) => ({
...prev,
excludedFilterConditions: [
...(prev.excludedFilterConditions || []).slice(
0,
index
),
value,
...(prev.excludedFilterConditions || []).slice(
index + 1
),
],
}));
}}
/>
</div>
</PageWrapper>
</TabsContent>
<TabsContent value="keyword" className="space-y-4">
<PageWrapper>
<HeadingWithPageControls heading="Keyword" />