diff --git a/.vscode/settings.json b/.vscode/settings.json index 1b6457c5..d1b4edb2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,4 @@ { "editor.formatOnSave": true, "editor.defaultFormatter": "esbenp.prettier-vscode" -} +} \ No newline at end of file diff --git a/packages/core/src/formatters/base.ts b/packages/core/src/formatters/base.ts index cb977d1a..d8434002 100644 --- a/packages/core/src/formatters/base.ts +++ b/packages/core/src/formatters/base.ts @@ -106,24 +106,55 @@ export interface ParseValue { } & typeof DebugToolReplacementConstants; } +/** + * Pre-compiled function that takes ParseValue and returns formatted string + */ +type CompiledParseFunction = (parseValue: ParseValue) => string; +type CompiledVariableWInsertFn = { resultFn: (parseValue: ParseValue) => ResolvedVariable, insertIndex: number }; +/** + * Pre-compiled function that takes ParseValue and returns `ResolvedVariable` (future: and the variable's context for caching purposes) + * + * Retrieves the resolved variable (including modifiers) given a ParseValue (e.g. `stream.cached:istrue` -> `{result: true}` or `stream.languages::istrue` -> `{error: "unknown_array_modifier(istrue)"}`) + */ +type CompiledModifiedVariableFn = (parseValue: ParseValue) => ResolvedVariable; + export abstract class BaseFormatter { protected config: FormatterConfig; protected userData: UserData; private regexBuilder: BaseFormatterRegexBuilder; - + private precompiledNameFunction: CompiledParseFunction | null = null; + private precompiledDescriptionFunction: CompiledParseFunction | null = null; + + private _compilationPromise: Promise; + constructor(config: FormatterConfig, userData: UserData) { this.config = config; this.userData = userData; this.regexBuilder = new BaseFormatterRegexBuilder(this.convertStreamToParseValue({} as ParsedStream)); + + // Start template compilation asynchronously in the background + this._compilationPromise = this.compileTemplatesAsync(); } - public format(stream: ParsedStream): { name: string; description: string } { + private async compileTemplatesAsync(): Promise { + this.precompiledNameFunction = await this.compileTemplate(this.config.name); + this.precompiledDescriptionFunction = await this.compileTemplate(this.config.description); + } + + public async format(stream: ParsedStream): Promise<{ name: string; description: string }> { + // Wait for template compilation to complete if it hasn't already + await this._compilationPromise; + + if (!this.precompiledNameFunction || !this.precompiledDescriptionFunction) { + throw new Error('Template compilation failed - formatter not ready'); + } + const parseValue = this.convertStreamToParseValue(stream); return { - name: this.parseString(this.config.name, parseValue) || '', - description: this.parseString(this.config.description, parseValue) || '', + name: this.precompiledNameFunction(parseValue), + description: this.precompiledDescriptionFunction(parseValue), }; } @@ -158,7 +189,7 @@ export abstract class BaseFormatter { const onlyUserSpecifiedLanguages = sortedLanguages?.filter((lang) => userSpecifiedLanguages.includes(lang as any) ); - let parseValue: ParseValue = { + const parseValue: ParseValue = { config: { addonName: this.userData.addonName || Env.ADDON_NAME, }, @@ -266,124 +297,171 @@ export abstract class BaseFormatter { return parseValue; } - protected parseString(str: string, value: ParseValue): string | null { - if (!str) return null; + protected async compileTemplate(str: string): Promise { + if (!str) return () => ''; const re = this.regexBuilder.buildRegexExpression(); let matches: RegExpExecArray | null; + let compiledMatchTemplateFns: CompiledVariableWInsertFn[] = []; + + for (const key in DebugToolReplacementConstants) { + str = str.replace(`{debug.${key}}`, DebugToolReplacementConstants[key as keyof typeof DebugToolReplacementConstants]); + } + + // Iterate through all {...} matches while (matches = re.exec(str)) { if (!matches.groups) continue; - const index = matches.index as number; + // looks like variableType.propertyName(::)* (no timezone or check) + let matchWithoutSuffix = matches[0].substring(1, (matches[0].length-1) - (matches.groups.suffix ?? "").length); - // Validate - variableType (exists in value) - const variableDict = value[matches.groups.variableType as keyof ParseValue]; - if (!variableDict) { - str = this.replaceCharsFromString( - str, - '{unknown_variableName}', - index, - re.lastIndex - ); - re.lastIndex = index; - continue; - } + // Split {>::::>...} into variableWithModifiers array and comparators array + const splitOnComparators = matchWithoutSuffix.split(RegExp(this.regexBuilder.buildComparatorRegexPattern(), 'gi')); + const variableWithModifiers = splitOnComparators.filter((_, i) => i % 2 == 0); + const comparators = splitOnComparators.filter((_, i) => i % 2 != 0); + const foundComparators = comparators.map(c => c as keyof typeof ComparatorConstants.comparatorKeyToFuncs); + let precompiledResolvedVariableFns: CompiledModifiedVariableFn[] = variableWithModifiers + .map(baseString => this.parseModifiedVariable(baseString, { + mod_tzlocale: matches?.groups?.mod_tzlocale ?? undefined + }) + ); + + + // COMPARATOR logic: compare all ResolvedVariables against each other to make one ResolvedVariable (as precompiled wrapper function (parseValue) => ResolvedVariable) + let precompiledResolvedVariableFn = (parseValue: ParseValue): ResolvedVariable => { + if (precompiledResolvedVariableFns.length == 1) return precompiledResolvedVariableFns[0](parseValue); - // Validate - property: variableDict[propertyName] - const property = variableDict[matches.groups.propertyName as keyof typeof variableDict]; - if (property === undefined) { - str = this.replaceCharsFromString( - str, - '{unknown_propertyName}', - index, - re.lastIndex - ); - re.lastIndex = index; - continue; - } - - // Validate and Process - Modifier(s) - if (matches.groups.modifiers) { - let result = this.applyModifiers(matches.groups, property, value); - // handle unknown modifier result - if (result === undefined) { - result = `{unknown_modifier(${matches.groups.modifiers})}`; - if (['string', 'number', 'boolean', 'object', 'array'].includes(typeof property)) { - result = `{unknown_${typeof property}_modifier(${matches.groups.modifiers})}`; + const resolvedVariablesWithContext = precompiledResolvedVariableFns.map(fn => fn(parseValue)); + const reducedResolvedVarWContext = resolvedVariablesWithContext.reduce((prev, cur, i) => { + if (prev.error !== undefined) return prev; + if (cur.error !== undefined) return cur; + // the comparator key between prev and cur (from splitOnComparators) + const compareKey = foundComparators[i - 1] as keyof typeof ComparatorConstants.comparatorKeyToFuncs; + const comparatorFn = ComparatorConstants.comparatorKeyToFuncs[compareKey]; + + try { + const result = comparatorFn(prev.result, cur.result); + const finalResult = { result: result }; + return finalResult; + } catch (e) { + const errorResult = { error: `{unable_to_compare(<${prev.result}>::${compareKey}::<${cur.result}>, ${e})}` }; + return errorResult; } - } - str = this.replaceCharsFromString( - str, - result, - index, - re.lastIndex - ); - re.lastIndex = index; - continue; - } + }); + return reducedResolvedVarWContext; + }; // end of COMPARATOR logic - str = this.replaceCharsFromString(str, property, index, re.lastIndex); + + // CHECK TRUE/FALSE logic: compile the true/false templates and apply them to the resolved variable + if (matches.groups.mod_check !== undefined) { + const check_trueFn = await this.compileTemplate(matches?.groups?.mod_check_true ?? ""); + const check_falseFn = await this.compileTemplate(matches?.groups?.mod_check_false ?? ""); + + const _compiledResolvedVariableFn = precompiledResolvedVariableFn; + precompiledResolvedVariableFn = (parseValue: ParseValue): ResolvedVariable => { + const resolved = _compiledResolvedVariableFn(parseValue); + if (![true, false].includes(resolved.result)) { + return { error: `{cannot_coerce_boolean_for_check_from(${resolved.result})}` }; + } + return { result: (resolved.result ? check_trueFn(parseValue) : check_falseFn(parseValue)) }; + }; + } // end of CHECK TRUE/FALSE logic + + str = str.slice(0, index) + str.slice(re.lastIndex); re.lastIndex = index; - } + compiledMatchTemplateFns.push({ resultFn: precompiledResolvedVariableFn, insertIndex: index }); + } // end of while loop - return str - .replace(/\\n/g, '\n') - .split('\n') - .filter( - (line) => line.trim() !== '' && !line.includes('{tools.removeLine}') - ) - .join('\n') - .replace(/\{tools.newLine\}/g, '\n'); + return (parseValue: ParseValue) => { + let resultStr = str; + + // Sort by startIndex to process in reverse order + for (const { resultFn, insertIndex } of compiledMatchTemplateFns.sort((a, b) => b.insertIndex - a.insertIndex)) { + const resolvedResult = resultFn(parseValue); + const replacement = resolvedResult.error ?? resolvedResult.result?.toString() ?? ''; + resultStr = resultStr.slice(0, insertIndex) + replacement + resultStr.slice(insertIndex); + } + + return resultStr + .replace(/\\n/g, '\n') + .split('\n') + .filter( + (line) => line.trim() !== '' && !line.includes('{tools.removeLine}') + ) + .join('\n') + .replace(/\{tools.newLine\}/g, '\n'); + } } - protected applyModifiers( - groups: {[key: string]: string}, - input: any, - parseValue: ParseValue, - ): string | undefined { - const singleModTerminator = '((::)|($))'; // :: if there's multiple modifiers or $ for the end of the string - const singleValidModRe = new RegExp(this.regexBuilder.buildModifierRegexPattern() + singleModTerminator, 'gi'); - - let result = input as any; - // iterate over modifiers in order of appearance - for (const modMatch of [...groups.modifiers.matchAll(singleValidModRe)].sort((a, b) => (a.index ?? 0) - (b.index ?? 0))) { - if (result === undefined) break; - result = this.applySingleModifier( - result, - modMatch[1], // First capture group (the modifier name) - groups.mod_tzlocale ?? "", - ); + /** + * @param baseString - string to parse, e.g. `.(::)*` + * @param value - ParseValue object + * @param fullStringModifiers - modifiers that are applied to the entire string (e.g. `::`) + * + * @returns (parseValue) => `{ result: }` or `{ error: "" }` + */ + protected parseModifiedVariable( + baseString: string, + fullStringModifiers: { + mod_tzlocale: string | undefined, + }, + ): CompiledModifiedVariableFn { + // get variableType and propertyName from baseString without regex + const variableType = baseString.split('.')[0]; + baseString = baseString.substring(variableType.length + 1); + const propertyName = baseString.split('::')[0]; + const allModifiers = baseString.substring(propertyName.length); + let sortedModMatches: string[] = []; + if (allModifiers.length) { + const singleModTerminator = '(?=::|$)'; // :: if there's multiple modifiers, or $ for the end of the string + const singleValidModRe = new RegExp(`${this.regexBuilder.buildModifierRegexPattern()}${singleModTerminator}`, 'g'); + + sortedModMatches = [...allModifiers.matchAll(singleValidModRe)].sort((a, b) => (a.index ?? 0) - (b.index ?? 0)).map(regExpExecArray => regExpExecArray[1] /* First capture group, aka the modifier name */); } - // handle unknown modifier result - switch (typeof result) { - case 'undefined': return undefined; - case 'boolean': - let check_true = groups.mod_check_true ?? ""; - let check_false = groups.mod_check_false ?? ""; - if (typeof check_true !== 'string' || typeof check_false !== 'string') - return `{unknown_conditional_modifier_check_true_or_false}`; + return (parseValue: ParseValue) => { + // PARSE VARIABLE logic + const variableDict = parseValue[variableType as keyof ParseValue]; + if (!variableDict) return { error: `{unknown_variableType(${variableType})}` }; // should never happen + const property = variableDict![propertyName as keyof typeof variableDict] as any; + if (property === undefined) return { error: `{unknown_propertyName(${variableType}.${propertyName})}` }; // should never happen + // end of PARSE VARIABLE logic - if (parseValue) { - check_true = this.parseString(check_true, parseValue) || check_true; - check_false = this.parseString(check_false, parseValue) || check_false; + // APPLY MULTIPLE MODIFIERS logic + let result = property; + for (const lastModMatched of sortedModMatches) { + result = this.applySingleModifier(result, lastModMatched, fullStringModifiers); + if (result === undefined) { + let getErrorResult = () => { + switch (typeof property) { + case "string": case "number": case "boolean": return { error: `{unknown_${typeof property}_modifier(${lastModMatched})}` }; + case "object": return { error: `{unknown_array_modifier(${lastModMatched})}` }; + default: return { error: `{unknown_modifier(${lastModMatched})}` }; + } + } + return getErrorResult(); } - return result ? check_true : check_false; - default: - return result; + } + // end of APPLY MULTIPLE MODIFIERS logic + + return { result: result } as ResolvedVariable; } } /** * @param variable - the variable to apply the modifier to (e.g. `123`, `"TorBox"`, `["English", "Italian"]`, etc.) * @param mod - the modifier to apply + * @param fullStringModifiers - modifiers that are applied to the entire string (e.g. `::`) + * @returns `{ result: }` or `{ error: "" }` */ protected applySingleModifier( variable: any, mod: string, - tzlocale?: string, + fullStringModifiers: { + mod_tzlocale: string | undefined, + }, ): string | boolean | undefined { const _mod = mod; mod = mod.toLowerCase(); @@ -400,13 +478,13 @@ export abstract class BaseFormatter { if (!ModifierConstants.conditionalModifiers.exact.exists(variable)) { conditional = false; } - + // EXACT else if (isExact) { const modAsKey = mod as keyof typeof ModifierConstants.conditionalModifiers.exact; conditional = ModifierConstants.conditionalModifiers.exact[modAsKey](variable); } - + // PREFIX else if (isPrefix) { // get the longest prefix match @@ -417,7 +495,7 @@ export abstract class BaseFormatter { let stringCheck = mod.substring(modPrefix.length).toLowerCase(); // remove whitespace from stringCheck if it isn't in stringValue stringCheck = !/\s/.test(stringValue) ? stringCheck.replace(/\s/g, '') : stringCheck; - + // parse value/check as if they're numbers (123,456 -> 123456) const [parsedNumericValue, parsedNumericCheck] = [Number(stringValue.replace(/,\s/g, '')), Number(stringCheck.replace(/,\s/g, ''))]; const isNumericComparison = ["<", "<=", ">", ">=", "="].includes(modPrefix) && @@ -463,17 +541,9 @@ export abstract class BaseFormatter { return undefined; } - - protected replaceCharsFromString( - str: string, - replace: string, - start: number, - end: number - ): string { - return str.slice(0, start) + replace + str.slice(end); - } } + /** * Used to store the actual value of a parsed, and potentially modified, variable * or an error message if the parsed/modified result becomes invalid for any reason @@ -491,27 +561,39 @@ class BaseFormatterRegexBuilder { } /** * RegEx Capture Pattern: `.` + * + * (no named capture group) */ public buildVariableRegexPattern(): string { - const validVariables: (keyof ParseValue)[] = Object.keys(this.hardcodedParseValueKeysForRegexMatching) as (keyof ParseValue)[]; - // Get all valid properties (subkeys) from ParseValue structure - const validProperties = validVariables.flatMap(sectionKey => { + // Get all valid variable names (keys as well as subkeys) from ParseValue structure + const validVariableNames = Object.keys(this.hardcodedParseValueKeysForRegexMatching).flatMap(sectionKey => { const section = this.hardcodedParseValueKeysForRegexMatching[sectionKey as keyof ParseValue]; if (section && typeof section === 'object' && section !== null) { - return Object.keys(section); + return Object.keys(section).map((key) => `${sectionKey}\\.${key}`); } - return []; + return []; // @flatMap }); - return `(?${validVariables.join('|')})\\.(?${validProperties.join('|')})`; + return `(${validVariableNames.join('|')})`; } /** * RegEx Capture Pattern: `::` + * + * (no named capture group) */ public buildModifierRegexPattern(): string { const validModifiers = Object.keys(ModifierConstants.modifiers) .map(key => key.replace(/[\(\)\'\"\$\^\~\=\>\<]/g, '\\$&')); return `::(${validModifiers.join('|')})`; } + /** + * RegEx Capture Pattern: `::::` + * + * (no named capture group) + */ + public buildComparatorRegexPattern(): string { + const comparatorKeys = Object.keys(ComparatorConstants.comparatorKeyToFuncs) + return `::(${comparatorKeys.join("|")})::` + } /** * RegEx Capture Pattern: `::` * @@ -539,11 +621,13 @@ class BaseFormatterRegexBuilder { public buildRegexExpression(): RegExp { const variable = this.buildVariableRegexPattern(); const modifier = this.buildModifierRegexPattern(); + const comparator = this.buildComparatorRegexPattern(); const modTZLocale = this.buildTZLocaleRegexPattern(); const checkTF = this.buildCheckRegexPattern(); - - const regexPattern = `\\{${variable}(?(${modifier})+)?(${modTZLocale})?(${checkTF})?\\}`; - + + const variableAndModifiers = `${variable}(${modifier})*`; + const regexPattern = `\\{${variableAndModifiers}(${comparator}${variableAndModifiers})*(?(${modTZLocale})?(${checkTF})?)\\}`; + return new RegExp(regexPattern, 'gi'); } } @@ -564,7 +648,7 @@ class ModifierConstants { 'reverse': (value: string) => value.split('').reverse().join(''), 'base64': (value: string) => btoa(value), 'string': (value: string) => value, -} + } static arrayModifierGetOrDefault = (value: string[], i: number) => value.length > 0 ? String(value[i]) : ''; static arrayModifiers = { @@ -638,6 +722,18 @@ class ModifierConstants { } } +class ComparatorConstants { + static comparatorKeyToFuncs = { + "and": (v1: any, v2: any) => v1 && v2, + "or": (v1: any, v2: any) => v1 || v2, + "xor": (v1: any, v2: any) => (v1 || v2) && !(v1 && v2), + "neq": (v1: any, v2: any) => v1 !== v2, + "equal": (v1: any, v2: any) => v1 === v2, + "left": (v1: any, _: any) => v1, + "right": (_: any, v2: any) => v2, + } +} + const DebugToolReplacementConstants = { modifier: ` String: {config.addonName} @@ -647,6 +743,7 @@ String: {config.addonName} ::length {config.addonName::length} ::reverse {config.addonName::reverse} {tools.newLine} + Number: {stream.size} ::bytes {stream.size::bytes} ::time {stream.size::time} @@ -654,12 +751,14 @@ Number: {stream.size} ::octal {stream.size::octal} ::binary {stream.size::binary} {tools.newLine} + Array: {stream.languages} ::join('-separator-') {stream.languages::join("-separator-")} ::length {stream.languages::length} ::first {stream.languages::first} ::last {stream.languages::last} {tools.newLine} + Conditional: String: {stream.filename} filename::exists {stream.filename::exists["true"||"false"]} @@ -676,10 +775,29 @@ Conditional: ::istrue {stream.proxied::istrue["true"||"false"]} ::isfalse {stream.proxied::isfalse["true"||"false"]} {tools.newLine} + [Advanced] Multiple modifiers ::reverse::title::reverse {config.addonName} -> {config.addonName::reverse::title::reverse} ::string::reverse {stream.size} -> {stream.size::string::reverse} ::string::reverse {stream.languages} -> {stream.languages::join("::")::reverse} ::length::>=2 {stream.languages} -> {stream.languages::length::>=2["true"||"false"]} `, -} + + comparator : ` +Comparators: ::comparator:: + ::and:: {stream.library::and::stream.proxied["true"||"false"]} + ::or:: {stream.library::or::stream.proxied["true"||"false"]} + ::xor:: {stream.library::xor::stream.proxied["true"||"false"]} + ::neq:: {stream.library::neq::stream.proxied["true"||"false"]} + ::equal:: {stream.library::equal::stream.proxied["true"||"false"]} + ::left:: {stream.library::left::stream.proxied["true"||"false"]} + ::right:: {stream.library::right::stream.proxied["true"||"false"]} +{tools.newLine} + +[Advanced] Multiple Comparators + Is English + stream.languages::~English::or::stream.languages::~dub::and::stream.languages::length::>0["Yes"||"Unknown"] -> {stream.languages::~English::or::stream.languages::~dub::and::stream.languages::length::>0["Yes"||"Unknown"]} + Is Fast Enough Link + service.cached::or::stream.library::or::stream.seeders::>10["true"||"false"] -> {service.cached::istrue::or::stream.library::or::stream.seeders::>10["true"||"false"]} +`, +} \ No newline at end of file diff --git a/packages/core/src/transformers/stremio.ts b/packages/core/src/transformers/stremio.ts index 3150d28f..1c7e7075 100644 --- a/packages/core/src/transformers/stremio.ts +++ b/packages/core/src/transformers/stremio.ts @@ -46,7 +46,7 @@ export class StremioTransformer { private async convertParsedStreamToStream( stream: ParsedStream, formatter: { - format: (stream: ParsedStream) => { name: string; description: string }; + format: (stream: ParsedStream) => Promise<{ name: string; description: string }>; }, index: number, provideStreamData: boolean @@ -56,7 +56,7 @@ export class StremioTransformer { name: stream.originalName || stream.addon.name, description: stream.originalDescription, } - : formatter.format(stream); + : await formatter.format(stream); const autoPlaySettings = { enabled: this.userData.autoPlay?.enabled ?? true, @@ -203,6 +203,7 @@ export class StremioTransformer { }>, options?: { provideStreamData: boolean } ): Promise { + const formatter = createFormatter(this.userData); const { data: { streams, statistics }, errors, @@ -211,8 +212,6 @@ export class StremioTransformer { let transformedStreams: AIOStream[] = []; - const formatter = createFormatter(this.userData); - const start = Date.now(); transformedStreams = await Promise.all( @@ -328,7 +327,7 @@ export class StremioTransformer { // Create formatter for stream conversion if needed let formatter: { - format: (stream: ParsedStream) => { name: string; description: string }; + format: (stream: ParsedStream) => Promise<{ name: string; description: string }>; } | null = null; if ( meta.videos?.some((video) => video.streams && video.streams.length > 0) diff --git a/packages/server/src/routes/api/format.ts b/packages/server/src/routes/api/format.ts index b893cc1f..10a48aa2 100644 --- a/packages/server/src/routes/api/format.ts +++ b/packages/server/src/routes/api/format.ts @@ -17,21 +17,9 @@ router.use(formatApiRateLimiter); const logger = createLogger('server'); -router.post('/', (req: Request, res: Response) => { +router.post('/', async (req: Request, res: Response) => { const { userData, stream } = req.body; - const { - success: streamSuccess, - error: streamError, - data: streamData, - } = ParsedStreamSchema.safeParse(stream); - if (!streamSuccess) { - logger.error('Invalid stream', { error: streamError }); - throw new APIError( - constants.ErrorCode.FORMAT_INVALID_STREAM, - 400, - formatZodError(streamError) - ); - } + const { success: userDataSuccess, error: userDataError, @@ -45,7 +33,21 @@ router.post('/', (req: Request, res: Response) => { formatZodError(userDataError) ); } - const formattedStream = createFormatter(userDataData).format(streamData); + const formatter = createFormatter(userDataData); + const { + success: streamSuccess, + error: streamError, + data: streamData, + } = ParsedStreamSchema.safeParse(stream); + if (!streamSuccess) { + logger.error('Invalid stream', { error: streamError }); + throw new APIError( + constants.ErrorCode.FORMAT_INVALID_STREAM, + 400, + formatZodError(streamError) + ); + } + const formattedStream = await formatter.format(streamData); res .status(200) .json(createResponse({ success: true, data: formattedStream }));