feat(core/formatters): add ::<comparator>:: to formatter for Advanced Custom Formatting Logic! (#381)

Co-authored-by: David Garcia <dgarcia3@atlassian.com>
2025-12-01 23:14:04 +01:00 · 2025-09-18 15:14:20 -04:00
parent aab0620f83
commit bc2e065645
4 changed files with 256 additions and 137 deletions
@@ -106,24 +106,55 @@ export interface ParseValue {
  } & typeof DebugToolReplacementConstants;
 }

+/**
+ * Pre-compiled function that takes ParseValue and returns formatted string
+ */
+type CompiledParseFunction = (parseValue: ParseValue) => string;
+type CompiledVariableWInsertFn = { resultFn: (parseValue: ParseValue) => ResolvedVariable, insertIndex: number };
+/**
+ * Pre-compiled function that takes ParseValue and returns `ResolvedVariable` (future: and the variable's context for caching purposes)
+ * 
+ * Retrieves the resolved variable (including modifiers) given a ParseValue (e.g. `stream.cached:istrue` -> `{result: true}` or `stream.languages::istrue` -> `{error: "unknown_array_modifier(istrue)"}`)
+ */
+type CompiledModifiedVariableFn = (parseValue: ParseValue) => ResolvedVariable;
+
 export abstract class BaseFormatter {
  protected config: FormatterConfig;
  protected userData: UserData;

  private regexBuilder: BaseFormatterRegexBuilder;
+  private precompiledNameFunction: CompiledParseFunction | null = null;
+  private precompiledDescriptionFunction: CompiledParseFunction | null = null;
+  
+  private _compilationPromise: Promise<void>;
  
  constructor(config: FormatterConfig, userData: UserData) {
    this.config = config;
    this.userData = userData;

    this.regexBuilder = new BaseFormatterRegexBuilder(this.convertStreamToParseValue({} as ParsedStream));
+
+    // Start template compilation asynchronously in the background
+    this._compilationPromise = this.compileTemplatesAsync();
  }

-  public format(stream: ParsedStream): { name: string; description: string } {
+  private async compileTemplatesAsync(): Promise<void> {
+    this.precompiledNameFunction = await this.compileTemplate(this.config.name);
+    this.precompiledDescriptionFunction = await this.compileTemplate(this.config.description);
+  }
+
+  public async format(stream: ParsedStream): Promise<{ name: string; description: string }> {
+    // Wait for template compilation to complete if it hasn't already
+    await this._compilationPromise;
+    
+    if (!this.precompiledNameFunction || !this.precompiledDescriptionFunction) {
+      throw new Error('Template compilation failed - formatter not ready');
+    }
+    
    const parseValue = this.convertStreamToParseValue(stream);
    return {
-      name: this.parseString(this.config.name, parseValue) || '',
-      description: this.parseString(this.config.description, parseValue) || '',
+      name: this.precompiledNameFunction(parseValue),
+      description: this.precompiledDescriptionFunction(parseValue),
    };
  }

@@ -158,7 +189,7 @@ export abstract class BaseFormatter {
    const onlyUserSpecifiedLanguages = sortedLanguages?.filter((lang) =>
      userSpecifiedLanguages.includes(lang as any)
    );
-    let parseValue: ParseValue = {
+    const parseValue: ParseValue = {
      config: {
        addonName: this.userData.addonName || Env.ADDON_NAME,
      },
@@ -266,124 +297,171 @@ export abstract class BaseFormatter {
    return parseValue;
  }

-  protected parseString(str: string, value: ParseValue): string | null {
-    if (!str) return null;

+  protected async compileTemplate(str: string): Promise<CompiledParseFunction> {
+    if (!str) return () => '';
    const re = this.regexBuilder.buildRegexExpression();
    let matches: RegExpExecArray | null;

+    let compiledMatchTemplateFns: CompiledVariableWInsertFn[] = [];
+
+    for (const key in DebugToolReplacementConstants) {
+        str = str.replace(`{debug.${key}}`, DebugToolReplacementConstants[key as keyof typeof DebugToolReplacementConstants]);
+    }
+
+    // Iterate through all {...} matches
    while (matches = re.exec(str)) {
      if (!matches.groups) continue;
-
      const index = matches.index as number;

+      // looks like variableType.propertyName(::<modifier|comparator>)* (no timezone or check)
+      let matchWithoutSuffix = matches[0].substring(1, (matches[0].length-1) - (matches.groups.suffix ?? "").length);

-      // Validate - variableType (exists in value)
-      const variableDict = value[matches.groups.variableType as keyof ParseValue];
-      if (!variableDict) {
-        str = this.replaceCharsFromString(
-          str,
-          '{unknown_variableName}',
-          index,
-          re.lastIndex
-        );
-        re.lastIndex = index;
-        continue;
-      }
+      // Split {<var1_with_modifiers>>::<comparator1>::<var2_with_modifiers>>...} into variableWithModifiers array and comparators array
+      const splitOnComparators = matchWithoutSuffix.split(RegExp(this.regexBuilder.buildComparatorRegexPattern(), 'gi'));
+      const variableWithModifiers = splitOnComparators.filter((_, i) => i % 2 == 0);
+      const comparators = splitOnComparators.filter((_, i) => i % 2 != 0);
+      const foundComparators = comparators.map(c => c as keyof typeof ComparatorConstants.comparatorKeyToFuncs);
+      let precompiledResolvedVariableFns: CompiledModifiedVariableFn[] = variableWithModifiers
+        .map(baseString => this.parseModifiedVariable(baseString, {
+            mod_tzlocale: matches?.groups?.mod_tzlocale ?? undefined
+          })
+      );
        
-      // Validate - property: variableDict[propertyName]
-      const property = variableDict[matches.groups.propertyName as keyof typeof variableDict];
-      if (property === undefined) {
-        str = this.replaceCharsFromString(
-          str,
-          '{unknown_propertyName}',
-          index,
-          re.lastIndex
-        );
-        re.lastIndex = index;
-        continue;
-      }
      
-      // Validate and Process - Modifier(s)
-      if (matches.groups.modifiers) {
-        let result = this.applyModifiers(matches.groups, property, value);
-        // handle unknown modifier result
-        if (result === undefined) {
-          result = `{unknown_modifier(${matches.groups.modifiers})}`;
-          if (['string', 'number', 'boolean', 'object', 'array'].includes(typeof property)) {
-            result = `{unknown_${typeof property}_modifier(${matches.groups.modifiers})}`;
+      // COMPARATOR logic: compare all ResolvedVariables against each other to make one ResolvedVariable (as precompiled wrapper function (parseValue) => ResolvedVariable)
+      let precompiledResolvedVariableFn = (parseValue: ParseValue): ResolvedVariable => {
+        if (precompiledResolvedVariableFns.length == 1) return precompiledResolvedVariableFns[0](parseValue);
+
+        const resolvedVariablesWithContext = precompiledResolvedVariableFns.map(fn => fn(parseValue));
+        const reducedResolvedVarWContext = resolvedVariablesWithContext.reduce((prev, cur, i) => {
+          if (prev.error !== undefined) return prev;
+          if (cur.error !== undefined) return cur;
+          // the comparator key between prev and cur (from splitOnComparators)
+          const compareKey = foundComparators[i - 1] as keyof typeof ComparatorConstants.comparatorKeyToFuncs;
+          const comparatorFn = ComparatorConstants.comparatorKeyToFuncs[compareKey];
+          
+          try {
+            const result = comparatorFn(prev.result, cur.result);
+            const finalResult = { result: result };
+            return finalResult;
+          } catch (e) {
+            const errorResult = { error: `{unable_to_compare(<${prev.result}>::${compareKey}::<${cur.result}>, ${e})}` };
+            return errorResult;
          }
-        }
-        str = this.replaceCharsFromString(
-          str,
-          result,
-          index,
-          re.lastIndex
-        );
-        re.lastIndex = index;
-        continue;
+        });
+        return reducedResolvedVarWContext;
+      }; // end of COMPARATOR logic
+
+
+      // CHECK TRUE/FALSE logic: compile the true/false templates and apply them to the resolved variable
+      if (matches.groups.mod_check !== undefined) {
+        const check_trueFn = await this.compileTemplate(matches?.groups?.mod_check_true ?? "");
+        const check_falseFn = await this.compileTemplate(matches?.groups?.mod_check_false ?? "");
+
+        const _compiledResolvedVariableFn = precompiledResolvedVariableFn;
+        precompiledResolvedVariableFn = (parseValue: ParseValue): ResolvedVariable => {
+          const resolved = _compiledResolvedVariableFn(parseValue);
+          if (![true, false].includes(resolved.result)) {
+            return { error: `{cannot_coerce_boolean_for_check_from(${resolved.result})}` };
+          }
+          return { result: (resolved.result ? check_trueFn(parseValue) : check_falseFn(parseValue)) };
+        };
+      } // end of CHECK TRUE/FALSE logic
+
+      str = str.slice(0, index) + str.slice(re.lastIndex);
+      re.lastIndex = index;
+      compiledMatchTemplateFns.push({ resultFn: precompiledResolvedVariableFn, insertIndex: index });
+    } // end of while loop
+
+    return (parseValue: ParseValue) => {
+      let resultStr = str;
+      
+      // Sort by startIndex to process in reverse order
+      for (const { resultFn, insertIndex } of compiledMatchTemplateFns.sort((a, b) => b.insertIndex - a.insertIndex)) {
+        const resolvedResult = resultFn(parseValue);
+        const replacement = resolvedResult.error ?? resolvedResult.result?.toString() ?? '';
+        resultStr = resultStr.slice(0, insertIndex) + replacement + resultStr.slice(insertIndex);
      }
      
-      str = this.replaceCharsFromString(str, property, index, re.lastIndex);
-      re.lastIndex = index;
+      return resultStr
+        .replace(/\\n/g, '\n')
+        .split('\n')
+        .filter(
+          (line) => line.trim() !== '' && !line.includes('{tools.removeLine}')
+        )
+        .join('\n')
+        .replace(/\{tools.newLine\}/g, '\n');
    }
-
-    return str
-      .replace(/\\n/g, '\n')
-      .split('\n')
-      .filter(
-        (line) => line.trim() !== '' && !line.includes('{tools.removeLine}')
-      )
-      .join('\n')
-      .replace(/\{tools.newLine\}/g, '\n');
  }

-  protected applyModifiers(
-    groups: {[key: string]: string},
-    input: any,
-    parseValue: ParseValue,
-  ): string | undefined {
-    const singleModTerminator = '((::)|($))'; // :: if there's multiple modifiers or $ for the end of the string
-    const singleValidModRe = new RegExp(this.regexBuilder.buildModifierRegexPattern() + singleModTerminator, 'gi');
+  /**
+   * @param baseString - string to parse, e.g. `<variableType>.<propertyName>(::<modifier>)*`
+   * @param value - ParseValue object
+   * @param fullStringModifiers - modifiers that are applied to the entire string (e.g. `::<tzLocale>`)
+   * 
+   * @returns (parseValue) => `{ result: <resolved modified variable> }` or `{ error: "<errorMessage>" }`
+   */
+  protected parseModifiedVariable(
+    baseString: string,
+    fullStringModifiers: {
+      mod_tzlocale: string | undefined,
+    },
+  ): CompiledModifiedVariableFn {
+    // get variableType and propertyName from baseString without regex
+    const variableType = baseString.split('.')[0];
+    baseString = baseString.substring(variableType.length + 1);
+    const propertyName = baseString.split('::')[0];
+    const allModifiers = baseString.substring(propertyName.length);
+    let sortedModMatches: string[] = [];
+    if (allModifiers.length) {
+      const singleModTerminator = '(?=::|$)'; // :: if there's multiple modifiers, or $ for the end of the string
+      const singleValidModRe = new RegExp(`${this.regexBuilder.buildModifierRegexPattern()}${singleModTerminator}`, 'g');
        
-    let result = input as any;
-    // iterate over modifiers in order of appearance
-    for (const modMatch of [...groups.modifiers.matchAll(singleValidModRe)].sort((a, b) => (a.index ?? 0) - (b.index ?? 0))) {
-      if (result === undefined) break;
-      result = this.applySingleModifier(
-        result,
-        modMatch[1], // First capture group (the modifier name)
-        groups.mod_tzlocale ?? "",
-      );
+      sortedModMatches = [...allModifiers.matchAll(singleValidModRe)].sort((a, b) => (a.index ?? 0) - (b.index ?? 0)).map(regExpExecArray => regExpExecArray[1] /* First capture group, aka the modifier name */);
    }

-    // handle unknown modifier result
-    switch (typeof result) {
-      case 'undefined': return undefined;
-      case 'boolean':
-        let check_true = groups.mod_check_true ?? "";
-        let check_false = groups.mod_check_false ?? "";
-        if (typeof check_true !== 'string' || typeof check_false !== 'string')
-          return `{unknown_conditional_modifier_check_true_or_false}`;
+    return (parseValue: ParseValue) => {
+      // PARSE VARIABLE logic
+      const variableDict = parseValue[variableType as keyof ParseValue];
+      if (!variableDict) return { error: `{unknown_variableType(${variableType})}` }; // should never happen
+      const property = variableDict![propertyName as keyof typeof variableDict] as any;
+      if (property === undefined) return { error: `{unknown_propertyName(${variableType}.${propertyName})}` }; // should never happen
+      // end of PARSE VARIABLE logic

-        if (parseValue) {
-          check_true = this.parseString(check_true, parseValue) || check_true;
-          check_false = this.parseString(check_false, parseValue) || check_false;
+      // APPLY MULTIPLE MODIFIERS logic
+      let result = property;
+      for (const lastModMatched of sortedModMatches) {
+        result = this.applySingleModifier(result, lastModMatched, fullStringModifiers);
+        if (result === undefined) {
+          let getErrorResult = () => {
+            switch (typeof property) {
+              case "string": case "number": case "boolean":  return { error: `{unknown_${typeof property}_modifier(${lastModMatched})}` };
+              case "object":  return { error: `{unknown_array_modifier(${lastModMatched})}` };
+              default:  return { error: `{unknown_modifier(${lastModMatched})}` };
+            }
+          }
+          return getErrorResult();
        }
-        return result ? check_true : check_false;
-      default:
-        return result;
+      }
+      // end of APPLY MULTIPLE MODIFIERS logic
+
+      return { result: result } as ResolvedVariable;
    }
  }

  /**
   * @param variable - the variable to apply the modifier to (e.g. `123`, `"TorBox"`, `["English", "Italian"]`, etc.)
   * @param mod - the modifier to apply
+   * @param fullStringModifiers - modifiers that are applied to the entire string (e.g. `::<tzLocale>`)
+   * @returns `{ result: <resolved modified variable> }` or `{ error: "<errorMessage>" }`
   */
  protected applySingleModifier(
    variable: any,
    mod: string,
-    tzlocale?: string,
+    fullStringModifiers: {
+      mod_tzlocale: string | undefined,
+    },
  ): string | boolean | undefined {
    const _mod = mod;
    mod = mod.toLowerCase();
@@ -463,17 +541,9 @@ export abstract class BaseFormatter {

    return undefined;
  }
-
-  protected replaceCharsFromString(
-    str: string,
-    replace: string,
-    start: number,
-    end: number
-  ): string {
-    return str.slice(0, start) + replace + str.slice(end);
-  }
 }

+
 /**
 * Used to store the actual value of a parsed, and potentially modified, variable
 * or an error message if the parsed/modified result becomes invalid for any reason
@@ -491,27 +561,39 @@ class BaseFormatterRegexBuilder {
  }
  /**
   * RegEx Capture Pattern: `<variableType>.<propertyName>`
+   * 
+   * (no named capture group)
   */
  public buildVariableRegexPattern(): string {
-    const validVariables: (keyof ParseValue)[] = Object.keys(this.hardcodedParseValueKeysForRegexMatching) as (keyof ParseValue)[];
-    // Get all valid properties (subkeys) from ParseValue structure
-    const validProperties = validVariables.flatMap(sectionKey => {
+    // Get all valid variable names (keys as well as subkeys) from ParseValue structure
+    const validVariableNames = Object.keys(this.hardcodedParseValueKeysForRegexMatching).flatMap(sectionKey => {
      const section = this.hardcodedParseValueKeysForRegexMatching[sectionKey as keyof ParseValue];
      if (section && typeof section === 'object' && section !== null) {
-        return Object.keys(section);
+        return Object.keys(section).map((key) => `${sectionKey}\\.${key}`);
      }
-      return [];
+      return []; // @flatMap
    });
-    return `(?<variableType>${validVariables.join('|')})\\.(?<propertyName>${validProperties.join('|')})`;
+    return `(${validVariableNames.join('|')})`;
  }
  /**
   * RegEx Capture Pattern: `::<modifier>`
+   * 
+   * (no named capture group)
   */
  public buildModifierRegexPattern(): string {
    const validModifiers = Object.keys(ModifierConstants.modifiers)
      .map(key => key.replace(/[\(\)\'\"\$\^\~\=\>\<]/g, '\\$&'));
    return `::(${validModifiers.join('|')})`;
  }
+  /**
+   * RegEx Capture Pattern: `::<comparator>::`
+   * 
+   * (no named capture group)
+   */
+  public buildComparatorRegexPattern(): string {
+    const comparatorKeys = Object.keys(ComparatorConstants.comparatorKeyToFuncs)
+    return `::(${comparatorKeys.join("|")})::`
+  }
  /**
   * RegEx Capture Pattern: `::<tzLocale>`
   * 
@@ -539,10 +621,12 @@ class BaseFormatterRegexBuilder {
  public buildRegexExpression(): RegExp {
    const variable = this.buildVariableRegexPattern();
    const modifier = this.buildModifierRegexPattern();
+    const comparator = this.buildComparatorRegexPattern();
    const modTZLocale = this.buildTZLocaleRegexPattern();
    const checkTF = this.buildCheckRegexPattern();
    
-    const regexPattern = `\\{${variable}(?<modifiers>(${modifier})+)?(${modTZLocale})?(${checkTF})?\\}`;
+    const variableAndModifiers = `${variable}(${modifier})*`;
+    const regexPattern = `\\{${variableAndModifiers}(${comparator}${variableAndModifiers})*(?<suffix>(${modTZLocale})?(${checkTF})?)\\}`;
    
    return new RegExp(regexPattern, 'gi');
  }
@@ -564,7 +648,7 @@ class ModifierConstants {
    'reverse': (value: string) => value.split('').reverse().join(''),
    'base64': (value: string) => btoa(value),
    'string': (value: string) => value,
-}
+  }

  static arrayModifierGetOrDefault = (value: string[], i: number) => value.length > 0 ? String(value[i]) : '';
  static arrayModifiers = {
@@ -638,6 +722,18 @@ class ModifierConstants {
  }
 }

+class ComparatorConstants {
+  static comparatorKeyToFuncs = {
+    "and": (v1: any, v2: any) => v1 && v2,
+    "or": (v1: any, v2: any) => v1 || v2,
+    "xor": (v1: any, v2: any) => (v1 || v2) && !(v1 && v2),
+    "neq": (v1: any, v2: any) => v1 !== v2,
+    "equal": (v1: any, v2: any) => v1 === v2,
+    "left": (v1: any, _: any) => v1,
+    "right": (_: any, v2: any) => v2,
+  }
+}
+
 const DebugToolReplacementConstants = {
  modifier: `
 String: {config.addonName}
@@ -647,6 +743,7 @@ String: {config.addonName}
  ::length {config.addonName::length}
  ::reverse {config.addonName::reverse}
 {tools.newLine}
+
 Number: {stream.size}
  ::bytes {stream.size::bytes}
  ::time {stream.size::time}
@@ -654,12 +751,14 @@ Number: {stream.size}
  ::octal {stream.size::octal}
  ::binary {stream.size::binary}
 {tools.newLine}
+
 Array: {stream.languages}
  ::join('-separator-') {stream.languages::join("-separator-")}
  ::length {stream.languages::length}
  ::first {stream.languages::first}
  ::last {stream.languages::last}
 {tools.newLine}
+
 Conditional:
  String: {stream.filename}
    filename::exists    {stream.filename::exists["true"||"false"]}
@@ -676,10 +775,29 @@ Conditional:
    ::istrue {stream.proxied::istrue["true"||"false"]}
    ::isfalse {stream.proxied::isfalse["true"||"false"]}
 {tools.newLine}
+
 [Advanced] Multiple modifiers
  <string>::reverse::title::reverse   {config.addonName} -> {config.addonName::reverse::title::reverse}
  <number>::string::reverse           {stream.size} -> {stream.size::string::reverse}
  <array>::string::reverse            {stream.languages} -> {stream.languages::join("::")::reverse}
  <boolean>::length::>=2              {stream.languages} -> {stream.languages::length::>=2["true"||"false"]}
 `,
+  
+  comparator : `
+Comparators: <stream.library({stream.library})>::comparator::<stream.proxied({stream.proxied})>
+  ::and:: {stream.library::and::stream.proxied["true"||"false"]}
+  ::or:: {stream.library::or::stream.proxied["true"||"false"]}
+  ::xor:: {stream.library::xor::stream.proxied["true"||"false"]}
+  ::neq:: {stream.library::neq::stream.proxied["true"||"false"]}
+  ::equal:: {stream.library::equal::stream.proxied["true"||"false"]}
+  ::left:: {stream.library::left::stream.proxied["true"||"false"]}
+  ::right:: {stream.library::right::stream.proxied["true"||"false"]}
+{tools.newLine}
+
+[Advanced] Multiple Comparators
+  Is English
+    stream.languages::~English::or::stream.languages::~dub::and::stream.languages::length::>0["Yes"||"Unknown"]  ->  {stream.languages::~English::or::stream.languages::~dub::and::stream.languages::length::>0["Yes"||"Unknown"]}
+  Is Fast Enough Link
+    service.cached::or::stream.library::or::stream.seeders::>10["true"||"false"]  ->  {service.cached::istrue::or::stream.library::or::stream.seeders::>10["true"||"false"]}
+`,
 }
@@ -46,7 +46,7 @@ export class StremioTransformer {
  private async convertParsedStreamToStream(
    stream: ParsedStream,
    formatter: {
-      format: (stream: ParsedStream) => { name: string; description: string };
+      format: (stream: ParsedStream) => Promise<{ name: string; description: string }>;
    },
    index: number,
    provideStreamData: boolean
@@ -56,7 +56,7 @@ export class StremioTransformer {
          name: stream.originalName || stream.addon.name,
          description: stream.originalDescription,
        }
-      : formatter.format(stream);
+      : await formatter.format(stream);

    const autoPlaySettings = {
      enabled: this.userData.autoPlay?.enabled ?? true,
@@ -203,6 +203,7 @@ export class StremioTransformer {
    }>,
    options?: { provideStreamData: boolean }
  ): Promise<AIOStreamResponse> {
+    const formatter = createFormatter(this.userData);
    const {
      data: { streams, statistics },
      errors,
@@ -211,8 +212,6 @@ export class StremioTransformer {

    let transformedStreams: AIOStream[] = [];

-    const formatter = createFormatter(this.userData);
-
    const start = Date.now();

    transformedStreams = await Promise.all(
@@ -328,7 +327,7 @@ export class StremioTransformer {

    // Create formatter for stream conversion if needed
    let formatter: {
-      format: (stream: ParsedStream) => { name: string; description: string };
+      format: (stream: ParsedStream) => Promise<{ name: string; description: string }>;
    } | null = null;
    if (
      meta.videos?.some((video) => video.streams && video.streams.length > 0)
@@ -17,21 +17,9 @@ router.use(formatApiRateLimiter);

 const logger = createLogger('server');

-router.post('/', (req: Request, res: Response) => {
+router.post('/', async (req: Request, res: Response) => {
  const { userData, stream } = req.body;
-  const {
-    success: streamSuccess,
-    error: streamError,
-    data: streamData,
-  } = ParsedStreamSchema.safeParse(stream);
-  if (!streamSuccess) {
-    logger.error('Invalid stream', { error: streamError });
-    throw new APIError(
-      constants.ErrorCode.FORMAT_INVALID_STREAM,
-      400,
-      formatZodError(streamError)
-    );
-  }
+
  const {
    success: userDataSuccess,
    error: userDataError,
@@ -45,7 +33,21 @@ router.post('/', (req: Request, res: Response) => {
      formatZodError(userDataError)
    );
  }
-  const formattedStream = createFormatter(userDataData).format(streamData);
+  const formatter = createFormatter(userDataData);
+  const {
+    success: streamSuccess,
+    error: streamError,
+    data: streamData,
+  } = ParsedStreamSchema.safeParse(stream);
+  if (!streamSuccess) {
+    logger.error('Invalid stream', { error: streamError });
+    throw new APIError(
+      constants.ErrorCode.FORMAT_INVALID_STREAM,
+      400,
+      formatZodError(streamError)
+    );
+  }
+  const formattedStream = await formatter.format(streamData);
  res
    .status(200)
    .json(createResponse({ success: true, data: formattedStream }));