feat: refresh regex patterns from URLs in intervals

.env.sample

@@ -200,6 +200,9 @@ REGEX_FILTER_ACCESS=trusted
 # ALLOWED_REGEX_PATTERNS_URLS=
 # ALLOWED_REGEX_PATTERNS_DESCRIPTION=
 
+# How often patterns from URLs will be refreshed. 
+# Default: 86400000 (1 day)
+# ALLOWED_REGEX_PATTERNS_URLS_REFRESH_INTERVAL=
 # --- Aliased Configurations (Vanity URLs) ---
 # Create shorter, memorable installation URLs.
 # Format: aliasName1:uuid1:encryptedPassword1,aliasName2:uuid2:encryptedPassword2

packages/core/src/utils/cache.ts

@@ -161,6 +161,12 @@ export class Cache<K, V> {
     return this.instances.get(name) as Cache<K, V>;
   }
 
+  public static async close() {
+    if (this.redisClient) {
+      await this.redisClient.disconnect();
+    }
+  }
+
   /**
    * Gets the statistics of the cache in use by the program. returns a formatted string containing a list of all cache instances
    * and their currently held items, max items

packages/core/src/utils/config.ts

@@ -500,9 +500,10 @@ async function validateRegexes(config: UserData) {
   ];
 
   if (!regexAllowed && regexes.length > 0) {
-    const allowedPatterns = await FeatureControl.allowedRegexPatterns;
+    const allowedPatterns = (await FeatureControl.allowedRegexPatterns())
+      .patterns;
     const allowedRegexes = regexes.filter((regex) =>
-      allowedPatterns.patterns.includes(regex)
+      allowedPatterns.includes(regex)
     );
     if (allowedRegexes.length === 0) {
       throw new Error(

packages/core/src/utils/env.ts

@@ -477,6 +477,10 @@ export const Env = cleanEnv(process.env, {
     default: undefined,
     desc: 'Comma separated list of allowed regex patterns URLs',
   }),
+  ALLOWED_REGEX_PATTERNS_URLS_REFRESH_INTERVAL: num({
+    default: 86400000,
+    desc: 'Interval for refreshing regex patterns from URLs in milliseconds',
+  }),
   ALLOWED_REGEX_PATTERNS_DESCRIPTION: str({
     default: undefined,
     desc: 'Description of the allowed regex patterns',

packages/core/src/utils/feature.ts

@@ -46,7 +46,11 @@ async function fetchPatternsFromUrl(url: string): Promise<string[]> {
     const parsedData = schema.parse(data);
     const patterns = parsedData.map((item) => item.pattern);
     if (remotePatternCache) {
-      await remotePatternCache.set(url, patterns, 60 * 60 * 24);
+      await remotePatternCache.set(
+        url,
+        patterns,
+        Math.floor(Env.ALLOWED_REGEX_PATTERNS_URLS_REFRESH_INTERVAL / 1000)
+      );
     }
     return patterns;
   } catch (error) {
@@ -56,33 +60,80 @@ async function fetchPatternsFromUrl(url: string): Promise<string[]> {
 }
 
 export class FeatureControl {
-  private static readonly _allowedRegexPatterns: Promise<{
+  private static _patternState: {
     patterns: string[];
     description?: string;
-  }> = (async () => {
-    const patterns: string[] = Env.ALLOWED_REGEX_PATTERNS;
-    let patternsFromUrls: string[] = [];
-    if (Env.ALLOWED_REGEX_PATTERNS_URLS?.length) {
-      const fetchPromises = await Promise.allSettled(
-        Env.ALLOWED_REGEX_PATTERNS_URLS.map(fetchPatternsFromUrl)
-      );
-      patternsFromUrls = fetchPromises
-        .filter(
-          (result): result is PromiseFulfilledResult<string[]> =>
-            result.status === 'fulfilled'
-        )
-        .flatMap((result) => result.value);
-      logger.debug(
-        `Fetched ${patternsFromUrls.length} regex patterns from URLs`
-      );
-    }
-    const allPatterns = [...new Set([...patterns, ...patternsFromUrls])];
+  } = {
+    patterns: Env.ALLOWED_REGEX_PATTERNS || [],
+    description: Env.ALLOWED_REGEX_PATTERNS_DESCRIPTION,
+  };
+  private static _initialisationPromise: Promise<void> | null = null;
+  private static _refreshInterval: NodeJS.Timeout | null = null;
 
-    return {
-      patterns: allPatterns,
-      description: Env.ALLOWED_REGEX_PATTERNS_DESCRIPTION,
-    };
-  })();
+  /**
+   * Initialises the FeatureControl service, performing the initial pattern fetch
+   * and setting up periodic refreshes.
+   */
+  public static initialise() {
+    if (!this._initialisationPromise) {
+      this._initialisationPromise = this._refreshPatterns().then(() => {
+        logger.info(
+          `Initialised with ${this._patternState.patterns.length} regex patterns.`
+        );
+        this._refreshInterval = setInterval(
+          () => this._refreshPatterns(),
+          Env.ALLOWED_REGEX_PATTERNS_URLS_REFRESH_INTERVAL
+        );
+      });
+    }
+    return this._initialisationPromise;
+  }
+
+  /**
+   * Cleans up resources for graceful shutdown.
+   */
+  public static cleanup() {
+    if (this._refreshInterval) {
+      clearInterval(this._refreshInterval);
+      this._refreshInterval = null;
+    }
+  }
+
+  /**
+   * Fetches patterns from all configured URLs and accumulates them.
+   */
+  private static async _refreshPatterns(): Promise<void> {
+    const urls = Env.ALLOWED_REGEX_PATTERNS_URLS;
+    if (!urls || urls.length === 0) {
+      return;
+    }
+
+    logger.debug(`Refreshing regex patterns from ${urls.length} URLs...`);
+    const fetchPromises = await Promise.allSettled(
+      urls.map(fetchPatternsFromUrl)
+    );
+
+    const patternsFromUrls = fetchPromises
+      .filter(
+        (result): result is PromiseFulfilledResult<string[]> =>
+          result.status === 'fulfilled'
+      )
+      .flatMap((result) => result.value);
+
+    if (patternsFromUrls.length > 0) {
+      const initialCount = this._patternState.patterns.length;
+      const allPatterns = [
+        ...new Set([...this._patternState.patterns, ...patternsFromUrls]),
+      ];
+      this._patternState.patterns = allPatterns;
+      const newCount = allPatterns.length - initialCount;
+      if (newCount > 0) {
+        logger.info(
+          `Accumulated ${newCount} new regex patterns from URLs. Total: ${allPatterns.length}`
+        );
+      }
+    }
+  }
 
   private static readonly _disabledHosts: Map<string, string> = (() => {
     const map = new Map<string, string>();
@@ -132,12 +183,13 @@ export class FeatureControl {
     return this._disabledServices;
   }
 
-  public static get allowedRegexPatterns() {
-    return this._allowedRegexPatterns;
+  public static async allowedRegexPatterns() {
+    await this.initialise();
+    return this._patternState;
   }
 
   public static async isRegexAllowed(userData: UserData, regexes?: string[]) {
-    const { patterns } = await this.allowedRegexPatterns;
+    const { patterns } = await this.allowedRegexPatterns();
     if (regexes && regexes.length > 0) {
       const areAllRegexesAllowed = regexes.every((regex) =>
         patterns.includes(regex)

packages/server/src/routes/api/status.ts

@@ -32,10 +32,10 @@ router.get('/', async (req: Request, res: Response) => {
       tmdbApiAvailable: !!Env.TMDB_ACCESS_TOKEN,
       regexFilterAccess: Env.REGEX_FILTER_ACCESS,
       allowedRegexPatterns:
-        (await FeatureControl.allowedRegexPatterns).patterns.length > 0
+        (await FeatureControl.allowedRegexPatterns()).patterns.length > 0
           ? {
-              patterns: (await FeatureControl.allowedRegexPatterns).patterns,
-              description: (await FeatureControl.allowedRegexPatterns)
+              patterns: (await FeatureControl.allowedRegexPatterns()).patterns,
+              description: (await FeatureControl.allowedRegexPatterns())
                 .description,
             }
           : undefined,

packages/server/src/server.ts

@@ -7,6 +7,7 @@ import {
   UserRepository,
   logStartupInfo,
   Cache,
+  FeatureControl,
 } from '@aiostreams/core';
 
 const logger = createLogger('server');
@@ -40,6 +41,7 @@ async function start() {
   try {
     await initialiseDatabase();
     await initialiseRedis();
+    FeatureControl.initialise();
     if (Env.PRUNE_MAX_DAYS >= 0) {
       startAutoPrune();
     }
@@ -53,15 +55,21 @@ async function start() {
   }
 }
 
+async function shutdown() {
+  await Cache.close();
+  FeatureControl.cleanup();
+  await DB.getInstance().close();
+}
+
 process.on('SIGTERM', async () => {
   logger.info('SIGTERM received. Shutting down gracefully...');
-  await DB.getInstance().close();
+  await shutdown();
   process.exit(0);
 });
 
 process.on('SIGINT', async () => {
   logger.info('SIGINT received. Shutting down gracefully...');
-  await DB.getInstance().close();
+  await shutdown();
   process.exit(0);
 });