feat(nab): add pagination handling

closes #489

Release-As: 2.17.6

.env.sample

@@ -178,6 +178,7 @@ BUILTIN_TORBOX_SEARCH_CACHE_PER_USER_SEARCH_ENGINE=false
 BUILTIN_NAB_SEARCH_TIMEOUT=30000
 BUILTIN_NAB_SEARCH_CACHE_TTL=604800
 BUILTIN_NAB_CAPABILITIES_CACHE_TTL=1209600
+BUILTIN_NAB_MAX_PAGES=5
 
 # --- Zilean --- 
 BUILTIN_ZILEAN_URL="https://zilean.elfhosted.com"

packages/core/src/builtins/base/nab/addon.ts

@@ -7,7 +7,12 @@ import {
   BaseDebridConfigSchema,
   SearchMetadata,
 } from '../debrid.js';
-import { BaseNabApi, Capabilities, SearchResultItem } from './api.js';
+import {
+  BaseNabApi,
+  Capabilities,
+  SearchResponse,
+  SearchResultItem,
+} from './api.js';
 import { createQueryLimit, useAllTitles } from '../../utils/general.js';
 
 export const NabAddonConfigSchema = BaseDebridConfigSchema.extend({
@@ -15,6 +20,8 @@ export const NabAddonConfigSchema = BaseDebridConfigSchema.extend({
   apiKey: z.string().optional(),
   apiPath: z.string().optional(),
   forceQuerySearch: z.boolean().default(false),
+  paginate: z.boolean().default(false),
+  forceInitialLimit: z.number().optional(),
 });
 export type NabAddonConfig = z.infer<typeof NabAddonConfigSchema>;
 
@@ -65,7 +72,10 @@ export abstract class BaseNabAddon<
       searchCapabilities,
     });
 
-    queryParams.limit = capabilities.limits?.max?.toString() ?? '10000';
+    queryParams.limit =
+      this.userData.forceInitialLimit?.toString() ??
+      capabilities.limits?.max?.toString() ??
+      '10000';
 
     if (this.userData.forceQuerySearch) {
     } else if (
@@ -132,15 +142,14 @@ export abstract class BaseNabAddon<
     if (queries.length > 0) {
       this.logger.debug('Performing queries', { queries });
       const searchPromises = queries.map((q) =>
-        queryLimit(() => this.api.search(searchFunction, { ...queryParams, q }))
+        queryLimit(() =>
+          this.fetchResults(searchFunction, { ...queryParams, q })
+        )
       );
       const allResults = await Promise.all(searchPromises);
-      results = allResults.flat() as SearchResultItem<A['namespace']>[];
+      results = allResults.flat();
     } else {
-      results = (await this.api.search(
-        searchFunction,
-        queryParams
-      )) as unknown as SearchResultItem<A['namespace']>[];
+      results = await this.fetchResults(searchFunction, queryParams);
     }
     this.logger.info(
       `Completed search for ${capabilities.server.title} in ${getTimeTakenSincePoint(start)}`,
@@ -170,14 +179,14 @@ export abstract class BaseNabAddon<
       const movieSearch = available.find((s) =>
         s.toLowerCase().includes('movie')
       );
-      if (movieSearch && (searching as any)[movieSearch].available)
+      if (movieSearch && searching[movieSearch].available)
         return {
-          capabilities: (searching as any)[movieSearch],
+          capabilities: searching[movieSearch],
           function: 'movie',
         };
     } else {
       const tvSearch = available.find((s) => s.toLowerCase().includes('tv'));
-      if (tvSearch && (searching as any)[tvSearch].available)
+      if (tvSearch && searching[tvSearch].available)
         return {
           capabilities: (searching as any)[tvSearch],
           function: 'tvsearch',
@@ -187,4 +196,159 @@ export abstract class BaseNabAddon<
       return { capabilities: (searching as any).search, function: 'search' };
     return undefined;
   }
+
+  private async fetchResults(
+    searchFunction: string,
+    params: Record<string, string>
+  ): Promise<SearchResultItem<A['namespace']>[]> {
+    const queryLimit = createQueryLimit();
+    const maxPages = Env.BUILTIN_NAB_MAX_PAGES;
+
+    const initialResponse: SearchResponse<A['namespace']> =
+      await this.api.search(searchFunction, params);
+    let allResults = [...initialResponse.results];
+
+    this.logger.debug('Initial search response', {
+      resultsCount: initialResponse.results.length,
+      offset: initialResponse.offset,
+      total: initialResponse.total,
+    });
+
+    // if both first and last items are duplicates, the page is likely a duplicate
+    const areResultsDuplicate = (
+      existing: SearchResultItem<A['namespace']>[],
+      newResults: SearchResultItem<A['namespace']>[]
+    ): boolean => {
+      if (newResults.length === 0) return false;
+
+      const firstNew = newResults[0];
+      const lastNew = newResults[newResults.length - 1];
+
+      const firstExists = existing.some((r) => r.guid === firstNew.guid);
+      const lastExists = existing.some((r) => r.guid === lastNew.guid);
+
+      return firstExists && lastExists;
+    };
+
+    if (!this.userData.paginate) {
+      this.logger.info(
+        'Pagination handling is disabled, returning initial results only'
+      );
+      return allResults;
+    }
+
+    if (initialResponse.total !== undefined && initialResponse.total > 0) {
+      const limit =
+        initialResponse.results.length > 0
+          ? initialResponse.results.length
+          : parseInt(params.limit || '100', 10);
+      const total = initialResponse.total;
+      const initialOffset = initialResponse.offset || 0;
+
+      // Calculate how many more pages we need
+      const remainingResults = total - (initialOffset + limit);
+      if (remainingResults > 0) {
+        const additionalPages = Math.ceil(remainingResults / limit);
+        const pagesToFetch = Math.min(additionalPages, maxPages - 1); // -1 because we already fetched first page
+
+        if (pagesToFetch > 0) {
+          this.logger.debug('Fetching additional pages with known total', {
+            total,
+            limit,
+            pagesToFetch,
+            remainingResults,
+          });
+
+          // Create requests for all remaining pages in parallel
+          const pagePromises = Array.from({ length: pagesToFetch }, (_, i) => {
+            const offset = initialOffset + limit * (i + 1);
+            return queryLimit(
+              () =>
+                this.api.search(searchFunction, {
+                  ...params,
+                  offset: offset.toString(),
+                }) as Promise<SearchResponse<A['namespace']>>
+            );
+          });
+
+          const pageResponses = await Promise.all(pagePromises);
+          for (const response of pageResponses) {
+            if (areResultsDuplicate(allResults, response.results)) {
+              this.logger.warn(
+                'Detected duplicate results in paginated response. Indexer may not support offset parameter despite claiming support. Stopping pagination.'
+              );
+              break;
+            }
+            allResults.push(...response.results);
+          }
+        }
+      }
+    } else {
+      // keep fetching until we get empty results or hit max pages
+      let pageCount = 1;
+      let currentOffset =
+        (initialResponse.offset || 0) + initialResponse.results.length;
+      const limit =
+        initialResponse.results.length > 0
+          ? initialResponse.results.length
+          : parseInt(params.limit || '100', 10);
+
+      this.logger.debug('Fetching pages without known total', {
+        initialResultsCount: initialResponse.results.length,
+        limit,
+      });
+
+      while (pageCount < maxPages) {
+        const response: SearchResponse<A['namespace']> = await this.api.search(
+          searchFunction,
+          {
+            ...params,
+            offset: currentOffset.toString(),
+          }
+        );
+
+        if (response.results.length === 0) {
+          this.logger.debug('Received empty page, stopping pagination');
+          break;
+        }
+
+        if (areResultsDuplicate(allResults, response.results)) {
+          this.logger.warn(
+            'Detected duplicate results in paginated response. Indexer may not support offset parameter. Stopping pagination.'
+          );
+          break;
+        }
+
+        allResults.push(...response.results);
+        currentOffset += response.results.length;
+        pageCount++;
+
+        this.logger.debug('Fetched additional page', {
+          pageCount,
+          resultsInPage: response.results.length,
+          totalResults: allResults.length,
+        });
+
+        // if this page returned less results than the limit, we can assume there are no more pages
+        if (response.results.length < limit) {
+          this.logger.debug(
+            'Received less results than limit, assuming last page'
+          );
+          break;
+        }
+      }
+
+      if (pageCount >= maxPages) {
+        this.logger.warn(
+          `Reached maximum page limit (${maxPages}), stopping pagination`
+        );
+      }
+    }
+
+    this.logger.info('Completed fetching all results', {
+      totalResults: allResults.length,
+    });
+
+    return allResults;
+  }
 }

packages/core/src/builtins/base/nab/api.ts

@@ -178,6 +178,19 @@ const createNewznabItemSchema = () =>
       newznab: item['newznab:attr'],
     }));
 
+// schema for response attributes (offset, total only)
+const ResponseAttributeSchema = z
+  .object({
+    $: z.object({
+      offset: convertString.optional(),
+      total: convertString.optional(),
+    }),
+  })
+  .transform((obj) => ({
+    offset: obj.$.offset as number | undefined,
+    total: obj.$.total as number | undefined,
+  }));
+
 // Type definitions for search result items
 export type TorznabSearchResultItem = z.infer<
   ReturnType<typeof createTorznabItemSchema>
@@ -190,12 +203,24 @@ export type NewznabSearchResultItem = z.infer<
 export type SearchResultItem<T extends 'torznab' | 'newznab'> =
   T extends 'torznab' ? TorznabSearchResultItem : NewznabSearchResultItem;
 
+export type SearchResponse<T extends 'torznab' | 'newznab'> = {
+  offset?: number;
+  total?: number;
+  results: SearchResultItem<T>[];
+};
+
+type RawSearchResponse = {
+  offset?: number;
+  total?: number;
+  results: (TorznabSearchResultItem | NewznabSearchResultItem)[];
+};
+
 // --- API Client Class ---
 export class BaseNabApi<N extends 'torznab' | 'newznab'> {
   private readonly xmlParser: Parser;
   private readonly capabilitiesCache: Cache<string, Capabilities>;
-  private readonly searchCache: Cache<string, SearchResultItem<N>[]>;
-  private readonly SearchResultSchema: z.ZodType<any[]>;
+  private readonly searchCache: Cache<string, SearchResponse<N>>;
+  private readonly SearchResultSchema: z.ZodType<RawSearchResponse>;
   private readonly logger: Logger;
 
   constructor(
@@ -210,7 +235,7 @@ export class BaseNabApi<N extends 'torznab' | 'newznab'> {
     this.apiPath = this.removeTrailingSlash(apiPath);
     this.xmlParser = new Parser();
     this.capabilitiesCache = Cache.getInstance(`${namespace}:api:caps`);
-    this.searchCache = Cache.getInstance(`${namespace}:api:search`);
+    this.searchCache = Cache.getInstance(`${namespace}:api:search:v2`);
 
     // Create the appropriate schema based on namespace
     if (namespace === 'torznab') {
@@ -220,11 +245,25 @@ export class BaseNabApi<N extends 'torznab' | 'newznab'> {
             channel: z.array(
               z.object({
                 item: z.array(createTorznabItemSchema()).optional().default([]),
+                'torznab:response': z.array(ResponseAttributeSchema).optional(),
+                'newznab:response': z.array(ResponseAttributeSchema).optional(),
+                response: z.array(ResponseAttributeSchema).optional(),
               })
             ),
           }),
         })
-        .transform((data) => data.rss.channel[0].item);
+        .transform((data) => {
+          const channel = data.rss.channel[0];
+          const response =
+            channel['torznab:response']?.[0] ??
+            channel['newznab:response']?.[0] ??
+            channel.response?.[0];
+          return {
+            offset: response?.offset,
+            total: response?.total,
+            results: channel.item,
+          };
+        });
     } else {
       this.SearchResultSchema = z
         .object({
@@ -232,11 +271,25 @@ export class BaseNabApi<N extends 'torznab' | 'newznab'> {
             channel: z.array(
               z.object({
                 item: z.array(createNewznabItemSchema()).optional().default([]),
+                'newznab:response': z.array(ResponseAttributeSchema).optional(),
+                'torznab:response': z.array(ResponseAttributeSchema).optional(),
+                response: z.array(ResponseAttributeSchema).optional(),
               })
             ),
           }),
         })
-        .transform((data) => data.rss.channel[0].item);
+        .transform((data) => {
+          const channel = data.rss.channel[0];
+          const response =
+            channel['newznab:response']?.[0] ??
+            channel['torznab:response']?.[0] ??
+            channel.response?.[0];
+          return {
+            offset: response?.offset,
+            total: response?.total,
+            results: channel.item,
+          };
+        });
     }
   }
 
@@ -252,13 +305,14 @@ export class BaseNabApi<N extends 'torznab' | 'newznab'> {
   public async search(
     searchFunction: string = 'search',
     params: Record<string, string | number | boolean> = {}
-  ): Promise<SearchResultItem<N>[]> {
+  ): Promise<SearchResponse<N>> {
     const cacheKey = `${this.baseUrl}${this.apiPath}?t=${searchFunction}&${JSON.stringify(params)}&apikey=${this.apiKey}`;
-    return this.searchCache.wrap(
+    const result = await this.searchCache.wrap(
       () => this.request(searchFunction, this.SearchResultSchema, params),
       cacheKey,
       Env.BUILTIN_NAB_SEARCH_CACHE_TTL
     );
+    return result as SearchResponse<N>;
   }
 
   private removeTrailingSlash = (path: string) =>

packages/core/src/parser/utils.ts

@@ -54,7 +54,7 @@ export function preprocessTitle(
         if (parts.length > 1 && parts[0]?.trim()) {
           const originalTitle = preprocessedTitle;
           preprocessedTitle = parts[0].trim();
-          logger.debug(
+          logger.silly(
             `Updated title from "${originalTitle}" to "${preprocessedTitle}"`
           );
           break;
@@ -84,11 +84,19 @@ export function normaliseTitle(title: string) {
 }
 
 export function cleanTitle(title: string) {
-  const umlautMap: Record<string,string> = {
-    'Ä':'Ae','ä':'ae','Ö':'Oe','ö':'oe','Ü':'Ue','ü':'ue','ß':'ss'
+  const umlautMap: Record<string, string> = {
+    Ä: 'Ae',
+    ä: 'ae',
+    Ö: 'Oe',
+    ö: 'oe',
+    Ü: 'Ue',
+    ü: 'ue',
+    ß: 'ss',
   };
   // replace German umlauts with ASCII equivalents, then normalize to NFD
-  let cleaned = title.replace(/[ÄäÖöÜüß]/g, c => umlautMap[c]).normalize('NFD');
+  let cleaned = title
+    .replace(/[ÄäÖöÜüß]/g, (c) => umlautMap[c])
+    .normalize('NFD');
 
   for (const char of ['♪', '♫', '★', '☆', '♡', '♥', '-']) {
     cleaned = cleaned.replaceAll(char, ' ');

packages/core/src/presets/animetosho.ts

@@ -91,6 +91,7 @@ export class AnimeToshoPreset extends TorznabPreset {
       ...this.getBaseConfig(userData, services),
       url: animetoshoUrl,
       apiPath: '/api',
+      paginate: false,
     };
 
     const configString = this.base64EncodeJSON(config, 'urlSafe');

packages/core/src/presets/bitmagnet.ts

@@ -22,6 +22,11 @@ export class BitmagnetPreset extends TorznabPreset {
         type: 'number',
         required: true,
         default: Env.BUILTIN_DEFAULT_BITMAGNET_TIMEOUT || Env.DEFAULT_TIMEOUT,
+        constraints: {
+          min: Env.MIN_TIMEOUT,
+          max: Env.MAX_TIMEOUT,
+          forceInUi: false,
+        },
       },
       {
         id: 'mediaTypes',
@@ -53,6 +58,24 @@ export class BitmagnetPreset extends TorznabPreset {
         default: undefined,
         emptyIsUndefined: true,
       },
+      {
+        id: 'useMultipleInstances',
+        name: 'Use Multiple Instances',
+        description:
+          'Torznab supports multiple services in one instance of the addon - which is used by default. If this is enabled, then the addon will be created for each service.',
+        type: 'boolean',
+        default: false,
+        showInSimpleMode: false,
+      },
+      {
+        id: 'paginate',
+        name: 'Paginate Results',
+        description:
+          'Whether to paginate through all available results when searching. Enabling this can provide more results at the cost of increased search time and more requests.',
+        type: 'boolean',
+        default: false,
+        required: false,
+      },
     ];
 
     return {
@@ -92,6 +115,7 @@ export class BitmagnetPreset extends TorznabPreset {
       url: `${Env.BUILTIN_BITMAGNET_URL.replace(/\/$/, '')}/torznab`,
       apiPath: '/api',
       forceQuerySearch: true,
+      paginate: options.paginate ?? false,
     };
 
     const configString = this.base64EncodeJSON(config, 'urlSafe');

packages/core/src/presets/newznab.ts

@@ -152,6 +152,15 @@ export class NewznabPreset extends BuiltinAddonPreset {
           { label: 'Both', value: 'both' },
         ],
       },
+      {
+        id: 'paginate',
+        name: 'Paginate Results',
+        description:
+          'Newznab endpoints can limit the number of results returned per request. Enabling this option will make the addon paginate through all available results to provide a more comprehensive set of results. Enabling this can increase the time taken to return results, some endpoints may not support pagination, and this will also increase the number of requests.',
+        type: 'boolean',
+        default: false,
+        showInSimpleMode: false,
+      },
       {
         id: 'useMultipleInstances',
         name: 'Use Multiple Instances',
@@ -262,6 +271,7 @@ export class NewznabPreset extends BuiltinAddonPreset {
       apiKey: options.apiKey,
       proxyAuth: options.proxyAuth,
       forceQuerySearch: options.forceQuerySearch ?? false,
+      paginate: options.paginate ?? false,
     };
 
     const configString = this.base64EncodeJSON(config, 'urlSafe');

packages/core/src/presets/nzbhydra.ts

@@ -158,6 +158,8 @@ export class NZBHydraPreset extends NewznabPreset {
       apiPath: options.apiPath,
       apiKey: nzbhydraApiKey,
       forceQuerySearch: options.forceQuerySearch ?? true,
+      forceInitialLimit: 10000,
+      paginate: false,
     };
 
     const configString = this.base64EncodeJSON(config, 'urlSafe');

packages/core/src/presets/torznab.ts

@@ -114,6 +114,15 @@ export class TorznabPreset extends BuiltinAddonPreset {
           { label: 'Both', value: 'both' },
         ],
       },
+      {
+        id: 'paginate',
+        name: 'Paginate Results',
+        description:
+          'Whether to paginate through all available results when searching. Enabling this can provide more results at the cost of increased search time and more requests.',
+        type: 'boolean',
+        default: false,
+        required: false,
+      },
       {
         id: 'useMultipleInstances',
         name: 'Use Multiple Instances',
@@ -223,6 +232,7 @@ export class TorznabPreset extends BuiltinAddonPreset {
       apiPath: options.apiPath,
       apiKey: options.apiKey,
       forceQuerySearch: options.forceQuerySearch ?? false,
+      paginate: options.paginate ?? false,
     };
 
     const configString = this.base64EncodeJSON(config, 'urlSafe');

packages/core/src/utils/env.ts

@@ -1733,6 +1733,10 @@ export const Env = cleanEnv(process.env, {
     default: 14 * 24 * 60 * 60, // 14 days
     desc: 'Builtin Torznab/Newznab Capabilities cache TTL',
   }),
+  BUILTIN_NAB_MAX_PAGES: num({
+    default: 5,
+    desc: 'Maximum number of pages to fetch from Torznab/Newznab indexers during pagination',
+  }),
 
   BUILTIN_ZILEAN_URL: url({
     default: 'https://zilean.elfhosted.com',

packages/core/src/utils/startup.ts

@@ -574,6 +574,7 @@ const logStartupInfo = () => {
       formatDurationAsText(Env.BUILTIN_NAB_CAPABILITIES_CACHE_TTL),
       '       '
     );
+    logKeyValue('Max Pages:', Env.BUILTIN_NAB_MAX_PAGES.toString(), '       ');
 
     // Bitmagnet
     logKeyValue(