From af79a18c6ae384d3e882ab96f27365c2f6500497 Mon Sep 17 00:00:00 2001 From: Viren070 Date: Wed, 19 Nov 2025 23:15:44 +0000 Subject: [PATCH] feat(nab): add pagination handling closes #489 Release-As: 2.17.6 --- .env.sample | 1 + packages/core/src/builtins/base/nab/addon.ts | 186 +++++++++++++++++-- packages/core/src/builtins/base/nab/api.ts | 68 ++++++- packages/core/src/parser/utils.ts | 16 +- packages/core/src/presets/animetosho.ts | 1 + packages/core/src/presets/bitmagnet.ts | 24 +++ packages/core/src/presets/newznab.ts | 10 + packages/core/src/presets/nzbhydra.ts | 2 + packages/core/src/presets/torznab.ts | 10 + packages/core/src/utils/env.ts | 4 + packages/core/src/utils/startup.ts | 1 + 11 files changed, 301 insertions(+), 22 deletions(-) diff --git a/.env.sample b/.env.sample index 083104b5..4625f63b 100644 --- a/.env.sample +++ b/.env.sample @@ -178,6 +178,7 @@ BUILTIN_TORBOX_SEARCH_CACHE_PER_USER_SEARCH_ENGINE=false BUILTIN_NAB_SEARCH_TIMEOUT=30000 BUILTIN_NAB_SEARCH_CACHE_TTL=604800 BUILTIN_NAB_CAPABILITIES_CACHE_TTL=1209600 +BUILTIN_NAB_MAX_PAGES=5 # --- Zilean --- BUILTIN_ZILEAN_URL="https://zilean.elfhosted.com" diff --git a/packages/core/src/builtins/base/nab/addon.ts b/packages/core/src/builtins/base/nab/addon.ts index 823224a8..c1e0b182 100644 --- a/packages/core/src/builtins/base/nab/addon.ts +++ b/packages/core/src/builtins/base/nab/addon.ts @@ -7,7 +7,12 @@ import { BaseDebridConfigSchema, SearchMetadata, } from '../debrid.js'; -import { BaseNabApi, Capabilities, SearchResultItem } from './api.js'; +import { + BaseNabApi, + Capabilities, + SearchResponse, + SearchResultItem, +} from './api.js'; import { createQueryLimit, useAllTitles } from '../../utils/general.js'; export const NabAddonConfigSchema = BaseDebridConfigSchema.extend({ @@ -15,6 +20,8 @@ export const NabAddonConfigSchema = BaseDebridConfigSchema.extend({ apiKey: z.string().optional(), apiPath: z.string().optional(), forceQuerySearch: z.boolean().default(false), + paginate: z.boolean().default(false), + forceInitialLimit: z.number().optional(), }); export type NabAddonConfig = z.infer; @@ -65,7 +72,10 @@ export abstract class BaseNabAddon< searchCapabilities, }); - queryParams.limit = capabilities.limits?.max?.toString() ?? '10000'; + queryParams.limit = + this.userData.forceInitialLimit?.toString() ?? + capabilities.limits?.max?.toString() ?? + '10000'; if (this.userData.forceQuerySearch) { } else if ( @@ -132,15 +142,14 @@ export abstract class BaseNabAddon< if (queries.length > 0) { this.logger.debug('Performing queries', { queries }); const searchPromises = queries.map((q) => - queryLimit(() => this.api.search(searchFunction, { ...queryParams, q })) + queryLimit(() => + this.fetchResults(searchFunction, { ...queryParams, q }) + ) ); const allResults = await Promise.all(searchPromises); - results = allResults.flat() as SearchResultItem[]; + results = allResults.flat(); } else { - results = (await this.api.search( - searchFunction, - queryParams - )) as unknown as SearchResultItem[]; + results = await this.fetchResults(searchFunction, queryParams); } this.logger.info( `Completed search for ${capabilities.server.title} in ${getTimeTakenSincePoint(start)}`, @@ -170,14 +179,14 @@ export abstract class BaseNabAddon< const movieSearch = available.find((s) => s.toLowerCase().includes('movie') ); - if (movieSearch && (searching as any)[movieSearch].available) + if (movieSearch && searching[movieSearch].available) return { - capabilities: (searching as any)[movieSearch], + capabilities: searching[movieSearch], function: 'movie', }; } else { const tvSearch = available.find((s) => s.toLowerCase().includes('tv')); - if (tvSearch && (searching as any)[tvSearch].available) + if (tvSearch && searching[tvSearch].available) return { capabilities: (searching as any)[tvSearch], function: 'tvsearch', @@ -187,4 +196,159 @@ export abstract class BaseNabAddon< return { capabilities: (searching as any).search, function: 'search' }; return undefined; } + + private async fetchResults( + searchFunction: string, + params: Record + ): Promise[]> { + const queryLimit = createQueryLimit(); + const maxPages = Env.BUILTIN_NAB_MAX_PAGES; + + const initialResponse: SearchResponse = + await this.api.search(searchFunction, params); + let allResults = [...initialResponse.results]; + + this.logger.debug('Initial search response', { + resultsCount: initialResponse.results.length, + offset: initialResponse.offset, + total: initialResponse.total, + }); + + // if both first and last items are duplicates, the page is likely a duplicate + const areResultsDuplicate = ( + existing: SearchResultItem[], + newResults: SearchResultItem[] + ): boolean => { + if (newResults.length === 0) return false; + + const firstNew = newResults[0]; + const lastNew = newResults[newResults.length - 1]; + + const firstExists = existing.some((r) => r.guid === firstNew.guid); + const lastExists = existing.some((r) => r.guid === lastNew.guid); + + return firstExists && lastExists; + }; + + if (!this.userData.paginate) { + this.logger.info( + 'Pagination handling is disabled, returning initial results only' + ); + return allResults; + } + + if (initialResponse.total !== undefined && initialResponse.total > 0) { + const limit = + initialResponse.results.length > 0 + ? initialResponse.results.length + : parseInt(params.limit || '100', 10); + const total = initialResponse.total; + const initialOffset = initialResponse.offset || 0; + + // Calculate how many more pages we need + const remainingResults = total - (initialOffset + limit); + if (remainingResults > 0) { + const additionalPages = Math.ceil(remainingResults / limit); + const pagesToFetch = Math.min(additionalPages, maxPages - 1); // -1 because we already fetched first page + + if (pagesToFetch > 0) { + this.logger.debug('Fetching additional pages with known total', { + total, + limit, + pagesToFetch, + remainingResults, + }); + + // Create requests for all remaining pages in parallel + const pagePromises = Array.from({ length: pagesToFetch }, (_, i) => { + const offset = initialOffset + limit * (i + 1); + return queryLimit( + () => + this.api.search(searchFunction, { + ...params, + offset: offset.toString(), + }) as Promise> + ); + }); + + const pageResponses = await Promise.all(pagePromises); + for (const response of pageResponses) { + if (areResultsDuplicate(allResults, response.results)) { + this.logger.warn( + 'Detected duplicate results in paginated response. Indexer may not support offset parameter despite claiming support. Stopping pagination.' + ); + break; + } + allResults.push(...response.results); + } + } + } + } else { + // keep fetching until we get empty results or hit max pages + let pageCount = 1; + let currentOffset = + (initialResponse.offset || 0) + initialResponse.results.length; + const limit = + initialResponse.results.length > 0 + ? initialResponse.results.length + : parseInt(params.limit || '100', 10); + + this.logger.debug('Fetching pages without known total', { + initialResultsCount: initialResponse.results.length, + limit, + }); + + while (pageCount < maxPages) { + const response: SearchResponse = await this.api.search( + searchFunction, + { + ...params, + offset: currentOffset.toString(), + } + ); + + if (response.results.length === 0) { + this.logger.debug('Received empty page, stopping pagination'); + break; + } + + if (areResultsDuplicate(allResults, response.results)) { + this.logger.warn( + 'Detected duplicate results in paginated response. Indexer may not support offset parameter. Stopping pagination.' + ); + break; + } + + allResults.push(...response.results); + currentOffset += response.results.length; + pageCount++; + + this.logger.debug('Fetched additional page', { + pageCount, + resultsInPage: response.results.length, + totalResults: allResults.length, + }); + + // if this page returned less results than the limit, we can assume there are no more pages + if (response.results.length < limit) { + this.logger.debug( + 'Received less results than limit, assuming last page' + ); + break; + } + } + + if (pageCount >= maxPages) { + this.logger.warn( + `Reached maximum page limit (${maxPages}), stopping pagination` + ); + } + } + + this.logger.info('Completed fetching all results', { + totalResults: allResults.length, + }); + + return allResults; + } } diff --git a/packages/core/src/builtins/base/nab/api.ts b/packages/core/src/builtins/base/nab/api.ts index fdd45df0..dd25a7c7 100644 --- a/packages/core/src/builtins/base/nab/api.ts +++ b/packages/core/src/builtins/base/nab/api.ts @@ -178,6 +178,19 @@ const createNewznabItemSchema = () => newznab: item['newznab:attr'], })); +// schema for response attributes (offset, total only) +const ResponseAttributeSchema = z + .object({ + $: z.object({ + offset: convertString.optional(), + total: convertString.optional(), + }), + }) + .transform((obj) => ({ + offset: obj.$.offset as number | undefined, + total: obj.$.total as number | undefined, + })); + // Type definitions for search result items export type TorznabSearchResultItem = z.infer< ReturnType @@ -190,12 +203,24 @@ export type NewznabSearchResultItem = z.infer< export type SearchResultItem = T extends 'torznab' ? TorznabSearchResultItem : NewznabSearchResultItem; +export type SearchResponse = { + offset?: number; + total?: number; + results: SearchResultItem[]; +}; + +type RawSearchResponse = { + offset?: number; + total?: number; + results: (TorznabSearchResultItem | NewznabSearchResultItem)[]; +}; + // --- API Client Class --- export class BaseNabApi { private readonly xmlParser: Parser; private readonly capabilitiesCache: Cache; - private readonly searchCache: Cache[]>; - private readonly SearchResultSchema: z.ZodType; + private readonly searchCache: Cache>; + private readonly SearchResultSchema: z.ZodType; private readonly logger: Logger; constructor( @@ -210,7 +235,7 @@ export class BaseNabApi { this.apiPath = this.removeTrailingSlash(apiPath); this.xmlParser = new Parser(); this.capabilitiesCache = Cache.getInstance(`${namespace}:api:caps`); - this.searchCache = Cache.getInstance(`${namespace}:api:search`); + this.searchCache = Cache.getInstance(`${namespace}:api:search:v2`); // Create the appropriate schema based on namespace if (namespace === 'torznab') { @@ -220,11 +245,25 @@ export class BaseNabApi { channel: z.array( z.object({ item: z.array(createTorznabItemSchema()).optional().default([]), + 'torznab:response': z.array(ResponseAttributeSchema).optional(), + 'newznab:response': z.array(ResponseAttributeSchema).optional(), + response: z.array(ResponseAttributeSchema).optional(), }) ), }), }) - .transform((data) => data.rss.channel[0].item); + .transform((data) => { + const channel = data.rss.channel[0]; + const response = + channel['torznab:response']?.[0] ?? + channel['newznab:response']?.[0] ?? + channel.response?.[0]; + return { + offset: response?.offset, + total: response?.total, + results: channel.item, + }; + }); } else { this.SearchResultSchema = z .object({ @@ -232,11 +271,25 @@ export class BaseNabApi { channel: z.array( z.object({ item: z.array(createNewznabItemSchema()).optional().default([]), + 'newznab:response': z.array(ResponseAttributeSchema).optional(), + 'torznab:response': z.array(ResponseAttributeSchema).optional(), + response: z.array(ResponseAttributeSchema).optional(), }) ), }), }) - .transform((data) => data.rss.channel[0].item); + .transform((data) => { + const channel = data.rss.channel[0]; + const response = + channel['newznab:response']?.[0] ?? + channel['torznab:response']?.[0] ?? + channel.response?.[0]; + return { + offset: response?.offset, + total: response?.total, + results: channel.item, + }; + }); } } @@ -252,13 +305,14 @@ export class BaseNabApi { public async search( searchFunction: string = 'search', params: Record = {} - ): Promise[]> { + ): Promise> { const cacheKey = `${this.baseUrl}${this.apiPath}?t=${searchFunction}&${JSON.stringify(params)}&apikey=${this.apiKey}`; - return this.searchCache.wrap( + const result = await this.searchCache.wrap( () => this.request(searchFunction, this.SearchResultSchema, params), cacheKey, Env.BUILTIN_NAB_SEARCH_CACHE_TTL ); + return result as SearchResponse; } private removeTrailingSlash = (path: string) => diff --git a/packages/core/src/parser/utils.ts b/packages/core/src/parser/utils.ts index 5341a3df..1e1866ba 100644 --- a/packages/core/src/parser/utils.ts +++ b/packages/core/src/parser/utils.ts @@ -54,7 +54,7 @@ export function preprocessTitle( if (parts.length > 1 && parts[0]?.trim()) { const originalTitle = preprocessedTitle; preprocessedTitle = parts[0].trim(); - logger.debug( + logger.silly( `Updated title from "${originalTitle}" to "${preprocessedTitle}"` ); break; @@ -84,11 +84,19 @@ export function normaliseTitle(title: string) { } export function cleanTitle(title: string) { - const umlautMap: Record = { - 'Ä':'Ae','ä':'ae','Ö':'Oe','ö':'oe','Ü':'Ue','ü':'ue','ß':'ss' + const umlautMap: Record = { + Ä: 'Ae', + ä: 'ae', + Ö: 'Oe', + ö: 'oe', + Ü: 'Ue', + ü: 'ue', + ß: 'ss', }; // replace German umlauts with ASCII equivalents, then normalize to NFD - let cleaned = title.replace(/[ÄäÖöÜüß]/g, c => umlautMap[c]).normalize('NFD'); + let cleaned = title + .replace(/[ÄäÖöÜüß]/g, (c) => umlautMap[c]) + .normalize('NFD'); for (const char of ['♪', '♫', '★', '☆', '♡', '♥', '-']) { cleaned = cleaned.replaceAll(char, ' '); diff --git a/packages/core/src/presets/animetosho.ts b/packages/core/src/presets/animetosho.ts index fbbf4f11..4316d591 100644 --- a/packages/core/src/presets/animetosho.ts +++ b/packages/core/src/presets/animetosho.ts @@ -91,6 +91,7 @@ export class AnimeToshoPreset extends TorznabPreset { ...this.getBaseConfig(userData, services), url: animetoshoUrl, apiPath: '/api', + paginate: false, }; const configString = this.base64EncodeJSON(config, 'urlSafe'); diff --git a/packages/core/src/presets/bitmagnet.ts b/packages/core/src/presets/bitmagnet.ts index 77d39bdb..0a11682e 100644 --- a/packages/core/src/presets/bitmagnet.ts +++ b/packages/core/src/presets/bitmagnet.ts @@ -22,6 +22,11 @@ export class BitmagnetPreset extends TorznabPreset { type: 'number', required: true, default: Env.BUILTIN_DEFAULT_BITMAGNET_TIMEOUT || Env.DEFAULT_TIMEOUT, + constraints: { + min: Env.MIN_TIMEOUT, + max: Env.MAX_TIMEOUT, + forceInUi: false, + }, }, { id: 'mediaTypes', @@ -53,6 +58,24 @@ export class BitmagnetPreset extends TorznabPreset { default: undefined, emptyIsUndefined: true, }, + { + id: 'useMultipleInstances', + name: 'Use Multiple Instances', + description: + 'Torznab supports multiple services in one instance of the addon - which is used by default. If this is enabled, then the addon will be created for each service.', + type: 'boolean', + default: false, + showInSimpleMode: false, + }, + { + id: 'paginate', + name: 'Paginate Results', + description: + 'Whether to paginate through all available results when searching. Enabling this can provide more results at the cost of increased search time and more requests.', + type: 'boolean', + default: false, + required: false, + }, ]; return { @@ -92,6 +115,7 @@ export class BitmagnetPreset extends TorznabPreset { url: `${Env.BUILTIN_BITMAGNET_URL.replace(/\/$/, '')}/torznab`, apiPath: '/api', forceQuerySearch: true, + paginate: options.paginate ?? false, }; const configString = this.base64EncodeJSON(config, 'urlSafe'); diff --git a/packages/core/src/presets/newznab.ts b/packages/core/src/presets/newznab.ts index d2b9705b..42e58572 100644 --- a/packages/core/src/presets/newznab.ts +++ b/packages/core/src/presets/newznab.ts @@ -152,6 +152,15 @@ export class NewznabPreset extends BuiltinAddonPreset { { label: 'Both', value: 'both' }, ], }, + { + id: 'paginate', + name: 'Paginate Results', + description: + 'Newznab endpoints can limit the number of results returned per request. Enabling this option will make the addon paginate through all available results to provide a more comprehensive set of results. Enabling this can increase the time taken to return results, some endpoints may not support pagination, and this will also increase the number of requests.', + type: 'boolean', + default: false, + showInSimpleMode: false, + }, { id: 'useMultipleInstances', name: 'Use Multiple Instances', @@ -262,6 +271,7 @@ export class NewznabPreset extends BuiltinAddonPreset { apiKey: options.apiKey, proxyAuth: options.proxyAuth, forceQuerySearch: options.forceQuerySearch ?? false, + paginate: options.paginate ?? false, }; const configString = this.base64EncodeJSON(config, 'urlSafe'); diff --git a/packages/core/src/presets/nzbhydra.ts b/packages/core/src/presets/nzbhydra.ts index 8ce053d8..0b6b2207 100644 --- a/packages/core/src/presets/nzbhydra.ts +++ b/packages/core/src/presets/nzbhydra.ts @@ -158,6 +158,8 @@ export class NZBHydraPreset extends NewznabPreset { apiPath: options.apiPath, apiKey: nzbhydraApiKey, forceQuerySearch: options.forceQuerySearch ?? true, + forceInitialLimit: 10000, + paginate: false, }; const configString = this.base64EncodeJSON(config, 'urlSafe'); diff --git a/packages/core/src/presets/torznab.ts b/packages/core/src/presets/torznab.ts index 5ae367c8..96edd06c 100644 --- a/packages/core/src/presets/torznab.ts +++ b/packages/core/src/presets/torznab.ts @@ -114,6 +114,15 @@ export class TorznabPreset extends BuiltinAddonPreset { { label: 'Both', value: 'both' }, ], }, + { + id: 'paginate', + name: 'Paginate Results', + description: + 'Whether to paginate through all available results when searching. Enabling this can provide more results at the cost of increased search time and more requests.', + type: 'boolean', + default: false, + required: false, + }, { id: 'useMultipleInstances', name: 'Use Multiple Instances', @@ -223,6 +232,7 @@ export class TorznabPreset extends BuiltinAddonPreset { apiPath: options.apiPath, apiKey: options.apiKey, forceQuerySearch: options.forceQuerySearch ?? false, + paginate: options.paginate ?? false, }; const configString = this.base64EncodeJSON(config, 'urlSafe'); diff --git a/packages/core/src/utils/env.ts b/packages/core/src/utils/env.ts index 85b478fe..8cc2080f 100644 --- a/packages/core/src/utils/env.ts +++ b/packages/core/src/utils/env.ts @@ -1733,6 +1733,10 @@ export const Env = cleanEnv(process.env, { default: 14 * 24 * 60 * 60, // 14 days desc: 'Builtin Torznab/Newznab Capabilities cache TTL', }), + BUILTIN_NAB_MAX_PAGES: num({ + default: 5, + desc: 'Maximum number of pages to fetch from Torznab/Newznab indexers during pagination', + }), BUILTIN_ZILEAN_URL: url({ default: 'https://zilean.elfhosted.com', diff --git a/packages/core/src/utils/startup.ts b/packages/core/src/utils/startup.ts index cd4c9b5c..1f6cd7ec 100644 --- a/packages/core/src/utils/startup.ts +++ b/packages/core/src/utils/startup.ts @@ -574,6 +574,7 @@ const logStartupInfo = () => { formatDurationAsText(Env.BUILTIN_NAB_CAPABILITIES_CACHE_TTL), ' ' ); + logKeyValue('Max Pages:', Env.BUILTIN_NAB_MAX_PAGES.toString(), ' '); // Bitmagnet logKeyValue(