diff --git a/.env.sample b/.env.sample index dabef3d7..257c72c1 100644 --- a/.env.sample +++ b/.env.sample @@ -102,6 +102,11 @@ BUILTIN_GET_TORRENT_TIMEOUT=5000 # How many of the get torrent tasks should be running at the same time # Higher values will increase search times but use more system resources. BUILTIN_GET_TORRENT_CONCURRENCY=100 +# The minimum interval between background refreshes for built-in addon search caches. Triggered during normal searches. +# e.g. Searches remain cached for the addon-specific TTLs but once this interval has passed, a background refresh can be triggered to ensure fresh results +# without making the user wait. +# Default: 1 day +BUILTIN_MINIMUM_BACKGROUND_REFRESH_INTERVAL=86400 # ---- Bitmagnet ---- diff --git a/packages/core/src/builtins/base/nab/api.ts b/packages/core/src/builtins/base/nab/api.ts index a241a5c3..d309ab8e 100644 --- a/packages/core/src/builtins/base/nab/api.ts +++ b/packages/core/src/builtins/base/nab/api.ts @@ -10,6 +10,7 @@ import { } from '../../../utils/index.js'; import { Parser } from 'xml2js'; import { Logger } from 'winston'; +import { searchWithBackgroundRefresh } from '../../utils/general.js'; // --- Generic Custom Error --- export class NabApiError extends Error { @@ -333,12 +334,21 @@ export class BaseNabApi { params: Record = {} ): Promise> { const cacheKey = `${this.baseUrl}${this.apiPath}?t=${searchFunction}&${JSON.stringify(params)}&apikey=${this.apiKey}`; - const result = await this.searchCache.wrap( - () => this.request(searchFunction, this.SearchResultSchema, params), - cacheKey, - Env.BUILTIN_NAB_SEARCH_CACHE_TTL - ); - return result as SearchResponse; + + return searchWithBackgroundRefresh({ + searchCache: this.searchCache as Cache>, + searchCacheKey: cacheKey, + bgCacheKey: `nab:${cacheKey}`, + cacheTTL: Env.BUILTIN_NAB_SEARCH_CACHE_TTL, + fetchFn: () => + this.request( + searchFunction, + this.SearchResultSchema, + params + ) as Promise>, + isEmptyResult: (result) => result.results.length === 0, + logger: this.logger, + }); } private removeTrailingSlash = (path: string) => diff --git a/packages/core/src/builtins/knaben/api.ts b/packages/core/src/builtins/knaben/api.ts index 91d39e79..225619aa 100644 --- a/packages/core/src/builtins/knaben/api.ts +++ b/packages/core/src/builtins/knaben/api.ts @@ -2,6 +2,7 @@ import { Cache } from '../../utils/cache.js'; import { Env } from '../../utils/env.js'; import { formatZodError, makeRequest } from '../../utils/index.js'; import { createLogger } from '../../utils/index.js'; +import { searchWithBackgroundRefresh } from '../utils/general.js'; import { z } from 'zod'; @@ -102,9 +103,10 @@ const API_VERSION = '1'; class KnabenAPI { private headers: Record; - private readonly searchCache = Cache.getInstance( - 'knaben:search' - ); + private readonly searchCache = Cache.getInstance< + string, + KnabenSearchResponse + >('knaben:search'); constructor() { this.headers = { @@ -117,18 +119,23 @@ class KnabenAPI { async search(options: KnabenSearchOptions): Promise { const body = KnabenSearchOptionsRequest.parse(options); + const cacheKey = JSON.stringify(options); - return this.searchCache.wrap( - () => + return searchWithBackgroundRefresh({ + searchCache: this.searchCache, + searchCacheKey: cacheKey, + bgCacheKey: `knaben:${cacheKey}`, + cacheTTL: Env.BUILTIN_KNABEN_SEARCH_CACHE_TTL, + fetchFn: () => this.request('', { schema: KnabenSearchResponse, method: 'POST', timeout: Env.BUILTIN_KNABEN_SEARCH_TIMEOUT, body, }), - `knaben:search:${JSON.stringify(options)}`, - Env.BUILTIN_KNABEN_SEARCH_CACHE_TTL - ); + isEmptyResult: (result) => result.hits.length === 0, + logger, + }); } private async request( diff --git a/packages/core/src/builtins/prowlarr/api.ts b/packages/core/src/builtins/prowlarr/api.ts index 0c4dc94b..d3a3bd59 100644 --- a/packages/core/src/builtins/prowlarr/api.ts +++ b/packages/core/src/builtins/prowlarr/api.ts @@ -1,12 +1,14 @@ import { fetch } from 'undici'; import { Cache, + createLogger, DistributedLock, Env, formatZodError, makeRequest, } from '../../utils/index.js'; import z from 'zod'; +import { searchWithBackgroundRefresh } from '../utils/general.js'; interface ResponseMeta { headers: Record; @@ -80,6 +82,8 @@ const ProwlarrApiSearchItemSchema = z.object({ const ProwlarrApiSearchSchema = z.array(ProwlarrApiSearchItemSchema); +const logger = createLogger('prowlarr'); + export type ProwlarrApiSearchItem = z.infer; class ProwlarrApi { @@ -90,7 +94,7 @@ class ProwlarrApi { private readonly searchCache = Cache.getInstance< string, - ProwlarrApiSearchItem[] + ProwlarrApiResponse >('prowlarr-api:search'); private readonly indexersCache = Cache.getInstance< @@ -156,8 +160,14 @@ class ProwlarrApi { limit?: number; offset?: number; }): Promise> { - return this.searchCache.wrap( - () => + const cacheKey = `${this.baseUrl}:${type}:${query}:${indexerIds.join(',')}:${limit}:${offset}`; + + return searchWithBackgroundRefresh({ + searchCache: this.searchCache, + searchCacheKey: cacheKey, + bgCacheKey: `prowlarr:${cacheKey}`, + cacheTTL: Env.BUILTIN_PROWLARR_SEARCH_CACHE_TTL, + fetchFn: () => this.request( 'search', { @@ -169,9 +179,9 @@ class ProwlarrApi { }, ProwlarrApiSearchSchema ), - `${this.baseUrl}:${type}:${query}:${indexerIds.join(',')}:${limit}:${offset}`, - Env.BUILTIN_PROWLARR_SEARCH_CACHE_TTL - ); + isEmptyResult: (result) => result.data.length === 0, + logger, + }); } private getPath(endpoint: string) { diff --git a/packages/core/src/builtins/torrent-galaxy/api.ts b/packages/core/src/builtins/torrent-galaxy/api.ts index f1b99f9f..d6f5e4a6 100644 --- a/packages/core/src/builtins/torrent-galaxy/api.ts +++ b/packages/core/src/builtins/torrent-galaxy/api.ts @@ -2,6 +2,7 @@ import { Cache } from '../../utils/cache.js'; import { Env } from '../../utils/env.js'; import { formatZodError, makeRequest } from '../../utils/index.js'; import { createLogger } from '../../utils/index.js'; +import { searchWithBackgroundRefresh } from '../utils/general.js'; import { z } from 'zod'; @@ -75,9 +76,10 @@ const API_BASE_URL = Env.BUILTIN_TORRENT_GALAXY_URL; class TorrentGalaxyAPI { private headers: Record; - private readonly searchCache = Cache.getInstance( - 'torrent-galaxy:search' - ); + private readonly searchCache = Cache.getInstance< + string, + TorrentGalaxySearchResponse + >('torrent-galaxy:search'); constructor() { this.headers = { @@ -95,8 +97,13 @@ class TorrentGalaxyAPI { if (options.page) { queryParams.set('page', options.page.toString()); } - return this.searchCache.wrap( - () => + const cacheKey = JSON.stringify(options); + return searchWithBackgroundRefresh({ + searchCache: this.searchCache, + searchCacheKey: cacheKey, + bgCacheKey: `tgx:${cacheKey}`, + cacheTTL: Env.BUILTIN_TORRENT_GALAXY_SEARCH_CACHE_TTL, + fetchFn: () => this.request( `/get-posts/keywords:${encodeURIComponent(options.query)}:format:json`, { @@ -105,9 +112,9 @@ class TorrentGalaxyAPI { queryParams, } ), - `${JSON.stringify(options)}`, - Env.BUILTIN_TORRENT_GALAXY_SEARCH_CACHE_TTL - ); + isEmptyResult: (result) => result.results.length === 0, + logger, + }); } private async request( diff --git a/packages/core/src/builtins/utils/general.ts b/packages/core/src/builtins/utils/general.ts index 64f23c59..d752b530 100644 --- a/packages/core/src/builtins/utils/general.ts +++ b/packages/core/src/builtins/utils/general.ts @@ -1,4 +1,7 @@ +import { Logger } from 'winston'; +import { Cache } from '../../utils/cache.js'; import { Env } from '../../utils/env.js'; +import { createLogger } from '../../utils/index.js'; import pLimit from 'p-limit'; export const createQueryLimit = () => @@ -37,3 +40,134 @@ export function useAllTitles(url: string): boolean { } return !!Env.BUILTIN_SCRAPE_WITH_ALL_TITLES; } + +export const bgRefreshCache = Cache.getInstance( + 'builtins:bg-refresh' +); + +/** + * Options for the searchWithBackgroundRefresh function + */ +interface SearchWithBgRefreshOptions { + searchCache: Cache; + searchCacheKey: string; + bgCacheKey: string; + cacheTTL: number; + fetchFn: () => Promise; + isEmptyResult: (result: T) => boolean; + logger: Logger; +} + +/** + * Performs a cached search with background refresh support. + * + * When a cached result exists: + * - Returns the cached result immediately + * - Schedules a background refresh if the minimum interval has passed + * + * When no cached result exists: + * - Performs the search synchronously + * - Caches the result (unless empty) + * - Records the refresh timestamp + * + * @param options - Configuration options for the search + * @returns The search result (cached or fresh) + */ +export async function searchWithBackgroundRefresh( + options: SearchWithBgRefreshOptions +): Promise { + const { + searchCacheKey, + bgCacheKey, + searchCache, + cacheTTL, + fetchFn, + isEmptyResult, + logger, + } = options; + + const cachedResult = await searchCache.get(searchCacheKey); + + if (cachedResult !== undefined) { + triggerBackgroundRefresh({ + searchCache, + searchCacheKey, + bgCacheKey, + cacheTTL, + fetchFn, + isEmptyResult, + logger, + }); + return cachedResult; + } + + const result = await fetchFn(); + + // Don't cache empty results + if (!isEmptyResult(result)) { + await searchCache.set(searchCacheKey, result, cacheTTL); + await bgRefreshCache.set( + bgCacheKey, + Date.now(), + Env.BUILTIN_MINIMUM_BACKGROUND_REFRESH_INTERVAL + ); + } + + return result; +} + +/** + * Triggers a background refresh if the minimum interval has passed. + * This function is fire-and-forget and does not block. + */ +function triggerBackgroundRefresh(options: { + searchCache: Cache; + searchCacheKey: string; + bgCacheKey: string; + cacheTTL: number; + fetchFn: () => Promise; + isEmptyResult: (result: T) => boolean; + logger: Logger; +}): void { + const { + searchCacheKey, + bgCacheKey, + searchCache, + cacheTTL, + fetchFn, + isEmptyResult, + logger, + } = options; + + (async () => { + try { + const lastRefresh = await bgRefreshCache.get(bgCacheKey); + const now = Date.now(); + const intervalMs = Env.BUILTIN_MINIMUM_BACKGROUND_REFRESH_INTERVAL * 1000; + + if (lastRefresh && now - lastRefresh < intervalMs) { + // Not enough time has passed since last refresh + return; + } + + // Perform background refresh + logger.debug(`Starting background refresh for: ${searchCacheKey}`); + const freshResult = await fetchFn(); + + // Update cache if result is not empty + if (!isEmptyResult(freshResult)) { + await searchCache.set(searchCacheKey, freshResult, cacheTTL, true); + await bgRefreshCache.set( + bgCacheKey, + now, + Env.BUILTIN_MINIMUM_BACKGROUND_REFRESH_INTERVAL + ); + logger.info(`Background refreshed cache for: ${searchCacheKey}`); + } + } catch (error) { + logger.error( + `Background refresh failed for: ${searchCacheKey} - ${error instanceof Error ? error.message : 'Unknown error'}` + ); + } + })(); +} diff --git a/packages/core/src/utils/env.ts b/packages/core/src/utils/env.ts index 76c10033..04c36d3a 100644 --- a/packages/core/src/utils/env.ts +++ b/packages/core/src/utils/env.ts @@ -1696,6 +1696,10 @@ export const Env = cleanEnv(process.env, { default: 7 * 24 * 60 * 60, // 7 days desc: 'Builtin Torrent metadata cache TTL', }), + BUILTIN_MINIMUM_BACKGROUND_REFRESH_INTERVAL: num({ + default: 1 * 24 * 60 * 60, // 1 day + desc: 'Minimum interval between background refreshes for built-in addon search caches. Triggered during normal searches.', + }), BUILTIN_GDRIVE_CLIENT_ID: str({ default: undefined, diff --git a/packages/core/src/utils/torrent.ts b/packages/core/src/utils/torrent.ts index f8120f1b..fa902359 100644 --- a/packages/core/src/utils/torrent.ts +++ b/packages/core/src/utils/torrent.ts @@ -23,7 +23,9 @@ interface TorrentMetadata { export class TorrentClient { static readonly #metadataCache = Cache.getInstance( - 'torrent-metadata' + 'torrent-metadata', + undefined, + Env.REDIS_URI ? 'redis' : 'sql' ); // Track in-progress fetches to avoid duplicate requests