feat: stuff

This commit is contained in:
Viren070
2025-09-13 17:32:20 +01:00
parent 36fdeb4672
commit 54573b571b
113 changed files with 17429 additions and 16550 deletions
+64 -5
View File
@@ -71,6 +71,25 @@ DATABASE_URI=sqlite://./data/db.sqlite
# Set the URL to your StremThru instance here:
BUILTIN_STREMTHRU_URL=https://stremthru.13377001.xyz
# How long the cache status of an item is cached. Only applies to cached items, uncached items are always checked again.
# Default: 30 minutes
BUILTIN_DEBRID_INSTANT_AVAILABILITY_CACHE_TTL=1800
# How long download links for an item remain cached for.
# Default: 1 hour
BUILTIN_DEBRID_PLAYBACK_LINK_CACHE_TTL=3600
# ---- Prowlarr ------
# To enable the built-in prowlarr addon, provide the URL and API key here.
BUILTIN_PROWLARR_URL=
BUILTIN_PROWLARR_API_KEY=
# Optionally provide a comma separated list of indexers to limit the addon to use. If not provided, it uses all indexers.
# BUILTIN_PROWLARR_INDEXERS=
# The timeout for search requests.
BUILTIN_PROWLARR_SEARCH_TIMEOUT=
# How long specific responses should be cached for.
BUILTIN_PROWLARR_SEARCH_CACHE_TTL=604800
BUILTIN_PROWLARR_INDEXERS_CACHE_TTL=1209600
# ---- Stremio GDrive -----
# Client ID and Secret generated following this guide: https://guides.viren070.me/stremio/addons/stremio-gdrive
@@ -92,9 +111,42 @@ BUILTIN_TORBOX_SEARCH_SEARCH_API_TIMEOUT=30000
# Whether to cache results for users who've enabled user search engines. This requires a cache entry for each user for each title rather than
# A shared cache.
BUILTIN_TORBOX_SEARCH_CACHE_PER_USER_SEARCH_ENGINE=false
# BUILTIN_TORBOX_SEARCH_SEARCH_API_CACHE_TTL=3600
# BUILTIN_TORBOX_SEARCH_METADATA_CACHE_TTL=604800
# BUILTIN_TORBOX_SEARCH_INSTANT_AVAILABILITY_CACHE_TTL=900
# The amount of time that the results of a search remain cached for. Applies to both usenet and torrents.
# Default: 7 days.
# BUILTIN_TORBOX_SEARCH_SEARCH_API_CACHE_TTL=604800
# BUILTIN_TORBOX_SEARCH_METADATA_CACHE_TTL=1209600
# ---- Torznab ----
BUILTIN_TORZNAB_SEARCH_TIMEOUT=30000
BUILTIN_TORZNAB_SEARCH_CACHE_TTL=604800
BUILTIN_TORZNAB_CAPABILITIES_CACHE_TTL=1209600
# --- Newznab ---
BUILTIN_NEWZNAB_SEARCH_TIMEOUT=30000
BUILTIN_NEWZNAB_SEARCH_CACHE_TTL=604800
BUILTIN_TORZNAB_CAPABILITIES_CACHE_TTL=1209600
# --- Zilean ---
BUILTIN_ZILEAN_URL="https://zilean.elfhosted.com"
# BUILTIN_ZILEAN_TIMEOUT=
# --- AnimeTosho ---
BUILTIN_ANIMETOSHO_URL="https://feed.animetosho.org"
# BUILTIN_ANIMETOSHO_TIMEOUT=
# =============================================================================
# # ANIME DATABASE
# =============================================================================
# Control how often the datasources for the local anime database will refresh.
ANIME_DB_FRIBB_MAPPINGS_REFRESH_INTERVAL=86400000
ANIME_DB_MANAMI_DB_REFRESH_INTERVAL=604800000
ANIME_DB_KITSU_IMDB_MAPPING_REFRESH_INTERVAL=86400000
ANIME_DB_EXTENDED_ANITRAKT_MOVIES_REFRESH_INTERVAL=86400000
ANIME_DB_EXTENDED_ANITRAKT_TV_REFRESH_INTERVAL=86400000
# ==============================================================================
# DEBRID & OTHER SERVICE API KEYS
@@ -104,6 +156,9 @@ BUILTIN_TORBOX_SEARCH_CACHE_PER_USER_SEARCH_ENGINE=false
TMDB_ACCESS_TOKEN=
TMDB_API_KEY=
# Provide a trakt client ID for authorised requests to get trakt aliases.
TRAKT_CLIENT_ID=
# Configure API keys for debrid services and others you plan to use.
# 'DEFAULT_' values are pre-filled in the user's config page.
# 'FORCED_' values override user settings and hide the option.
@@ -463,6 +518,10 @@ FORMAT_API_RATE_LIMIT_MAX_REQUESTS=30
CATALOG_API_RATE_LIMIT_WINDOW=5
CATALOG_API_RATE_LIMIT_MAX_REQUESTS=5
# --- Anime API ---
ANIME_API_RATE_LIMIT_WINDOW=60
ANIME_API_RATE_LIMIT_MAX_REQUESTS=120
# --- Stremio Stream ---
STREMIO_STREAM_RATE_LIMIT_WINDOW=15
STREMIO_STREAM_RATE_LIMIT_MAX_REQUESTS=10
@@ -543,14 +602,14 @@ PRUNE_MAX_DAYS=-1
# FORCE_JACKETTIO_PROTOCOL=
# --------- STREMTHRU-STORE ---------
# STREMTHRU_STORE_URL=https://stremthru.elfhosted.com/stremio/store/
# STREMTHRU_STORE_URL=https://stremthru.13377001.xyz/stremio/store/
# DEFAULT_STREMTHRU_STORE_TIMEOUT=
# Advanced: Override StremThru Store hostname/port/protocol (similar to Comet).
# FORCE_STREMTHRU_STORE_HOSTNAME=
# FORCE_STREMTHRU_STORE_PORT=
# FORCE_STREMTHRU_STORE_PROTOCOL=
# --------- STREMTHRU-TORZ -----
# STREMTHRU_TORZ_URL=https://stremthru.elfhosted.com/stremio/torz/
# STREMTHRU_TORZ_URL=https://stremthru.13377001.xyz/stremio/torz/
# DEFAULT_STREMTHRU_TORZ_TIMEOUT=
# Advanced: Override StremThru Torz hostname/port/protocol (similar to Comet).
# FORCE_STREMTHRU_TORZ_HOSTNAME=
+25 -6
View File
@@ -1,4 +1,10 @@
FROM node:22-alpine AS builder
FROM node:22-alpine AS base
ENV PNPM_HOME="/pnpm"
ENV PATH="$PNPM_HOME:$PATH"
RUN corepack enable
FROM base AS builder
WORKDIR /build
@@ -10,9 +16,11 @@ COPY package*.json ./
COPY packages/server/package*.json ./packages/server/
COPY packages/core/package*.json ./packages/core/
COPY packages/frontend/package*.json ./packages/frontend/
COPY pnpm-workspace.yaml ./pnpm-workspace.yaml
COPY pnpm-lock.yaml ./pnpm-lock.yaml
# Install dependencies.
RUN npm install
RUN pnpm install --frozen-lockfile
# Copy source files.
COPY tsconfig.*json ./
@@ -25,12 +33,18 @@ COPY resources ./resources
# Build the project.
RUN npm run build
RUN pnpm run build
# Remove development dependencies.
RUN npm --workspaces prune --omit=dev
RUN rm -rf node_modules
RUN rm -rf packages/core/node_modules
RUN rm -rf packages/server/node_modules
RUN rm -rf packages/frontend/node_modules
FROM node:22-alpine AS final
RUN pnpm install --prod --frozen-lockfile
FROM base AS final
RUN apk add --no-cache curl
WORKDIR /app
@@ -38,6 +52,8 @@ WORKDIR /app
# Copy the built files from the builder.
# The package.json files must be copied as well for NPM workspace symlinks between local packages to work.
COPY --from=builder /build/package*.json /build/LICENSE ./
COPY --from=builder /build/pnpm-workspace.yaml ./pnpm-workspace.yaml
COPY --from=builder /build/pnpm-lock.yaml ./pnpm-lock.yaml
COPY --from=builder /build/packages/core/package.*json ./packages/core/
COPY --from=builder /build/packages/frontend/package.*json ./packages/frontend/
@@ -51,10 +67,13 @@ COPY --from=builder /build/packages/server/src/static ./packages/server/dist/sta
COPY --from=builder /build/resources ./resources
COPY --from=builder /build/node_modules ./node_modules
COPY --from=builder /build/packages/core/node_modules ./packages/core/node_modules
COPY --from=builder /build/packages/server/node_modules ./packages/server/node_modules
COPY --from=builder /build/packages/frontend/node_modules ./packages/frontend/node_modules
HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
CMD curl -fsS http://localhost:${PORT:-3000}/api/v1/status || exit 1
EXPOSE ${PORT:-3000}
ENTRYPOINT ["npm", "run", "start"]
ENTRYPOINT ["pnpm", "run", "start"]
-14997
View File
File diff suppressed because it is too large Load Diff
+6 -5
View File
@@ -3,18 +3,19 @@
"version": "2.12.2",
"description": "AIOStreams consolidates multiple Stremio addons and debrid services into a single, easily configurable addon. It allows highly customisable filtering, sorting, and formatting of results and supports proxying all your streams through MediaFlow Proxy or StremThru for improved compatibility and IP restriction bypassing.",
"main": "dist/index.js",
"type": "module",
"scripts": {
"test": "npm run test --workspaces",
"test": "pnpm run test --workspaces",
"release": "commit-and-tag-version",
"format": "prettier --write .",
"metadata": "node scripts/generateMetadata.js",
"build": "npm -w packages/core run build && npm -w packages/server run build && npm -w packages/frontend run build",
"build": "pnpm -F core run build && pnpm -F server run build && pnpm -F frontend run build",
"build:watch": "tsc --build --watch",
"start": "node packages/server/dist/server",
"start:addon": "npm run start",
"start:addon": "pnpm run start",
"start:dev": "cross-env NODE_ENV=development tsx watch packages/server/src/server.ts",
"start:addon:dev": "npm run start:dev",
"start:frontend:dev": "npm -w packages/frontend run dev"
"start:addon:dev": "pnpm run start:dev",
"start:frontend:dev": "pnpm -F frontend run dev"
},
"author": "Viren070",
"license": "MIT",
+5 -1
View File
@@ -16,6 +16,8 @@
"envalid": "^8.0.0",
"expr-eval": "^2.0.2",
"fetch-socks": "^1.3.2",
"fuzzball": "^2.2.3",
"go-ptt": "^0.4.0",
"moment-timezone": "^0.5.48",
"parse-torrent-title": "github:TheBeastLT/parse-torrent-title",
"pg": "^8.16.0",
@@ -26,12 +28,14 @@
"super-regex": "^1.0.0",
"undici": "^7.2.3",
"winston": "^3.17.0",
"xml2js": "^0.6.2",
"zod": "^4.1.5"
},
"devDependencies": {
"@types/bcrypt": "^5.0.2",
"@types/bytes": "^3.1.5",
"@types/node": "^20.14.10",
"@types/pg": "^8.15.2"
"@types/pg": "^8.15.2",
"@types/xml2js": "^0.4.14"
}
}
+385
View File
@@ -0,0 +1,385 @@
import { Manifest, Meta, Stream } from '../../db/schemas';
import { z, ZodError } from 'zod';
import { IdParser, IdType, ParsedId } from '../../utils/id-parser';
import {
AnimeDatabase,
constants,
Env,
formatZodError,
getTimeTakenSincePoint,
SERVICE_DETAILS,
} from '../../utils';
import { TorrentClient } from '../../utils/torrent';
import {
BuiltinDebridServices,
PlaybackInfo,
Torrent,
NZB,
TorrentWithSelectedFile,
NZBWithSelectedFile,
UnprocessedTorrent,
ServiceAuth,
} from '../../debrid';
import { processTorrents, processNZBs } from '../utils/debrid';
import { calculateAbsoluteEpisode } from '../utils/general';
import { TitleMetadata } from '../torbox-search/source-handlers';
import { MetadataService } from '../../metadata/service';
import { Logger } from 'winston';
export interface SearchMetadata extends TitleMetadata {
primaryTitle?: string;
year?: number;
imdbId?: string | null;
tmdbId?: string | null;
tvdbId?: string | null;
}
export const BaseDebridConfigSchema = z.object({
services: BuiltinDebridServices,
tmdbApiKey: z.string().optional(),
tmdbReadAccessToken: z.string().optional(),
});
export type BaseDebridConfig = z.infer<typeof BaseDebridConfigSchema>;
export abstract class BaseDebridAddon<T extends BaseDebridConfig> {
abstract readonly id: string;
abstract readonly name: string;
abstract readonly version: string;
get addonId(): string {
return `com.${this.name.toLowerCase().replace(/\s/g, '')}.viren070`;
}
abstract readonly logger: Logger;
protected readonly userData: T;
protected readonly clientIp?: string;
private static readonly supportedIdTypes: IdType[] = [
'imdbId',
'kitsuId',
'malId',
'themoviedbId',
'thetvdbId',
];
constructor(userData: T, configSchema: z.ZodType<T>, clientIp?: string) {
try {
this.userData = configSchema.parse(userData);
} catch (error) {
throw new Error(
`Invalid user data: ${formatZodError(error as ZodError)}`
);
}
this.clientIp = clientIp;
}
public getManifest(): Manifest {
return {
id: this.addonId,
name: this.name,
version: this.version,
types: ['movie', 'series', 'anime'],
catalogs: [],
description: `${this.name} addon`,
resources: [
{
name: 'stream',
types: ['movie', 'series', 'anime'],
idPrefixes: IdParser.getPrefixes(BaseDebridAddon.supportedIdTypes),
},
],
};
}
public async getStreams(type: string, id: string): Promise<Stream[]> {
const parsedId = IdParser.parse(id, type);
if (
!parsedId ||
!BaseDebridAddon.supportedIdTypes.includes(parsedId.type)
) {
throw new Error(`Unsupported ID: ${id}`);
}
this.logger.info(`Handling stream request for ${this.name}`, {
requestType: type,
requestId: id,
});
const searchMetadata = await this._getSearchMetadata(parsedId, type);
const searchPromises = await Promise.allSettled([
this._searchTorrents(parsedId, searchMetadata),
this._searchNzbs(parsedId, searchMetadata),
]);
let torrentResults =
searchPromises[0].status === 'fulfilled' ? searchPromises[0].value : [];
const nzbResults =
searchPromises[1].status === 'fulfilled' ? searchPromises[1].value : [];
const searchErrors: Stream[] = [];
if (searchPromises[0].status === 'rejected') {
searchErrors.push(
this._createErrorStream({
title: `${this.name}`,
description: searchPromises[0].reason.message,
})
);
}
if (searchPromises[1].status === 'rejected') {
searchErrors.push(
this._createErrorStream({
title: `${this.name}`,
description: searchPromises[1].reason.message,
})
);
}
if (torrentResults.some((t) => !t.hash && t.downloadUrl)) {
// Process torrents in batches of 5 to avoid overwhelming the system
const BATCH_SIZE = 15;
const enrichedResults: Torrent[] = [];
const torrentsToProcess = [...torrentResults];
while (torrentsToProcess.length > 0) {
const batch = torrentsToProcess.splice(0, BATCH_SIZE);
const metadataPromises = batch.map(async (torrent) => {
try {
const metadata = await TorrentClient.getMetadata(torrent);
if (!metadata) {
return torrent.hash ? (torrent as Torrent) : null;
}
return {
...torrent,
hash: metadata.hash,
sources: metadata.sources,
files: metadata.files,
} as Torrent;
} catch (error) {
this.logger.error(`Failed to fetch metadata for torrent: ${error}`);
return torrent.hash ? (torrent as Torrent) : null;
}
});
const batchResults = await Promise.all(metadataPromises);
enrichedResults.push(
...batchResults.filter((r): r is Torrent => r !== null)
);
}
torrentResults = enrichedResults;
}
const [processedTorrents, processedNzbs] = await Promise.all([
processTorrents(
torrentResults as Torrent[],
this.userData.services,
id,
searchMetadata,
this.clientIp
),
processNZBs(
nzbResults,
this.userData.services,
id,
searchMetadata,
this.clientIp
),
]);
const resultStreams = [
...processedTorrents.results,
...processedNzbs.results,
].map((result) =>
this._createStream(result, this.userData, searchMetadata)
);
const processingErrors = [
...processedTorrents.errors,
...processedNzbs.errors,
].map((error) =>
this._createErrorStream({
title: `${this.name} ${constants.SERVICE_DETAILS[error.serviceId].shortName}`,
description: error.error.message,
})
);
return [...resultStreams, ...searchErrors, ...processingErrors];
}
protected abstract _searchTorrents(
parsedId: ParsedId,
metadata: SearchMetadata
): Promise<UnprocessedTorrent[]>;
protected abstract _searchNzbs(
parsedId: ParsedId,
metadata: SearchMetadata
): Promise<NZB[]>;
protected async _getSearchMetadata(
parsedId: ParsedId,
type: string
): Promise<SearchMetadata> {
const start = Date.now();
const animeEntry = AnimeDatabase.getInstance().getEntryById(
parsedId.type,
parsedId.value
);
// Update season from anime entry if available
if (animeEntry && !parsedId.season) {
parsedId.season =
animeEntry.imdb?.fromImdbSeason?.toString() ??
animeEntry.trakt?.season?.toString();
}
const metadata = await new MetadataService({
tmdbAccessToken: this.userData.tmdbReadAccessToken,
tmdbApiKey: this.userData.tmdbApiKey,
}).getMetadata(parsedId, type === 'movie' ? 'movie' : 'series');
// Calculate absolute episode if needed
let absoluteEpisode: number | undefined;
if (animeEntry && parsedId.season && parsedId.episode && metadata.seasons) {
const seasons = metadata.seasons.map(
({ season_number, episode_count }) => ({
number: season_number.toString(),
episodes: episode_count,
})
);
this.logger.debug(
`Calculating absolute episode with current season and episode: ${parsedId.season}, ${parsedId.episode} and seasons: ${JSON.stringify(seasons)}`
);
absoluteEpisode = Number(
calculateAbsoluteEpisode(parsedId.season, parsedId.episode, seasons)
);
}
// Map IDs
const imdbId =
parsedId.type === 'imdbId'
? parsedId.value.toString()
: (animeEntry?.mappings?.imdbId?.toString() ?? null);
const tmdbId =
parsedId.type === 'themoviedbId'
? parsedId.value.toString()
: (animeEntry?.mappings?.themoviedbId?.toString() ?? null);
const tvdbId =
parsedId.type === 'thetvdbId'
? parsedId.value.toString()
: (animeEntry?.mappings?.thetvdbId?.toString() ?? null);
const searchMetadata: SearchMetadata = {
primaryTitle: metadata.title,
titles: metadata.titles ?? [],
season: parsedId.season ? Number(parsedId.season) : undefined,
episode: parsedId.episode ? Number(parsedId.episode) : undefined,
absoluteEpisode,
year: metadata.year,
imdbId,
tmdbId,
tvdbId,
};
this.logger.debug(
`Got search metadata for ${parsedId.type}:${parsedId.value} in ${getTimeTakenSincePoint(start)}`,
{
...searchMetadata,
titles: searchMetadata.titles.length,
}
);
return searchMetadata;
}
protected _createStream(
torrentOrNzb: TorrentWithSelectedFile | NZBWithSelectedFile,
userData: T,
titleMetadata?: TitleMetadata
): Stream {
// Handle debrid streaming
const storeAuth: ServiceAuth | undefined = torrentOrNzb.service
? {
id: torrentOrNzb.service!.id,
credential:
userData.services.find(
(service) => service.id === torrentOrNzb.service!.id
)?.credential ?? '',
}
: undefined;
const playbackInfo: PlaybackInfo | undefined = torrentOrNzb.service
? torrentOrNzb.type === 'torrent'
? {
type: 'torrent',
hash: torrentOrNzb.hash,
sources: torrentOrNzb.sources,
title: torrentOrNzb.title,
file: torrentOrNzb.file,
metadata: titleMetadata,
}
: {
type: 'usenet',
nzb: torrentOrNzb.nzb,
title: torrentOrNzb.title,
hash: torrentOrNzb.hash,
file: torrentOrNzb.file,
metadata: titleMetadata,
}
: undefined;
const svcMeta = torrentOrNzb.service
? SERVICE_DETAILS[torrentOrNzb.service.id]
: undefined;
// const svcMeta = SERVICE_DETAILS[torrentOrNzb.service.id];
const shortCode = svcMeta?.shortName || 'P2P';
const cacheIndicator = torrentOrNzb.service
? torrentOrNzb.service.cached
? '⚡'
: '⏳'
: '';
const name = `[${shortCode} ${cacheIndicator}${torrentOrNzb.service?.owned ? ' ☁️' : ''}] ${this.name}`;
const description = `${torrentOrNzb.title}\n${torrentOrNzb.file.name}\n${
torrentOrNzb.indexer ? `🔍 ${torrentOrNzb.indexer}` : ''
} ${'seeders' in torrentOrNzb && torrentOrNzb.seeders ? `👤 ${torrentOrNzb.seeders}` : ''} ${
torrentOrNzb.age && torrentOrNzb.age !== '0d'
? `🕒 ${torrentOrNzb.age}`
: ''
}`;
return {
url: torrentOrNzb.service
? `${Env.BASE_URL}/api/v1/debrid/playback/${encodeURIComponent(
Buffer.from(JSON.stringify(storeAuth)).toString('base64')
)}/${encodeURIComponent(
Buffer.from(JSON.stringify(playbackInfo)).toString('base64')
)}/${encodeURIComponent(torrentOrNzb.file.name || torrentOrNzb.title || 'unknown')}`
: undefined,
name,
description,
type: torrentOrNzb.type,
infoHash: torrentOrNzb.hash,
fileIdx: torrentOrNzb.file.index,
behaviorHints: {
videoSize: torrentOrNzb.file.size,
filename: torrentOrNzb.file.name,
},
};
}
protected _createErrorStream({
title,
description,
}: {
title: string;
description: string;
}): Stream {
return {
name: `[❌] ${title}`,
description: description,
externalUrl: 'stremio:///',
};
}
}
@@ -0,0 +1,181 @@
import { z } from 'zod';
import { ParsedId } from '../../../utils/id-parser';
import { getTimeTakenSincePoint } from '../../../utils';
import { Logger } from 'winston';
import {
BaseDebridAddon,
BaseDebridConfigSchema,
SearchMetadata,
} from '../debrid';
import { BaseNabApi, Capabilities, SearchResultItem } from './api';
export const NabAddonConfigSchema = BaseDebridConfigSchema.extend({
url: z.string(),
apiKey: z.string().optional(),
apiPath: z.string().optional(),
forceQuerySearch: z.boolean().default(false),
});
export type NabAddonConfig = z.infer<typeof NabAddonConfigSchema>;
export abstract class BaseNabAddon<
C extends NabAddonConfig,
A extends BaseNabApi<'torznab' | 'newznab'>,
> extends BaseDebridAddon<C> {
abstract api: A;
protected async performSearch(
parsedId: ParsedId,
metadata: SearchMetadata
): Promise<SearchResultItem<A['namespace']>[]> {
const start = Date.now();
const queryParams: Record<string, string> = {};
let capabilities: Capabilities;
try {
capabilities = await this.api.getCapabilities();
} catch (error) {
throw new Error(
`Could not get capabilities: ${error instanceof Error ? error.message : String(error)}`
);
}
this.logger.debug(`Capabilities: ${JSON.stringify(capabilities)}`);
const chosenFunction = this.getSearchFunction(
parsedId.mediaType,
capabilities.searching
);
if (!chosenFunction)
throw new Error(
`Could not find a search function for ${capabilities.server.title}`
);
const { capabilities: searchCapabilities, function: searchFunction } =
chosenFunction;
this.logger.debug(`Using search function: ${searchFunction}`, {
searchCapabilities,
});
queryParams.limit = capabilities.limits?.max?.toString() ?? '10000';
if (this.userData.forceQuerySearch) {
} else if (
searchCapabilities.supportedParams.includes('imdbid') &&
metadata.imdbId
)
queryParams.imdbid = metadata.imdbId.replace('tt', '');
else if (
searchCapabilities.supportedParams.includes('tmdbid') &&
metadata.tmdbId
)
queryParams.tmdbid = metadata.tmdbId;
else if (
searchCapabilities.supportedParams.includes('tvdbid') &&
metadata.tvdbId
)
queryParams.tvdbid = metadata.tvdbId;
if (
!this.userData.forceQuerySearch &&
searchCapabilities.supportedParams.includes('season') &&
parsedId.season
)
queryParams.season = parsedId.season.toString();
if (
!this.userData.forceQuerySearch &&
searchCapabilities.supportedParams.includes('ep') &&
parsedId.episode
)
queryParams.ep = parsedId.episode.toString();
if (
!this.userData.forceQuerySearch &&
searchCapabilities.supportedParams.includes('year') &&
metadata.year &&
parsedId.mediaType === 'movie'
)
queryParams.year = metadata.year.toString();
let queries: string[] = [];
if (
!queryParams.imdbid &&
!queryParams.tmdbid &&
!queryParams.tvdbid &&
searchCapabilities.supportedParams.includes('q') &&
metadata.primaryTitle
) {
queries.push(metadata.primaryTitle);
if (
parsedId.season &&
parsedId.episode &&
!queryParams.season &&
!queryParams.ep
) {
queries = [
`${metadata.primaryTitle} S${parsedId.season.toString().padStart(2, '0')}`,
`${metadata.primaryTitle} S${parsedId.season.toString().padStart(2, '0')}E${parsedId.episode.toString().padStart(2, '0')}`,
];
if (metadata.absoluteEpisode)
queries.push(
`${metadata.primaryTitle} ${metadata.absoluteEpisode.toString().padStart(2, '0')}`
);
}
if (
metadata.year &&
parsedId.mediaType === 'movie' &&
!queryParams.year
) {
queries = queries.map((q) => `${q} ${metadata.year}`);
}
}
let results: SearchResultItem<A['namespace']>[] = [];
if (queries.length > 0) {
this.logger.debug('Performing queries', { queries });
const searchPromises = queries.map((q) =>
this.api.search(searchFunction, { ...queryParams, q })
);
const allResults = await Promise.all(searchPromises);
results = allResults.flat() as SearchResultItem<A['namespace']>[];
} else {
results = (await this.api.search(
searchFunction,
queryParams
)) as unknown as SearchResultItem<A['namespace']>[];
}
this.logger.info(
`Completed search for ${capabilities.server.title} in ${getTimeTakenSincePoint(start)}`,
{
results: results.length,
}
);
return results;
}
private getSearchFunction(
type: string,
searching: Capabilities['searching']
) {
const available = Object.keys(searching);
this.logger.debug(
`Available search functions: ${JSON.stringify(available)}`
);
if (type === 'movie') {
const movieSearch = available.find((s) =>
s.toLowerCase().includes('movie')
);
if (movieSearch && (searching as any)[movieSearch].available)
return {
capabilities: (searching as any)[movieSearch],
function: 'movie',
};
} else {
const tvSearch = available.find((s) => s.toLowerCase().includes('tv'));
if (tvSearch && (searching as any)[tvSearch].available)
return {
capabilities: (searching as any)[tvSearch],
function: 'tvsearch',
};
}
if ((searching as any).search.available)
return { capabilities: (searching as any).search, function: 'search' };
return undefined;
}
}
+322
View File
@@ -0,0 +1,322 @@
import { z } from 'zod';
import { fetch } from 'undici';
import {
Cache,
DistributedLock,
Env,
formatZodError,
getTimeTakenSincePoint,
createLogger,
} from '../../../utils';
import { Parser } from 'xml2js';
import { Logger } from 'winston';
// --- Generic Custom Error ---
export class NabApiError extends Error {
constructor(
public readonly code: number,
public readonly description: string
) {
super(`${description} (Error Code: ${code})`);
this.name = 'NabApiError';
}
}
// --- Zod Schemas ---
const convertString = z
.string()
.optional()
.transform((val) => {
if (val === 'yes') return true;
if (val === 'no') return false;
if (val && !Number.isNaN(Number(val))) return Number(val);
return val;
});
const NabSearchFunctionSchema = z
.array(
z.object({
$: z.object({
available: convertString,
supportedParams: z
.string()
.transform((val) => val.split(','))
.default([]),
}),
})
)
.transform((arr) => arr[0].$);
const NabCapsSearchingSchema = z
.object({
search: NabSearchFunctionSchema,
})
.catchall(NabSearchFunctionSchema);
const CapabilitiesSchema = z
.object({
caps: z.object({
server: z.array(z.object({ $: z.object({ title: z.string() }) })),
limits: z
.array(
z.object({
$: z.object({ default: convertString, max: convertString }),
})
)
.optional(),
searching: z.array(NabCapsSearchingSchema),
}),
})
.transform((obj) => ({
server: obj.caps.server[0].$,
limits: obj.caps.limits?.[0].$,
searching: obj.caps.searching[0],
}));
export type Capabilities = z.infer<typeof CapabilitiesSchema>;
const AttributeSchema = z
.object({ $: z.object({ name: z.string(), value: convertString }) })
.transform((attr) => ({ [attr.$.name]: attr.$.value }));
// Create specific schemas for each namespace
const createTorznabItemSchema = () =>
z
.object({
title: z.array(z.string()).transform((arr) => arr[0]),
link: z.array(z.string()).transform((arr) => arr[0]),
guid: z
.array(z.union([z.string(), z.object({ _: z.string() })]))
.transform((arr) => (typeof arr[0] === 'string' ? arr[0] : arr[0]._)),
pubDate: z.array(z.string()).transform((arr) => arr[0]),
size: z
.array(z.string())
.optional()
.transform((arr) => (arr?.[0] ? Number(arr[0]) : undefined)),
enclosure: z.array(
z
.object({
$: z.object({
url: z.string(),
length: convertString,
type: z.string(),
}),
})
.transform((obj) => obj.$)
),
'torznab:attr': z
.array(AttributeSchema)
.optional()
.transform(
(arr) => arr?.reduce((acc, attr) => ({ ...acc, ...attr }), {}) ?? {}
),
})
.transform((item) => ({
title: item.title,
link: item.link,
guid: item.guid,
pubDate: item.pubDate,
size: item.size,
enclosure: item.enclosure,
torznab: item['torznab:attr'],
}));
const createNewznabItemSchema = () =>
z
.object({
title: z.array(z.string()).transform((arr) => arr[0]),
link: z.array(z.string()).transform((arr) => arr[0]),
guid: z
.array(z.union([z.string(), z.object({ _: z.string() })]))
.transform((arr) => (typeof arr[0] === 'string' ? arr[0] : arr[0]._)),
pubDate: z.array(z.string()).transform((arr) => arr[0]),
size: z
.array(z.string())
.optional()
.transform((arr) => (arr?.[0] ? Number(arr[0]) : undefined)),
enclosure: z.array(
z
.object({
$: z.object({
url: z.string(),
length: convertString,
type: z.string(),
}),
})
.transform((obj) => obj.$)
),
'newznab:attr': z
.array(AttributeSchema)
.optional()
.transform(
(arr) => arr?.reduce((acc, attr) => ({ ...acc, ...attr }), {}) ?? {}
),
})
.transform((item) => ({
title: item.title,
link: item.link,
guid: item.guid,
pubDate: item.pubDate,
size: item.size,
enclosure: item.enclosure,
newznab: item['newznab:attr'],
}));
// Type definitions for search result items
export type TorznabSearchResultItem = z.infer<
ReturnType<typeof createTorznabItemSchema>
>;
export type NewznabSearchResultItem = z.infer<
ReturnType<typeof createNewznabItemSchema>
>;
// Union type for all possible search result items
export type SearchResultItem<T extends 'torznab' | 'newznab'> =
T extends 'torznab' ? TorznabSearchResultItem : NewznabSearchResultItem;
// --- API Client Class ---
export class BaseNabApi<N extends 'torznab' | 'newznab'> {
private readonly xmlParser: Parser;
private readonly capabilitiesCache: Cache<string, Capabilities>;
private readonly searchCache: Cache<string, SearchResultItem<N>[]>;
private readonly SearchResultSchema: z.ZodType<any[]>;
private readonly logger: Logger;
constructor(
public readonly namespace: N,
logger: Logger,
private readonly baseUrl: string,
private readonly apiKey?: string,
private readonly apiPath: string = '/api'
) {
this.logger = logger;
this.baseUrl = this.removeTrailingSlash(baseUrl);
this.apiPath = this.removeTrailingSlash(apiPath);
this.xmlParser = new Parser();
this.capabilitiesCache = Cache.getInstance(`${namespace}:api:caps`);
this.searchCache = Cache.getInstance(`${namespace}:api:search`);
// Create the appropriate schema based on namespace
if (namespace === 'torznab') {
this.SearchResultSchema = z
.object({
rss: z.object({
channel: z.array(
z.object({
item: z.array(createTorznabItemSchema()).optional().default([]),
})
),
}),
})
.transform((data) => data.rss.channel[0].item);
} else {
this.SearchResultSchema = z
.object({
rss: z.object({
channel: z.array(
z.object({
item: z.array(createNewznabItemSchema()).optional().default([]),
})
),
}),
})
.transform((data) => data.rss.channel[0].item);
}
}
public async getCapabilities(): Promise<Capabilities> {
const cacheKey = `${this.baseUrl}${this.apiPath}?t=caps`;
return this.capabilitiesCache.wrap(
() => this.request('caps', CapabilitiesSchema),
cacheKey,
Env.BUILTIN_TORZNAB_CAPABILITIES_CACHE_TTL
);
}
public async search(
searchFunction: string = 'search',
params: Record<string, string | number | boolean> = {}
): Promise<SearchResultItem<N>[]> {
const cacheKey = `${this.baseUrl}${this.apiPath}?t=${searchFunction}&${JSON.stringify(params)}&apikey=${this.apiKey}`;
return this.searchCache.wrap(
() => this.request(searchFunction, this.SearchResultSchema, params),
cacheKey,
Env.BUILTIN_TORZNAB_SEARCH_CACHE_TTL
);
}
private removeTrailingSlash = (path: string) =>
path.endsWith('/') ? path.slice(0, -1) : path;
private getHeaders = () => ({
'Content-Type': 'application/xml',
'User-Agent': Env.DEFAULT_USER_AGENT,
});
private async request<T>(
func: string,
schema: z.ZodSchema<T>,
params: Record<string, string | number | boolean> = {}
): Promise<T> {
const lockKey = `${this.baseUrl}${this.apiPath}?t=${func}&${JSON.stringify(params)}&apikey=${this.apiKey}`;
const { result } = await DistributedLock.getInstance().withLock(
lockKey,
() => this._request(func, schema, params),
{ timeout: 30000, ttl: 32000 }
);
return result;
}
private async _request<T>(
func: string,
schema: z.ZodSchema<T>,
params: Record<string, string | number | boolean> = {}
): Promise<T> {
const start = Date.now();
const url = new URL(`${this.baseUrl}${this.apiPath}`);
const searchParams = new URLSearchParams({
t: func,
...Object.fromEntries(
Object.entries(params).map(([k, v]) => [k, String(v)])
),
});
if (this.apiKey) searchParams.set('apikey', this.apiKey);
url.search = searchParams.toString();
const urlString = url.toString();
this.logger.info(`Making ${this.namespace} request to: ${urlString}`);
try {
const response = await fetch(urlString, {
method: 'GET',
headers: this.getHeaders(),
});
const data = await response.text();
const result = await this.xmlParser.parseStringPromise(data);
this.xmlParser.reset();
if (result.error) {
const code = parseInt(result.error.$.code, 10);
const description = result.error.$.description;
throw new NabApiError(code, description);
}
if (!response.ok) {
throw new Error(
`${response.status} - ${response.statusText}${data ? `: ${data}` : ''}`
);
}
const parsedResult = schema.parse(result);
this.logger.debug(
`Completed ${this.namespace} request for ${urlString} in ${getTimeTakenSincePoint(start)}`
);
return parsedResult;
} catch (error) {
if (error instanceof z.ZodError) {
const message = `Response validation failed: ${formatZodError(error)}`;
this.logger.error(`${this.namespace} ${message}`);
throw new Error(message);
}
this.logger.error(`${this.namespace} request error: ${error}`);
throw error;
}
}
}
+51 -25
View File
@@ -1,5 +1,5 @@
import { Manifest, Meta, MetaPreview, Stream, Subtitle } from '../../db';
import { Env, ExtrasParser, createLogger } from '../../utils';
import { AnimeDatabase, Env, ExtrasParser, createLogger } from '../../utils';
import {
GDriveAPI,
GoogleOAuth,
@@ -9,10 +9,10 @@ import {
import { GDriveFile, UserData } from './schemas';
import { IMDBMetadata } from '../../metadata/imdb';
import { TMDBMetadata } from '../../metadata/tmdb';
import { KitsuMetadata } from '../../metadata/kitsu';
import { formatBytes, formatDuration } from '../../formatters';
import { IdParser, ParsedId } from '../utils/id-parser';
import { TorboxSearchApiIdType } from '../torbox-search/search-api';
import { IdParser, ParsedId } from '../../utils/id-parser';
import { IdType } from '../../utils/id-parser';
import { getTraktAliases } from '../../metadata/trakt';
const logger = createLogger('gdrive');
@@ -21,15 +21,13 @@ export class GDriveAddon {
private api: GDriveAPI;
private oauth: GoogleOAuth;
private manifest: Manifest;
private static supportedIdTypes: TorboxSearchApiIdType[] = [
'imdb_id',
'tmdb',
'thetvdb_id',
'kitsu_id',
private static readonly supportedIdTypes: IdType[] = [
'imdbId',
'themoviedbId',
'thetvdbId',
'kitsuId',
'malId',
];
private static readonly idParser: IdParser = new IdParser(
GDriveAddon.supportedIdTypes
);
constructor(userData: UserData) {
this.userData = UserData.parse(userData);
@@ -88,7 +86,7 @@ export class GDriveAddon {
{
name: 'stream',
types: ['movie', 'series', 'anime'],
idPrefixes: GDriveAddon.idParser.supportedPrefixes,
idPrefixes: IdParser.getPrefixes(GDriveAddon.supportedIdTypes),
},
{
name: 'catalog',
@@ -116,12 +114,12 @@ export class GDriveAddon {
}
public async getStreams(type: string, id: string): Promise<Stream[]> {
const parsedId = GDriveAddon.idParser.parse(id);
if (!parsedId) {
const parsedId = IdParser.parse(id, type);
if (!parsedId || !GDriveAddon.supportedIdTypes.includes(parsedId.type)) {
throw new Error(`Requested ID ${id} is not a valid or supported ID`);
}
logger.debug(`Parsed ID: ${id}`, parsedId);
const { id: titleId, season, episode } = parsedId;
const { value: idValue, type: idType, season, episode } = parsedId;
let searchQuery: string;
try {
const { titles, year } = await this.getMetadata(parsedId, type);
@@ -129,7 +127,7 @@ export class GDriveAddon {
logger.debug(`Search query: ${searchQuery}`);
} catch (error) {
logger.error(
`Failed to get metadata for ${titleId}: ${error instanceof Error ? error.message : error}`
`Failed to get metadata for ${idType}:${idValue}: ${error instanceof Error ? error.message : error}`
);
throw new Error('Failed to get metadata');
}
@@ -179,17 +177,38 @@ export class GDriveAddon {
let year: number;
switch (true) {
case parsedId.type === 'kitsu_id': {
const kitsuMetadata = new KitsuMetadata();
const metadata = await kitsuMetadata.getMetadata(parsedId, type);
titles = metadata.titles ?? [metadata.title];
year = metadata.year;
case parsedId.type === 'malId':
case parsedId.type === 'kitsuId': {
const animeEntry = AnimeDatabase.getInstance().getEntryById(
parsedId.type,
parsedId.value
);
if (!animeEntry || !animeEntry.animeSeason?.year) {
throw new Error('Anime entry not found');
}
titles = [];
if (animeEntry.synonyms) {
titles.push(...animeEntry.synonyms);
}
if (animeEntry.title) {
titles.push(animeEntry.title);
}
titles = [...new Set(titles)];
year = animeEntry.animeSeason?.year;
break;
}
case this.userData.metadataSource === 'imdb': {
const imdbMetadata = new IMDBMetadata();
const metadata = await imdbMetadata.getTitleAndYear(parsedId.id, type);
const metadata = await imdbMetadata.getTitleAndYear(
parsedId.value.toString(),
type
);
titles = metadata.titles ?? [metadata.title];
const traktAliases = await getTraktAliases(parsedId);
if (traktAliases) {
titles.push(...traktAliases);
}
titles = [...new Set(titles)];
year = metadata.year;
break;
}
@@ -201,11 +220,18 @@ export class GDriveAddon {
accessToken: this.userData.tmdbReadAccessToken,
});
const metadata = await tmdbMetadata.getMetadata(
parsedId.id,
parsedId.value.toString(),
type as any
);
titles = metadata.titles;
titles = metadata.titles ?? [metadata.title];
year = Number(metadata.year);
if (parsedId.type === 'imdbId') {
const traktAliases = await getTraktAliases(parsedId);
if (traktAliases) {
titles.push(...traktAliases);
}
titles = [...new Set(titles)];
}
break;
}
default:
+3
View File
@@ -1,2 +1,5 @@
export * from './gdrive';
export * from './torbox-search';
export * from './torznab';
export * from './newznab';
export * from './prowlarr';
@@ -0,0 +1,91 @@
import { z } from 'zod';
import { ParsedId } from '../../utils/id-parser';
import { constants, createLogger } from '../../utils';
import { Torrent, NZB } from '../../debrid';
import { SearchMetadata } from '../base/debrid';
import { createHash } from 'crypto';
import { BaseNabApi, SearchResultItem } from '../base/nab/api';
import {
BaseNabAddon,
NabAddonConfigSchema,
NabAddonConfig,
} from '../base/nab/addon';
const logger = createLogger('newznab');
class NewznabApi extends BaseNabApi<'newznab'> {
constructor(baseUrl: string, apiKey?: string, apiPath?: string) {
super('newznab', logger, baseUrl, apiKey, apiPath);
}
}
// Addon class
export class NewznabAddon extends BaseNabAddon<NabAddonConfig, NewznabApi> {
readonly name = 'Newznab';
readonly version = '1.0.0';
readonly id = 'newznab';
readonly logger = logger;
readonly api: NewznabApi;
constructor(userData: NabAddonConfig, clientIp?: string) {
super(userData, NabAddonConfigSchema, clientIp);
if (
!userData.services.find((s) => s.id === constants.TORBOX_SERVICE) ||
userData.services.length > 1
) {
throw new Error('The Newznab addon only supports TorBox');
}
this.api = new NewznabApi(
this.userData.url,
this.userData.apiKey,
this.userData.apiPath
);
}
protected async _searchNzbs(
parsedId: ParsedId,
metadata: SearchMetadata
): Promise<NZB[]> {
const results = await this.performSearch(parsedId, metadata);
const seenNzbs = new Set<string>();
const nzbs: NZB[] = [];
for (const result of results) {
const nzbUrl = this.getNzbUrl(result);
if (!nzbUrl) continue;
if (seenNzbs.has(nzbUrl)) continue;
seenNzbs.add(nzbUrl);
const md5 =
result.newznab?.infohash?.toString() ||
createHash('md5').update(nzbUrl).digest('hex');
const age = Math.ceil(
Math.abs(new Date().getTime() - new Date(result.pubDate).getTime()) /
(1000 * 60 * 60 * 24)
);
nzbs.push({
hash: md5,
nzb: nzbUrl,
age: `${age}d`,
title: result.title,
size:
result.size ??
(result.newznab?.size ? Number(result.newznab.size) : 0),
type: 'usenet',
});
}
return nzbs;
}
protected async _searchTorrents(
parsedId: ParsedId,
metadata: SearchMetadata
): Promise<Torrent[]> {
return [];
}
private getNzbUrl(result: any): string | undefined {
return result.enclosure.find((e: any) => e.type === 'application/x-nzb')
?.url;
}
}
@@ -0,0 +1 @@
export * from './addon';
@@ -0,0 +1,151 @@
import { BaseDebridAddon, BaseDebridConfigSchema } from '../base/debrid';
import { z } from 'zod';
import { createLogger, Env, getTimeTakenSincePoint } from '../../utils';
import ProwlarrApi, {
ProwlarrApiIndexer,
ProwlarrApiSearchItem,
ProwlarrApiError,
} from './api';
import { ParsedId } from '../../utils/id-parser';
import { SearchMetadata } from '../base/debrid';
import { Torrent, NZB, UnprocessedTorrent } from '../../debrid';
import {
extractInfoHashFromMagnet,
extractTrackersFromMagnet,
} from '../utils/debrid';
export const ProwlarrAddonConfigSchema = BaseDebridConfigSchema.extend({
url: z.string(),
apiKey: z.string(),
indexers: z.array(z.string()),
});
export type ProwlarrAddonConfig = z.infer<typeof ProwlarrAddonConfigSchema>;
const logger = createLogger('prowlarr');
export class ProwlarrAddon extends BaseDebridAddon<ProwlarrAddonConfig> {
readonly id = 'prowlarr';
readonly name = 'Prowlarr';
readonly version = '1.0.0';
readonly logger = logger;
readonly api: ProwlarrApi;
private readonly indexers: string[] = [];
constructor(config: ProwlarrAddonConfig, clientIp?: string) {
super(config, ProwlarrAddonConfigSchema, clientIp);
this.indexers = config.indexers.map((x) => x.toLowerCase());
this.api = new ProwlarrApi({
baseUrl: config.url,
apiKey: config.apiKey,
timeout: Env.BUILTIN_PROWLARR_SEARCH_TIMEOUT,
});
}
protected async _searchTorrents(
parsedId: ParsedId,
metadata: SearchMetadata
): Promise<UnprocessedTorrent[]> {
let availableIndexers: ProwlarrApiIndexer[] = [];
try {
const { data } = await this.api.indexers();
availableIndexers = data;
} catch (error) {
if (error instanceof ProwlarrApiError) {
throw new Error(
`Failed to get Prowlarr indexers: ${error.message}: ${error.status} - ${error.statusText}`
);
}
throw new Error(`Failed to get Prowlarr indexers: ${error}`);
}
const chosenIndexerIds = availableIndexers
.filter(
(indexer) =>
indexer.enable &&
(!this.indexers.length ||
this.indexers.includes(indexer.name.toLowerCase()) ||
this.indexers.includes(indexer.definitionName.toLowerCase()) ||
this.indexers.includes(indexer.sortName.toLowerCase()))
)
.map((indexer) => indexer.id);
let queries: string[] = [];
if (metadata.primaryTitle) {
queries.push(metadata.primaryTitle);
if (parsedId.season && parsedId.episode) {
queries = [
`${metadata.primaryTitle} S${parsedId.season.toString().padStart(2, '0')}E${parsedId.episode.toString().padStart(2, '0')}`,
`${metadata.primaryTitle} S${parsedId.season.toString().padStart(2, '0')}`,
];
if (metadata.absoluteEpisode)
queries.push(
`${metadata.primaryTitle} ${metadata.absoluteEpisode.toString().padStart(2, '0')}`
);
}
if (metadata.year && parsedId.mediaType === 'movie') {
queries = queries.map((q) => `${q} ${metadata.year}`);
}
}
if (queries.length === 0) {
return [];
}
const searchPromises = queries.map(async (q) => {
const start = Date.now();
const { data } = await this.api.search({
query: q,
indexerIds: chosenIndexerIds,
type: 'search',
});
this.logger.info(
`Prowlarr search for ${q} took ${getTimeTakenSincePoint(start)}`,
{
results: data.length,
}
);
return data;
});
const allResults = await Promise.all(searchPromises);
const results = allResults.flat();
const seenTorrents = new Set<string>();
const torrents: UnprocessedTorrent[] = [];
for (const result of results) {
const magnetUrl = result.guid.includes('magnet:')
? result.guid
: result.magnetUrl;
const downloadUrl = result.magnetUrl?.startsWith('http')
? result.magnetUrl
: result.downloadUrl;
const infoHash =
result.infoHash ||
(magnetUrl ? extractInfoHashFromMagnet(magnetUrl) : undefined);
if (!infoHash) continue;
if (seenTorrents.has(infoHash)) continue;
seenTorrents.add(infoHash);
torrents.push({
hash: infoHash,
downloadUrl: downloadUrl,
sources: magnetUrl ? extractTrackersFromMagnet(magnetUrl) : [],
seeders: result.seeders,
title: result.title,
size: result.size,
indexer: result.indexer,
type: 'torrent',
});
}
return torrents;
}
protected async _searchNzbs(
parsedId: ParsedId,
metadata: SearchMetadata
): Promise<NZB[]> {
return [];
}
}
+248
View File
@@ -0,0 +1,248 @@
import { fetch } from 'undici';
import {
Cache,
DistributedLock,
Env,
formatZodError,
makeRequest,
} from '../../utils';
import z from 'zod';
interface ResponseMeta {
headers: Record<string, string>;
status: number;
statusText: string;
}
interface ProwlarrApiResponse<T> {
data: T;
meta: ResponseMeta;
}
interface ProwlarrConfig {
baseUrl: string;
apiKey: string;
timeout: number;
}
export class ProwlarrApiError extends Error {
constructor(
message: string,
public readonly status: number,
public readonly statusText: string
) {
super(message);
}
}
const ProwlarrErrorSchema = z.object({
message: z.string(),
description: z.string(),
});
// minimise schema to only include the fields we need
const ProwlarrApiIndexerSchema = z.object({
id: z.number(),
name: z.string(),
sortName: z.string(),
definitionName: z.string(),
enable: z.boolean(),
});
export type ProwlarrApiIndexer = z.infer<typeof ProwlarrApiIndexerSchema>;
const ProwlarrApiIndexersListSchema = z.array(ProwlarrApiIndexerSchema);
const ProwlarrApiSearchItemSchema = z.object({
guid: z.string(), // can sometimes be the raw magnet url
age: z.number(), // in days
size: z.number(),
indexerId: z.number(),
indexer: z.string(),
title: z.string(),
downloadUrl: z.url().optional(),
indexerFlags: z.array(
z.enum(['freeleech', 'public', 'private', 'semi-private'])
),
magnetUrl: z.url().optional(),
infoHash: z
.string()
.optional()
.transform((val) => val?.toLowerCase()),
seeders: z.number().optional(),
});
const ProwlarrApiSearchSchema = z.array(ProwlarrApiSearchItemSchema);
export type ProwlarrApiSearchItem = z.infer<typeof ProwlarrApiSearchItemSchema>;
class ProwlarrApi {
private readonly baseUrl: string;
private readonly apiKey: string;
private readonly baseApiPath = '/api/v1';
private readonly searchCache = Cache.getInstance<
string,
ProwlarrApiSearchItem[]
>('prowlarr-api:search');
private readonly indexersCache = Cache.getInstance<
string,
ProwlarrApiIndexer[]
>('prowlarr-api:indexers');
#headers: Record<string, string>;
#timeout: number;
constructor(config: ProwlarrConfig) {
this.baseUrl = config.baseUrl;
this.apiKey = config.apiKey;
this.#headers = {
'Content-Type': 'application/json',
'X-Api-Key': this.apiKey,
'User-Agent': Env.DEFAULT_USER_AGENT,
};
this.#timeout = config.timeout;
}
async indexers(): Promise<ProwlarrApiResponse<ProwlarrApiIndexer[]>> {
return this.indexersCache.wrap(
() =>
this.request<ProwlarrApiIndexer[]>(
'indexer',
{},
ProwlarrApiIndexersListSchema,
1000
),
`${this.baseUrl}:indexer`,
Env.BUILTIN_PROWLARR_INDEXERS_CACHE_TTL
);
}
async search({
query,
indexerIds,
type,
limit,
offset,
}: {
query: string;
indexerIds: number[];
type: 'search';
limit?: number;
offset?: number;
}): Promise<ProwlarrApiResponse<ProwlarrApiSearchItem[]>> {
return this.searchCache.wrap(
() =>
this.request<ProwlarrApiSearchItem[]>(
'search',
{
query,
type,
indexerIds,
...(limit !== undefined && { limit }),
...(offset !== undefined && { offset }),
},
ProwlarrApiSearchSchema
),
`${this.baseUrl}:search:${query}`,
Env.BUILTIN_PROWLARR_SEARCH_CACHE_TTL
);
}
private getPath(endpoint: string) {
return `${this.baseUrl}${this.baseApiPath}/${endpoint}`;
}
private async request<T>(
endpoint: string,
params: Record<
string,
string | number | boolean | (string | number)[]
> = {},
schema: z.ZodType<T>,
timeout?: number
): Promise<ProwlarrApiResponse<T>> {
const { result } = await DistributedLock.getInstance().withLock(
`${this.getPath(endpoint)}:${JSON.stringify(params)}`,
() => this._request(endpoint, params, schema, timeout),
{
timeout: timeout ?? this.#timeout,
ttl: (timeout ?? this.#timeout) * 2,
}
);
return result;
}
private async _request<T>(
endpoint: string,
params: Record<
string,
string | number | boolean | (string | number)[]
> = {},
schema: z.ZodType<T>,
timeout?: number
): Promise<ProwlarrApiResponse<T>> {
const url = new URL(this.getPath(endpoint));
const headers = this.#headers;
// Create URLSearchParams and handle array parameters
const searchParams = new URLSearchParams();
for (const [key, value] of Object.entries(params)) {
if (Array.isArray(value)) {
// Handle array parameters by adding multiple entries with the same key
value.forEach((item) => searchParams.append(key, String(item)));
} else {
// Handle non-array parameters
searchParams.append(key, String(value));
}
}
url.search = searchParams.toString();
const response = await makeRequest(url.toString(), {
method: 'GET',
headers,
timeout: timeout ?? this.#timeout,
});
const meta: ResponseMeta = {
headers: Object.fromEntries(response.headers.entries()),
status: response.status,
statusText: response.statusText,
};
if (!response.ok) {
try {
throw new ProwlarrApiError(
ProwlarrErrorSchema.parse(await response.json()).message,
response.status,
response.statusText
);
} catch (error) {
throw new ProwlarrApiError(
`Generic HTTP error: ${response.status} - ${response.statusText}`,
response.status,
response.statusText
);
}
}
const { success, data, error } = schema.safeParse(await response.json());
if (!success) {
throw new ProwlarrApiError(
`Prowlarr API error: ${formatZodError(error)}`,
response.status,
response.statusText
);
}
return {
data,
meta,
};
}
}
export default ProwlarrApi;
@@ -0,0 +1 @@
export * from './addon';
@@ -1,6 +1,7 @@
import { z } from 'zod';
import { Manifest, Stream } from '../../db';
import {
AnimeDatabase,
createLogger,
formatZodError,
getTimeTakenSincePoint,
@@ -8,11 +9,10 @@ import {
import { TorBoxSearchAddonUserDataSchema } from './schemas';
import { TorboxApi } from '@torbox/torbox-api';
import TorboxSearchApi from './search-api';
import { IdParser } from '../utils/id-parser';
import { IdParser } from '../../utils/id-parser';
import { TorrentSourceHandler, UsenetSourceHandler } from './source-handlers';
import { TorBoxSearchAddonError } from './errors';
import { supportedIdTypes } from './search-api';
import { KitsuMetadata } from '../../metadata/kitsu';
const logger = createLogger('torbox-search');
@@ -20,7 +20,6 @@ export class TorBoxSearchAddon {
private readonly userData: z.infer<typeof TorBoxSearchAddonUserDataSchema>;
private readonly searchApi: TorboxSearchApi;
private readonly torboxApi: TorboxApi;
private static readonly idParser: IdParser = new IdParser(supportedIdTypes);
private readonly sourceHandlers: (
| TorrentSourceHandler
| UsenetSourceHandler
@@ -71,7 +70,7 @@ export class TorBoxSearchAddon {
{
name: 'stream',
types: ['movie', 'series', 'anime'],
idPrefixes: TorBoxSearchAddon.idParser.supportedPrefixes,
idPrefixes: IdParser.getPrefixes(supportedIdTypes),
},
],
catalogs: [],
@@ -98,7 +97,9 @@ export class TorBoxSearchAddon {
new UsenetSourceHandler(
this.searchApi,
this.torboxApi,
this.userData.searchUserEngines
this.userData.searchUserEngines,
this.userData.services,
this.clientIp
)
);
}
@@ -110,39 +111,29 @@ export class TorBoxSearchAddon {
}
public async getStreams(type: string, id: string): Promise<Stream[]> {
const parsedId = TorBoxSearchAddon.idParser.parse(id);
if (!parsedId) {
const parsedId = IdParser.parse(id, type);
if (!parsedId || !supportedIdTypes.includes(parsedId.type)) {
throw new TorBoxSearchAddonError(`Unsupported ID: ${id}`, 400);
}
const metadataStart = Date.now();
if (
['mal_id', 'kitsu_id', 'anilist_id', 'anidb_id'].includes(parsedId.type)
) {
try {
const kitsuMetadata = new KitsuMetadata();
const metadata = await kitsuMetadata.getMetadata(parsedId, type);
parsedId.season = metadata.seasons?.[0]?.season_number
? metadata.seasons[0].season_number.toString()
: undefined;
logger.debug(
`Fetched season metadata for ${id} in ${getTimeTakenSincePoint(metadataStart)}:`,
{
season: parsedId.season,
}
);
} catch (error) {
logger.error(`Error fetching anime metadata for ${id}:`, error);
}
const animeEntry = AnimeDatabase.getInstance().getEntryById(
parsedId.type,
parsedId.value
);
if (animeEntry && !parsedId.season) {
parsedId.season =
animeEntry.imdb?.fromImdbSeason?.toString() ??
animeEntry.trakt?.season?.toString();
logger.debug(`Updated season for ${id} to ${parsedId.season}`);
}
logger.info(`Getting streams for ${id}`, {
logger.info(`Handling stream request`, {
type,
id,
idType: parsedId.type,
idValue: parsedId.value,
season: parsedId.season,
episode: parsedId.episode,
titleId: parsedId.id,
sources: this.userData.sources,
});
@@ -1,200 +0,0 @@
import {
Cache,
constants,
createLogger,
Env,
getSimpleTextHash,
getTimeTakenSincePoint,
} from '../../utils';
import { DebridInterface } from '../../debrid/interface';
import { FileParser } from '../../parser';
import { ParsedId } from '../utils/id-parser';
import { Torrent } from './torrent';
import { ServiceId } from '../../utils';
import { z } from 'zod';
import { TorBoxSearchAddonUserDataSchema } from './schemas';
import { findMatchingFileInTorrent, isVideoFile } from '../../debrid/utils';
import { StremThruError } from 'stremthru';
import { TitleMetadata } from './source-handlers';
import { calculateAbsoluteEpisode } from '../utils/general';
const logger = createLogger('torbox-search');
export interface DebridFile {
hash: string;
filename: string;
size: number;
index?: number;
service: {
id: ServiceId;
cached: boolean;
owned: boolean;
};
}
export class DebridService {
private readonly debridCache = Cache.getInstance<string, DebridFile[]>(
'torbox-search-debrid'
);
private readonly debridInterface: DebridInterface;
private readonly serviceConfig: z.infer<
typeof TorBoxSearchAddonUserDataSchema
>['services'][0];
constructor(
serviceConfig: z.infer<
typeof TorBoxSearchAddonUserDataSchema
>['services'][0],
private readonly clientIp?: string
) {
this.serviceConfig = serviceConfig;
this.debridInterface = new DebridInterface(
{
storeName: serviceConfig.id,
storeCredential: serviceConfig.credential,
},
this.clientIp
);
}
public async getAvailableFiles(
torrents: Torrent[],
parsedId: ParsedId,
titleMetadata?: TitleMetadata
): Promise<DebridFile[] | { error: { title: string; description: string } }> {
const { id, season, episode } = parsedId;
const absoluteEpisode =
season && episode && titleMetadata?.seasons
? calculateAbsoluteEpisode(season, episode, titleMetadata.seasons)
: undefined;
const cachedResults: DebridFile[] = [];
const torrentsToCheck: Torrent[] = [];
for (const torrent of torrents) {
const cacheKey = getSimpleTextHash(
`${this.serviceConfig.id}:${torrent.hash}`
);
const cached = await this.debridCache.get(cacheKey);
if (cached && cached.length > 0) {
cachedResults.push(...cached);
} else {
torrentsToCheck.push(torrent);
}
}
let newResults: DebridFile[] = [];
const start = Date.now();
if (torrentsToCheck.length > 0) {
try {
const instantAvailability = await this.debridInterface.checkMagnets(
torrentsToCheck.map((t) => t.hash),
id
);
for (const torrent of torrentsToCheck) {
const item = instantAvailability.data.items.find(
(avail) => avail.hash === torrent.hash
);
let file;
if (!item) {
logger.debug(
`[${this.serviceConfig.id}] Hash ${torrent.hash} (${torrent.title}) not found in instant availability response`
);
file = { name: torrent.title, size: torrent.size, index: -1 };
} else {
file =
item.files && item.files.length > 0
? findMatchingFileInTorrent(
item.files.map((file) => ({
...file,
parsed: FileParser.parse(file.name),
isVideo: isVideoFile(file.name),
})),
torrent.fileIdx,
undefined,
titleMetadata?.titles,
season,
episode,
absoluteEpisode,
false
)
: { name: torrent.title, size: torrent.size, index: -1 }; // Fallback for torrents with no file list
}
if (file) {
const result: DebridFile = {
hash: torrent.hash,
filename: file.name,
size: file.size,
index: file.index !== -1 ? file.index : undefined,
service: {
id: this.serviceConfig.id,
cached: item?.status === 'cached',
owned:
this.serviceConfig.id === 'torbox'
? (torrent.owned ?? false)
: false,
},
};
newResults.push(result);
const cacheKey = getSimpleTextHash(
`${this.serviceConfig.id}:${torrent.hash}`
);
this.debridCache.set(
cacheKey,
[result],
Env.BUILTIN_TORBOX_SEARCH_INSTANT_AVAILABILITY_CACHE_TTL
);
}
}
} catch (error) {
if (error instanceof StremThruError) {
logger.error(
`Got StremThru error during debrid check: ${error.code}: ${error.message}`
);
const serviceName =
constants.SERVICE_DETAILS[this.serviceConfig.id].shortName;
switch (error.code) {
case 'FORBIDDEN':
case 'UNAUTHORIZED':
return {
error: {
title: serviceName,
description: 'Invalid/expired credentials',
},
};
default:
return {
error: {
title: serviceName,
description: 'Internal Server Error',
},
};
}
}
logger.error(`Unexpected error during debrid check:`, error);
return [];
}
}
// count number of cached results
const cachedCount = [...cachedResults, ...newResults].filter(
(result) => result.service.cached
).length;
logger.debug(
`[${this.serviceConfig.id}] Checked ${torrents.length} magnets in ${getTimeTakenSincePoint(start)}. ${cachedCount} / ${torrents.length} cached`
);
return [...cachedResults, ...newResults];
}
}
@@ -1,6 +1,5 @@
import { z } from 'zod';
import { StremThruPreset } from '../../presets/stremthru';
import { ServiceId } from '../../utils';
import { BuiltinDebridServices } from '../../debrid/utils';
const TorBoxApiErrorResponseSchema = z.object({
success: z.literal(false),
@@ -32,7 +31,7 @@ const TorBoxSearchApiMetadataSchema = z.object({
tmdb_id: z.number().nullable(),
});
const TorBoxSearchApiTorrentSchema = z.object({
const TorBoxSearchApiResultSchema = z.object({
hash: z.string(),
raw_title: z.string(),
title: z.string(),
@@ -67,11 +66,9 @@ const TorBoxSearchApiTorrentSchema = z.object({
});
export const TorBoxSearchApiDataSchema = z.object({
metadata: z.union([TorBoxSearchApiMetadataSchema, z.null()]).optional(),
torrents: z
.union([z.array(TorBoxSearchApiTorrentSchema), z.null()])
.optional(),
nzbs: z.union([z.array(TorBoxSearchApiTorrentSchema), z.null()]).optional(),
metadata: TorBoxSearchApiMetadataSchema.nullable().optional(),
torrents: z.array(TorBoxSearchApiResultSchema).nullable().optional(),
nzbs: z.array(TorBoxSearchApiResultSchema).nullable().optional(),
});
export const TorBoxApiUsenetDownloadSchema = z.object({
@@ -90,14 +87,8 @@ export const TorBoxSearchAddonUserDataSchema = z.object({
sources: z
.array(z.enum(['torrent', 'usenet']))
.min(1, 'At least one source must be configured'),
services: z
.array(
z.object({
id: z.enum(
StremThruPreset.supportedServices as [ServiceId, ...ServiceId[]]
),
credential: z.string(),
})
)
.min(1, 'At least one service must be configured'),
services: BuiltinDebridServices.min(
1,
'At least one service must be configured'
),
});
@@ -2,12 +2,13 @@ import { fetch, RequestInit, Response } from 'undici';
import { z } from 'zod';
import { TorBoxApiResponseSchema, TorBoxSearchApiDataSchema } from './schemas';
import {
Cache,
createLogger,
DistributedLock,
Env,
formatZodError,
maskSensitiveInfo,
} from '../../utils';
import { IdType } from '../../utils/id-parser';
type TorboxSuccessResponse<T> = {
success: true;
@@ -25,20 +26,20 @@ type TorboxErrorResponse = {
type TorboxResponse<T> = TorboxSuccessResponse<T> | TorboxErrorResponse;
export const supportedIdTypes: TorboxSearchApiIdType[] = [
'anime-planet_id',
'anidb_id',
'anilist_id',
'anisearch_id',
'imdb_id',
'kitsu_id',
'livechart_id',
'mal_id',
'notify.moe_id',
'thetvdb_id',
'themoviedb_id',
'tmdb',
export const supportedIdTypes: IdType[] = [
'animePlanetId',
'anidbId',
'anilistId',
'anisearchId',
'imdbId',
'kitsuId',
'livechartId',
'malId',
'notifyMoeId',
'thetvdbId',
'themoviedbId',
];
export type TorboxSearchApiIdType =
| 'anime-planet_id'
| 'anidb_id'
@@ -50,8 +51,7 @@ export type TorboxSearchApiIdType =
| 'mal_id'
| 'notify.moe_id'
| 'thetvdb_id'
| 'themoviedb_id'
| 'tmdb';
| 'themoviedb_id';
const logger = createLogger('torbox-search');
@@ -61,14 +61,14 @@ function isErrorResponse<T>(
return !response.success;
}
export class TorboxApiError extends Error {
export class TorboxSearchApiError extends Error {
constructor(
message: string,
public readonly statusCode: number,
public readonly errorCode?: string
) {
super(message);
this.name = 'TorboxApiError';
this.name = 'TorboxSearchApiError';
}
}
@@ -82,23 +82,26 @@ class TorboxSearchApi {
constructor(public readonly apiKey: string) {}
private createRequestLock<T>(
private async createRequestLock<T>(
key: string,
executor: () => Promise<T>
): Promise<T> {
if (TorboxSearchApi.ongoingRequests.has(key)) {
const { result, cached } = await DistributedLock.getInstance().withLock(
`tb-search-api:${key}`,
executor,
{
timeout: TorboxSearchApi.timeout,
ttl: TorboxSearchApi.timeout * 2,
}
);
if (cached) {
logger.debug(
`Found ongoing request for ${key.replace(this.apiKey, maskSensitiveInfo(this.apiKey))}. Waiting for it to complete.`
`Found cached result for ${key.replace(this.apiKey, maskSensitiveInfo(this.apiKey))}`
);
return TorboxSearchApi.ongoingRequests.get(key)!;
}
const requestPromise = executor().finally(() => {
TorboxSearchApi.ongoingRequests.delete(key);
});
TorboxSearchApi.ongoingRequests.set(key, requestPromise);
return requestPromise;
return result;
}
async request<T>(
@@ -137,7 +140,7 @@ class TorboxSearchApi {
});
} catch (error) {
if (error instanceof Error && error.name === 'TimeoutError') {
throw new TorboxApiError('Request timed out', 408, 'TIMEOUT');
throw new TorboxSearchApiError('Request timed out', 408, 'TIMEOUT');
}
throw error;
}
@@ -147,7 +150,7 @@ class TorboxSearchApi {
const parsedResponse = TorBoxApiResponseSchema(schema).safeParse(data);
if (!parsedResponse.success) {
throw new TorboxApiError(
throw new TorboxSearchApiError(
`Failed to parse API response: ${formatZodError(parsedResponse.error)}`,
response.status,
'PARSE_ERROR'
@@ -157,7 +160,7 @@ class TorboxSearchApi {
const result = parsedResponse.data as TorboxResponse<T>;
if (isErrorResponse(result)) {
throw new TorboxApiError(
throw new TorboxSearchApiError(
result.detail || result.message || 'Unknown',
response.status,
result.error
@@ -1,43 +1,47 @@
import { number, z } from 'zod';
import { Stream } from '../../db';
import {
AnimeDatabase,
Cache,
Env,
SERVICE_DETAILS,
createLogger,
getTimeTakenSincePoint,
} from '../../utils';
import { DebridService, DebridFile } from './debrid-service';
import { ParsedId } from '../utils/id-parser';
// import { DebridService, DebridFile } from './debrid-service';
import { ParsedId } from '../../utils/id-parser';
import { TorBoxSearchAddonUserDataSchema } from './schemas';
import TorboxSearchApi, {
TorboxApiError,
TorboxSearchApiError,
TorboxSearchApiIdType,
} from './search-api';
import { Torrent, convertDataToTorrents } from './torrent';
import { TMDBMetadata } from '../../metadata/tmdb';
import { calculateAbsoluteEpisode } from '../utils/general';
import { TorboxApi } from '@torbox/torbox-api';
import { processNZBs, processTorrents } from '../utils/debrid';
import {
NZBWithSelectedFile,
TorrentWithSelectedFile,
} from '../../debrid/utils';
import { DebridFile, PlaybackInfo } from '../../debrid';
import { getTraktAliases } from '../../metadata/trakt';
const logger = createLogger('torbox-search');
export interface TitleMetadata {
titles: string[];
seasons?: {
number: string;
episodes: number;
}[];
season?: number;
episode?: number;
absoluteEpisode?: number;
}
abstract class SourceHandler {
protected searchCache = Cache.getInstance<string, Torrent[]>(
'torbox-search-torrents'
'tb-search:torrents'
);
protected metadataCache = Cache.getInstance<string, TitleMetadata>(
'torbox-search-metadata'
);
protected instantAvailabilityCache = Cache.getInstance<string, boolean>(
'tb-search-usenet-instant'
'tb-search:metadata'
);
protected errorStreams: Stream[] = [];
@@ -61,7 +65,7 @@ abstract class SourceHandler {
parsedId: ParsedId,
type: 'torrent' | 'usenet'
): string {
let cacheKey = `${type}:${parsedId.type}:${parsedId.id}:${parsedId.season}:${parsedId.episode}`;
let cacheKey = `${type}:${parsedId.type}:${parsedId.value}:${parsedId.season}:${parsedId.episode}`;
if (this.searchUserEngines) {
cacheKey += `:${this.searchApi.apiKey}`;
}
@@ -70,54 +74,62 @@ abstract class SourceHandler {
protected createStream(
id: ParsedId,
torrent: Torrent,
file: DebridFile,
torrentOrNZB: TorrentWithSelectedFile | NZBWithSelectedFile,
userData: z.infer<typeof TorBoxSearchAddonUserDataSchema>,
season?: string,
episode?: string,
absoluteEpisode?: string
titleMetadata?: TitleMetadata
): Stream & { type: 'torrent' | 'usenet' } {
if (!torrentOrNZB.service) {
throw new Error('Torrent or NZB has no service');
}
const storeAuth = {
storeName: file.service.id,
storeCredential: userData.services.find(
(service) => service.id === file.service.id
id: torrentOrNZB.service.id,
credential: userData.services.find(
(service) => service.id === torrentOrNZB.service!.id
)?.credential,
};
const playbackInfo =
torrent.type === 'torrent'
// const playbackInfo: PlaybackInfo = {
// type: 'usenet',
// hash: torrent.hash,
// magnet: torrent.type === 'torrent' ? torrent.magnet : undefined,
// title: torrent.title,
// nzb: torrent.type === 'usenet' ? torrent.nzb : undefined,
// file: torrent.file,
// metadata: titleMetadata,
// };
const playbackInfo: PlaybackInfo =
torrentOrNZB.type === 'torrent'
? {
parsedId: {
id: id.id,
type: id.type,
season: season || undefined,
episode: episode || undefined,
absoluteEpisode: absoluteEpisode || undefined,
},
type: 'torrent',
hash: torrent.hash,
index: file.index,
title: torrent.title,
hash: torrentOrNZB.hash,
sources: torrentOrNZB.sources,
// magnet: torrentOrNZB.magnet,
title: torrentOrNZB.title,
file: torrentOrNZB.file,
metadata: titleMetadata,
}
: {
type: 'usenet',
nzb: torrent.nzb,
title: torrent.title,
nzb: torrentOrNZB.nzb,
title: torrentOrNZB.title,
hash: torrentOrNZB.hash,
file: torrentOrNZB.file,
metadata: titleMetadata,
};
const svcMeta = SERVICE_DETAILS[file.service.id];
const name = `[${svcMeta.shortName} ${file.service.cached ? '⚡' : '⏳'}${file.service.owned ? ' ☁️' : ''}] TorBox Search`;
const description = `${torrent.title}\n${file.filename}\n${torrent.indexer ? `🔍 ${torrent.indexer}` : ''} ${torrent.seeders ? `👤 ${torrent.seeders}` : ''} ${torrent.age && torrent.age !== '0d' ? `🕒 ${torrent.age}` : ''}`;
const svcMeta = SERVICE_DETAILS[torrentOrNZB.service.id];
const name = `[${svcMeta.shortName} ${torrentOrNZB.service.cached ? '⚡' : '⏳'}${torrentOrNZB.service.owned ? ' ☁️' : ''}] TorBox Search`;
const description = `${torrentOrNZB.title}\n${torrentOrNZB.file.name}\n${torrentOrNZB.indexer ? `🔍 ${torrentOrNZB.indexer}` : ''} ${torrentOrNZB.seeders ? `👤 ${torrentOrNZB.seeders}` : ''} ${torrentOrNZB.age && torrentOrNZB.age !== '0d' ? `🕒 ${torrentOrNZB.age}` : ''}`;
return {
url: `${Env.BASE_URL}/api/v1/debrid/resolve/${encodeURIComponent(Buffer.from(JSON.stringify(storeAuth)).toString('base64'))}/${encodeURIComponent(Buffer.from(JSON.stringify(playbackInfo)).toString('base64'))}/${encodeURIComponent(file.filename || torrent.title)}`,
url: `${Env.BASE_URL}/api/v1/debrid/playback/${encodeURIComponent(Buffer.from(JSON.stringify(storeAuth)).toString('base64'))}/${encodeURIComponent(Buffer.from(JSON.stringify(playbackInfo)).toString('base64'))}/${encodeURIComponent(torrentOrNZB.file.name || torrentOrNZB.title || 'unknown')}`,
name,
description,
type: torrent.type,
infoHash: torrent.hash,
type: torrentOrNZB.type,
infoHash: torrentOrNZB.hash,
behaviorHints: {
videoSize: file.size,
filename: file.filename,
videoSize: torrentOrNZB.file.size,
filename: torrentOrNZB.file.name,
},
};
}
@@ -132,10 +144,88 @@ abstract class SourceHandler {
externalUrl: 'stremio:///',
};
}
protected async processMetadata(
parsedId: ParsedId,
metadata?: {
tmdb_id?: string | number | null;
titles: string[];
globalID?: string;
title?: string;
imdb_id?: string | null;
},
tmdbAccessToken?: string
): Promise<TitleMetadata | undefined> {
if (!metadata) return undefined;
const { tmdb_id, titles } = metadata;
let absoluteEpisode;
const animeEntry = AnimeDatabase.getInstance().getEntryById(
parsedId.type,
parsedId.value
);
const tmdbId = animeEntry?.mappings?.themoviedbId ?? tmdb_id;
const traktAliases = await getTraktAliases(parsedId);
// For anime sources, fetch additional season info from TMDB
if (animeEntry && parsedId.season && parsedId.episode) {
const seasonFetchStart = Date.now();
try {
const tmdbMetadata = await new TMDBMetadata({
accessToken: tmdbAccessToken,
}).getMetadata(`tmdb:${tmdbId}`, 'series');
const seasons = tmdbMetadata?.seasons?.map(
({ season_number, episode_count }) => ({
number: season_number.toString(),
episodes: episode_count,
})
);
if (seasons) {
absoluteEpisode = calculateAbsoluteEpisode(
parsedId.season.toString(),
parsedId.episode.toString(),
seasons
);
}
logger.debug(
`Fetched additional season info for ${parsedId.type}:${parsedId.value} in ${getTimeTakenSincePoint(seasonFetchStart)}`
);
} catch (error) {
logger.error(
`Failed to fetch TMDB metadata for ${parsedId.type}:${parsedId.value} - ${error}`
);
}
}
const titleMetadata: TitleMetadata = {
titles: [...new Set([...(traktAliases ?? []), ...titles])],
season: parsedId.season ? Number(parsedId.season) : undefined,
episode: parsedId.episode ? Number(parsedId.episode) : undefined,
absoluteEpisode: absoluteEpisode ? Number(absoluteEpisode) : undefined,
};
// Store metadata in cache
await this.metadataCache.set(
`metadata:${parsedId.type}:${parsedId.value}`,
titleMetadata,
Env.BUILTIN_TORBOX_SEARCH_METADATA_CACHE_TTL
);
return titleMetadata;
}
}
export class TorrentSourceHandler extends SourceHandler {
private readonly debridServices: DebridService[];
// private readonly debridServices: DebridService[];
private readonly services: z.infer<
typeof TorBoxSearchAddonUserDataSchema
>['services'];
private readonly clientIp?: string;
constructor(
searchApi: TorboxSearchApi,
@@ -144,27 +234,23 @@ export class TorrentSourceHandler extends SourceHandler {
clientIp?: string
) {
super(searchApi, searchUserEngines);
this.debridServices = services.map(
(service) => new DebridService(service, clientIp)
);
this.services = services;
this.clientIp = clientIp;
}
async getStreams(
parsedId: ParsedId,
userData: z.infer<typeof TorBoxSearchAddonUserDataSchema>
): Promise<Stream[]> {
const { type, id, season, episode } = parsedId;
let torrents: Torrent[] = [];
const { type, value, season, episode } = parsedId;
let fetchResult: { torrents: Torrent[]; metadata?: TitleMetadata };
try {
torrents = await this.fetchTorrents(
type,
id,
season,
episode,
fetchResult = await this.fetchTorrents(
parsedId,
userData.tmdbAccessToken
);
} catch (error) {
if (error instanceof TorboxApiError) {
if (error instanceof TorboxSearchApiError) {
switch (error.errorCode) {
case 'BAD_TOKEN':
return [
@@ -174,145 +260,108 @@ export class TorrentSourceHandler extends SourceHandler {
}),
];
default:
logger.error(`Error fetching torrents for ${type}:${id}: ${error}`);
logger.error(
`Error fetching torrents for ${type}:${value}: ${error}`
);
throw error;
}
}
logger.error(
`Unexpected error fetching torrents for ${type}:${id}: ${error}`
`Unexpected error fetching torrents for ${type}:${value}: ${error}`
);
throw error;
}
if (torrents.length === 0) return [];
if (fetchResult.torrents.length === 0) return [];
if (userData.onlyShowUserSearchResults) {
const userSearchResults = torrents.filter(
const userSearchResults = fetchResult.torrents.filter(
(torrent) => torrent.userSearch
);
logger.info(
`Filtered out ${torrents.length - userSearchResults.length} torrents that were not user search results`
`Filtered out ${fetchResult.torrents.length - userSearchResults.length} torrents that were not user search results`
);
if (userSearchResults.length > 0) {
torrents = userSearchResults;
fetchResult.torrents = userSearchResults;
} else {
return [];
}
}
const titleMetadata = await this.metadataCache.get(
`metadata:${type}:${id}`
const { results, errors } = await processTorrents(
fetchResult.torrents.map((torrent) => ({
...torrent,
type: 'torrent',
})),
this.services,
parsedId.fullId,
fetchResult.metadata,
this.clientIp
);
const filesByHash = await this.getAvailableFilesFromDebrid(
torrents,
parsedId,
titleMetadata
results.forEach((result) => {
result.service!.owned =
fetchResult.torrents.find((torrent) => torrent.hash === result.hash)
?.owned ?? false;
});
return results.map((result) =>
this.createStream(parsedId, result, userData, fetchResult.metadata)
);
const absoluteEpisode =
season && episode && titleMetadata?.seasons
? calculateAbsoluteEpisode(season, episode, titleMetadata.seasons)
: undefined;
const streams: Stream[] = [];
for (const torrent of torrents) {
const availableFiles = filesByHash.get(torrent.hash);
if (availableFiles) {
for (const file of availableFiles) {
streams.push(
this.createStream(
parsedId,
torrent,
file,
userData,
season,
episode,
absoluteEpisode
)
);
}
}
}
streams.push(...this.errorStreams);
return streams;
}
private async fetchTorrents(
idType: TorboxSearchApiIdType,
id: string,
season?: string,
episode?: string,
parsedId: ParsedId,
tmdbAccessToken?: string
): Promise<Torrent[]> {
const cacheKey = this.getCacheKey(
{ type: idType, id, season, episode },
'torrent'
);
): Promise<{ torrents: Torrent[]; metadata?: TitleMetadata }> {
const { type, value, season, episode, externalType } = parsedId;
const cacheKey = this.getCacheKey(parsedId, 'torrent');
const cachedTorrents = await this.searchCache.get(cacheKey);
const cachedMetadata = await this.metadataCache.get(
`metadata:${type}:${value}`
);
if (
cachedTorrents &&
(!this.searchUserEngines ||
Env.BUILTIN_TORBOX_SEARCH_CACHE_PER_USER_SEARCH_ENGINE)
) {
logger.info(`Found ${cachedTorrents.length} (cached) torrents for ${id}`);
return cachedTorrents;
logger.info(
`Found ${cachedTorrents.length} (cached) torrents for ${type}:${value}`
);
return { torrents: cachedTorrents, metadata: cachedMetadata };
}
const start = Date.now();
const data = await this.searchApi.getTorrentsById(idType, id, {
search_user_engines: this.searchUserEngines ? 'true' : 'false',
season,
episode,
metadata: 'true',
check_owned: 'true',
});
const data = await this.searchApi.getTorrentsById(
externalType as TorboxSearchApiIdType,
value.toString(),
{
search_user_engines: this.searchUserEngines ? 'true' : 'false',
season,
episode,
metadata: 'true',
check_owned: 'true',
}
);
const torrents = convertDataToTorrents(data.torrents);
logger.info(
`Found ${torrents.length} torrents for ${id} in ${getTimeTakenSincePoint(start)}`
`Found ${torrents.length} torrents for ${type}:${value} in ${getTimeTakenSincePoint(start)}`
);
let titleMetadata: TitleMetadata | undefined;
if (data.metadata) {
const { tmdb_id, titles } = data.metadata;
let seasons;
if (
['kitsu_id', 'mal_id', 'anilist_id', 'anidb_id'].includes(idType) &&
tmdb_id
) {
const seasonFetchStart = Date.now();
try {
const tmdbMetadata = await new TMDBMetadata({
accessToken: tmdbAccessToken,
}).getMetadata(`tmdb:${tmdb_id}`, 'series');
seasons = tmdbMetadata?.seasons?.map(
({ season_number, episode_count }) => ({
number: season_number.toString(),
episodes: episode_count,
})
);
logger.debug(
`Fetched additional season info for ${id} in ${getTimeTakenSincePoint(seasonFetchStart)}`
);
} catch (error) {
logger.error(
`Failed to fetch TMDB metadata for ${idType}:${id} - ${error}`
);
}
}
this.metadataCache.set(
`metadata:${idType}:${id}`,
{ titles: [...new Set(titles)], seasons },
Env.BUILTIN_TORBOX_SEARCH_METADATA_CACHE_TTL
titleMetadata = await this.processMetadata(
parsedId,
data.metadata,
tmdbAccessToken
);
}
if (torrents.length === 0) {
return [];
return { torrents: [], metadata: titleMetadata };
}
if (this.useCache) {
@@ -328,79 +377,68 @@ export class TorrentSourceHandler extends SourceHandler {
);
}
return torrents;
}
private async getAvailableFilesFromDebrid(
torrents: Torrent[],
parsedId: ParsedId,
titleMetadata?: TitleMetadata
): Promise<Map<string, DebridFile[]>> {
const allFiles = new Map<string, DebridFile[]>();
const servicePromises = this.debridServices.map((service) =>
service.getAvailableFiles(torrents, parsedId, titleMetadata)
);
const results = await Promise.allSettled(servicePromises);
for (const result of results) {
if (result.status === 'fulfilled') {
if (result.value instanceof Array) {
for (const file of result.value) {
const existing = allFiles.get(file.hash) || [];
allFiles.set(file.hash, [...existing, file]);
}
} else {
this.errorStreams.push(
this.createErrorStream({
title: result.value.error.title,
description: result.value.error.description,
})
);
}
}
}
return allFiles;
return { torrents, metadata: titleMetadata };
}
}
export class UsenetSourceHandler extends SourceHandler {
private readonly torboxApi: TorboxApi;
private readonly services: z.infer<
typeof TorBoxSearchAddonUserDataSchema
>['services'];
private readonly clientIp?: string;
constructor(
searchApi: TorboxSearchApi,
torboxApi: TorboxApi,
searchUserEngines: boolean
searchUserEngines: boolean,
services: z.infer<typeof TorBoxSearchAddonUserDataSchema>['services'],
clientIp?: string
) {
super(searchApi, searchUserEngines);
this.torboxApi = torboxApi;
this.services = services.filter((service) => service.id === 'torbox');
this.clientIp = clientIp;
}
async getStreams(
parsedId: ParsedId,
userData: z.infer<typeof TorBoxSearchAddonUserDataSchema>
): Promise<Stream[]> {
const { type, id, season, episode } = parsedId;
const { type, value, season, episode, externalType } = parsedId;
const cacheKey = this.getCacheKey(parsedId, 'usenet');
let usingCachedSearch = false;
let titleMetadata: TitleMetadata | undefined;
let torrents = await this.searchCache.get(cacheKey);
if (!torrents) {
const start = Date.now();
try {
const data = await this.searchApi.getUsenetById(type, id, {
season,
episode,
check_cache: 'true',
check_owned: 'true',
search_user_engines: this.searchUserEngines ? 'true' : 'false',
});
const data = await this.searchApi.getUsenetById(
externalType as TorboxSearchApiIdType,
value.toString(),
{
season,
episode,
check_cache: 'true',
check_owned: 'true',
search_user_engines: this.searchUserEngines ? 'true' : 'false',
metadata: 'true',
}
);
torrents = convertDataToTorrents(data.nzbs);
logger.info(
`Found ${torrents.length} NZBs for ${id} in ${getTimeTakenSincePoint(start)}`
`Found ${torrents.length} NZBs for ${parsedId.type}:${parsedId.value} in ${getTimeTakenSincePoint(start)}`
);
if (data.metadata) {
titleMetadata = await this.processMetadata(
parsedId,
data.metadata,
userData.tmdbAccessToken
);
}
if (torrents.length === 0) {
return [];
}
@@ -412,7 +450,7 @@ export class UsenetSourceHandler extends SourceHandler {
);
}
} catch (error) {
if (error instanceof TorboxApiError) {
if (error instanceof TorboxSearchApiError) {
switch (error.errorCode) {
case 'BAD_TOKEN':
return [
@@ -422,16 +460,21 @@ export class UsenetSourceHandler extends SourceHandler {
}),
];
default:
logger.error(`Error fetching NZBs for ${id}: ${error.message}`);
logger.error(
`Error fetching NZBs for ${type}:${value}: ${error.message}`
);
throw error;
}
}
logger.error(`Unexpected error fetching NZBs for ${id}: ${error}`);
logger.error(
`Unexpected error fetching NZBs for ${type}:${value}: ${error}`
);
throw error;
}
} else {
usingCachedSearch = true;
logger.info(`Found ${torrents.length} (cached) NZBs for ${id}`);
logger.info(
`Found ${torrents.length} (cached) NZBs for ${type}:${value}`
);
}
if (userData.onlyShowUserSearchResults) {
@@ -448,90 +491,29 @@ export class UsenetSourceHandler extends SourceHandler {
}
}
let instantAvailability: Map<string, boolean> | undefined;
// when using a cached search, we need to check instant availability separately
if (usingCachedSearch) {
instantAvailability = await this.getInstantAvailability(torrents);
}
const nzbs = torrents
.filter((torrent) => torrent.nzb)
.map((torrent) => ({
...torrent,
type: 'usenet' as const,
nzb: torrent.nzb!,
}));
return torrents.map((torrent) => {
const file: DebridFile = {
hash: torrent.hash,
filename: torrent.title,
size: torrent.size,
index: -1,
service: {
id: 'torbox',
cached:
instantAvailability?.get(torrent.hash) ?? torrent.cached ?? false,
owned: torrent.owned ?? false,
},
};
return this.createStream(
parsedId,
torrent,
file,
userData,
season,
episode
);
const { results, errors } = await processNZBs(
nzbs,
this.services,
parsedId.fullId,
titleMetadata,
this.clientIp
);
results.forEach((result) => {
result.service!.owned =
nzbs.find((nzb) => nzb.hash === result.hash)?.owned ?? false;
});
}
private async getInstantAvailability(
torrents: Torrent[]
): Promise<Map<string, boolean> | undefined> {
const start = Date.now();
const instantAvailability = new Map<string, boolean>();
const torrentsToCheck: Torrent[] = [];
for (const torrent of torrents) {
const cachedStatus = await this.instantAvailabilityCache.get(
torrent.hash
);
if (cachedStatus !== undefined) {
instantAvailability.set(torrent.hash, cachedStatus);
} else {
torrentsToCheck.push(torrent);
}
}
if (torrentsToCheck.length > 0) {
logger.debug(
`Checking instant availability for ${torrentsToCheck.length} NZBs`
);
const data = await this.torboxApi.usenet.getUsenetCachedAvailability(
'v1',
{
hash: torrentsToCheck.map((torrent) => torrent.hash).join(','),
format: 'list',
}
);
if (!data.data?.success) {
throw new Error(
`Failed to check instant availability: ${data.data?.detail} - ${data.data?.error}`
);
}
if (!Array.isArray(data.data.data)) {
throw new Error('Invalid response from Torbox API');
}
for (const torrent of torrentsToCheck) {
const item = data.data.data.find((item) => item.hash === torrent.hash);
instantAvailability.set(torrent.hash, Boolean(item));
this.instantAvailabilityCache.set(
torrent.hash,
Boolean(item),
Env.BUILTIN_TORBOX_SEARCH_INSTANT_AVAILABILITY_CACHE_TTL
);
}
}
const cachedTorrents = torrents.filter(
(torrent) => instantAvailability.get(torrent.hash) ?? torrent.cached
return results.map((result) =>
this.createStream(parsedId, result, userData, titleMetadata)
);
logger.info(
`Checked instant availability for ${torrents.length} NZBs in ${getTimeTakenSincePoint(start)}: ${cachedTorrents.length} / ${torrents.length} cached (with ${
torrents.length - torrentsToCheck.length
} cached statuses)`
);
return instantAvailability;
}
}
@@ -1,11 +1,13 @@
import { z } from 'zod';
import { TorBoxSearchApiDataSchema } from './schemas';
import { ServiceId } from '../../utils';
import { DebridFile } from './debrid-service';
import {
extractInfoHashFromMagnet,
extractTrackersFromMagnet,
} from '../utils/debrid';
export interface Torrent {
hash: string;
magnet?: string;
// magnet?: string;
title: string;
fileIdx?: number;
size: number;
@@ -13,19 +15,22 @@ export interface Torrent {
age?: string;
seeders?: number;
type: 'torrent' | 'usenet';
sources: string[];
nzb?: string;
userSearch?: boolean;
cached?: boolean;
owned?: boolean;
availableFiles?: DebridFile[];
}
export function convertDataToTorrents(
data: z.infer<typeof TorBoxSearchApiDataSchema>['torrents']
): Torrent[] {
return (data || []).map((file) => ({
hash: file.hash,
magnet: file.magnet ?? undefined,
hash:
file.hash ??
(file.magnet ? extractInfoHashFromMagnet(file.magnet) : undefined),
// magnet: file.magnet ?? undefined,
sources: file.magnet ? extractTrackersFromMagnet(file.magnet) : [],
title: file.raw_title,
size: file.size,
indexer: file.tracker,
@@ -0,0 +1,97 @@
import { z } from 'zod';
import { ParsedId } from '../../utils/id-parser';
import { createLogger } from '../../utils';
import { Torrent, NZB, UnprocessedTorrent } from '../../debrid';
import { SearchMetadata } from '../base/debrid';
import { extractTrackersFromMagnet } from '../utils/debrid';
import { BaseNabApi, Capabilities } from '../base/nab/api';
import {
BaseNabAddon,
NabAddonConfigSchema,
NabAddonConfig,
} from '../base/nab/addon';
const logger = createLogger('torznab');
// API client is now just a thin wrapper
class TorznabApi extends BaseNabApi<'torznab'> {
constructor(baseUrl: string, apiKey?: string, apiPath?: string) {
super('torznab', logger, baseUrl, apiKey, apiPath);
}
}
// Addon class
export class TorznabAddon extends BaseNabAddon<NabAddonConfig, TorznabApi> {
readonly name = 'Torznab';
readonly version = '1.0.0';
readonly id = 'torznab';
readonly logger = logger;
readonly api: TorznabApi;
constructor(userData: NabAddonConfig, clientIp?: string) {
super(userData, NabAddonConfigSchema, clientIp);
this.api = new TorznabApi(
this.userData.url,
this.userData.apiKey,
this.userData.apiPath
);
}
protected async _searchTorrents(
parsedId: ParsedId,
metadata: SearchMetadata
): Promise<UnprocessedTorrent[]> {
const results = await this.performSearch(parsedId, metadata);
const seenTorrents = new Set<string>();
const torrents: UnprocessedTorrent[] = [];
for (const result of results) {
const infoHash = this.extractInfoHash(result);
const downloadUrl = result.enclosure.find(
(e: any) =>
e.type === 'application/x-bittorrent' && e.url.includes('.torrent')
)?.url;
if (!infoHash && !downloadUrl) continue;
if (seenTorrents.has(infoHash ?? downloadUrl!)) continue;
seenTorrents.add(infoHash ?? downloadUrl!);
torrents.push({
hash: infoHash,
downloadUrl,
sources: result.torznab?.magneturl?.toString()
? extractTrackersFromMagnet(result.torznab.magneturl.toString())
: [],
seeders:
typeof result.torznab?.seeders === 'number' &&
![-1, 999].includes(result.torznab.seeders)
? result.torznab.seeders
: undefined,
title: result.title,
size: result.size ?? 0,
type: 'torrent',
});
}
return torrents;
}
protected async _searchNzbs(
parsedId: ParsedId,
metadata: SearchMetadata
): Promise<NZB[]> {
// This addon does not support NZBs, so we return an empty array.
return [];
}
private extractInfoHash(result: any): string | undefined {
return (
result.torznab?.infohash?.toString() ||
result.torznab?.magneturl
?.toString()
?.match(/(?:urn(?::|%3A)btih(?::|%3A))([a-f0-9]{40})/i)?.[1]
?.toLowerCase()
);
}
}
@@ -0,0 +1 @@
export * from './addon';
+391
View File
@@ -0,0 +1,391 @@
// utility function for debrid-based builtins
import z from 'zod';
import {
BuiltinServiceId,
constants,
createLogger,
getTimeTakenSincePoint,
} from '../../utils';
import {
BuiltinDebridServices,
DebridDownload,
DebridError,
DebridFile,
getDebridService,
selectFileInTorrentOrNZB,
Torrent,
TorrentWithSelectedFile,
NZBWithSelectedFile,
NZB,
isSeasonWrong,
isEpisodeWrong,
} from '../../debrid';
import { PTT } from '../../parser';
import { ParseResult } from 'go-ptt';
// we have a list of torrents which need to be
// - 1. checked for instant availability for each configured debrid service
// - 2. pick a file from file list if available
// - 3. return list of torrents but with service info too.
const logger = createLogger('debrid');
// export function
interface Metadata {
titles: string[];
season?: number;
episode?: number;
absoluteEpisode?: number;
}
export function extractTrackersFromMagnet(magnet: string): string[] {
return new URL(magnet.replace('&amp;', '&')).searchParams.getAll('tr');
}
export function extractInfoHashFromMagnet(magnet: string): string | undefined {
return magnet
.match(/(?:urn(?::|%3A)btih(?::|%3A))([a-f0-9]{40})/i)?.[1]
?.toLowerCase();
}
export async function processTorrents(
torrents: Torrent[],
debridServices: BuiltinDebridServices,
stremioId: string,
metadata?: Metadata,
clientIp?: string
): Promise<{
results: TorrentWithSelectedFile[];
errors: { serviceId: BuiltinServiceId; error: Error }[];
}> {
if (torrents.length === 0) {
return { results: [], errors: [] };
}
const results: TorrentWithSelectedFile[] = [];
const errors: { serviceId: BuiltinServiceId; error: Error }[] = [];
// Run all service checks in parallel and collect both results and errors
const servicePromises = debridServices.map(async (service) => {
try {
const serviceResults = await processTorrentsForDebridService(
torrents,
service,
stremioId,
metadata,
clientIp
);
return { serviceId: service.id, results: serviceResults, error: null };
} catch (error) {
logger.error(
`Error processing torrents for ${service.id}: ${error}`,
error
);
return { serviceId: service.id, results: [], error };
}
});
const settledResults = await Promise.all(servicePromises);
for (const { results: serviceResults, error, serviceId } of settledResults) {
if (serviceResults && serviceResults.length > 0) {
results.push(...serviceResults);
}
if (error instanceof Error) {
errors.push({ serviceId, error });
}
}
return { results, errors };
}
async function processTorrentsForDebridService(
torrents: Torrent[],
service: BuiltinDebridServices[number],
stremioId: string,
metadata?: Metadata,
clientIp?: string
): Promise<TorrentWithSelectedFile[]> {
const startTime = Date.now();
const debridService = getDebridService(
service.id,
service.credential,
clientIp
);
const results: TorrentWithSelectedFile[] = [];
const magnetCheckResults = await debridService.checkMagnets(
torrents.map((torrent) => torrent.hash),
stremioId
);
const allStrings: string[] = [];
// First add all torrent titles
for (const torrent of torrents) {
allStrings.push(torrent.title ?? '');
}
// Then add all filenames from all torrents
for (const torrent of magnetCheckResults) {
if (torrent.files && Array.isArray(torrent.files)) {
for (const file of torrent.files) {
allStrings.push(file.name ?? '');
}
}
}
// Parse all strings in one call
const allParsedResults = await PTT.parse(allStrings);
// Split the results into parsed titles and files
const parsedFiles = new Map<string, ParseResult>();
for (const [index, result] of allParsedResults.entries()) {
if (result) {
parsedFiles.set(allStrings[index], result);
}
}
for (const [index, torrent] of torrents.entries()) {
let file: DebridFile | undefined;
const magnetCheckResult = magnetCheckResults.find(
(result) => result.hash === torrent.hash
);
const parsedTorrent = parsedFiles.get(torrent.title ?? '');
if (metadata && parsedTorrent) {
if (isSeasonWrong(parsedTorrent, metadata)) {
continue;
}
if (isEpisodeWrong(parsedTorrent, metadata)) {
continue;
}
}
file = magnetCheckResult
? await selectFileInTorrentOrNZB(
torrent,
magnetCheckResult,
parsedFiles,
metadata
)
: { name: torrent.title, size: torrent.size, index: -1 };
if (file) {
results.push({
...torrent,
file,
service: {
id: service.id,
cached: magnetCheckResult?.status === 'cached',
owned: false,
},
});
}
}
logger.debug(
`Processed ${torrents.length} torrents for ${service.id} in ${getTimeTakenSincePoint(startTime)}`
);
return results;
}
export async function processTorrentsForP2P(
torrents: Torrent[],
metadata?: Metadata
): Promise<TorrentWithSelectedFile[]> {
const results: TorrentWithSelectedFile[] = [];
const allStrings: string[] = [];
for (const torrent of torrents) {
allStrings.push(torrent.title ?? '');
if (torrent.files && Array.isArray(torrent.files)) {
for (const file of torrent.files) {
allStrings.push(file.name ?? '');
}
}
}
const allParsedResults = await PTT.parse(allStrings);
const parsedFiles = new Map<string, ParseResult>();
for (const [index, result] of allParsedResults.entries()) {
if (result) {
parsedFiles.set(allStrings[index], result);
}
}
for (const [index, torrent] of torrents.entries()) {
let file: DebridFile | undefined;
const parsedTorrent = parsedFiles.get(torrent.title ?? '');
if (metadata && parsedTorrent) {
if (isSeasonWrong(parsedTorrent, metadata)) {
continue;
}
if (isEpisodeWrong(parsedTorrent, metadata)) {
continue;
}
}
file = torrent.files
? await selectFileInTorrentOrNZB(
torrent,
{
id: 'p2p',
name: torrent.title,
size: torrent.size,
status: 'downloaded',
files: torrent.files,
},
parsedFiles,
metadata
)
: undefined;
if (file) {
results.push({
...torrent,
file,
});
}
}
return results;
}
export async function processNZBs(
nzbs: NZB[],
debridServices: BuiltinDebridServices,
stremioId: string,
metadata?: Metadata,
clientIp?: string
): Promise<{
results: NZBWithSelectedFile[];
errors: { serviceId: BuiltinServiceId; error: Error }[];
}> {
if (nzbs.length === 0) {
return { results: [], errors: [] };
}
const results: NZBWithSelectedFile[] = [];
const errors: { serviceId: BuiltinServiceId; error: Error }[] = [];
const servicePromises = debridServices.map(async (service) => {
try {
const serviceResults = await processNZBsForDebridService(
nzbs,
service,
stremioId,
metadata,
clientIp
);
return { serviceId: service.id, results: serviceResults, error: null };
} catch (error) {
logger.error(`Error processing NZBs for ${service.id}: ${error}`, error);
return { serviceId: service.id, results: [], error };
}
});
const settledResults = await Promise.all(servicePromises);
for (const { results: serviceResults, error, serviceId } of settledResults) {
if (serviceResults && serviceResults.length > 0) {
results.push(...serviceResults);
}
if (error instanceof Error) {
errors.push({ serviceId, error });
}
}
return { results, errors };
}
async function processNZBsForDebridService(
nzbs: NZB[],
service: BuiltinDebridServices[number],
stremioId: string,
metadata?: Metadata,
clientIp?: string
): Promise<NZBWithSelectedFile[]> {
const startTime = Date.now();
const debridService = getDebridService(
service.id,
service.credential,
clientIp
);
if (!debridService.supportsUsenet || !debridService.checkNzbs) {
throw new Error(`Service ${service.id} does not support usenet`);
}
const results: NZBWithSelectedFile[] = [];
const nzbCheckResults = await debridService.checkNzbs(
nzbs.map((nzb) => nzb.hash)
);
// parse all files from all nzbs in one call
const allStrings: string[] = [];
for (const nzb of nzbCheckResults) {
allStrings.push(nzb.name ?? '');
if (nzb.files && Array.isArray(nzb.files)) {
for (const file of nzb.files) {
allStrings.push(file.name ?? '');
}
}
}
const allParsedResults = await PTT.parse(allStrings);
const parsedFiles = new Map<string, ParseResult>();
for (const [index, result] of allParsedResults.entries()) {
if (result) {
parsedFiles.set(allStrings[index], result);
}
}
for (const [index, nzb] of nzbs.entries()) {
let file: DebridFile | undefined;
const nzbCheckResult = nzbCheckResults.find(
(result) => result.hash === nzb.hash
);
const parsedNzb = parsedFiles.get(nzb.title ?? '');
if (metadata && parsedNzb) {
if (isSeasonWrong(parsedNzb, metadata)) {
continue;
}
if (isEpisodeWrong(parsedNzb, metadata)) {
continue;
}
}
file = nzbCheckResult
? await selectFileInTorrentOrNZB(
nzb,
nzbCheckResult,
parsedFiles,
metadata
)
: { name: nzb.title, size: nzb.size, index: -1 };
if (file) {
results.push({
...nzb,
file,
service: {
id: service.id,
cached: nzbCheckResult?.status === 'cached',
owned: false,
},
});
}
}
logger.debug(
`Processed ${nzbs.length} NZBs for ${service.id} in ${getTimeTakenSincePoint(startTime)}`
);
return results;
}
@@ -1,92 +0,0 @@
import { TorboxSearchApiIdType } from '../torbox-search/search-api';
export interface ParsedId {
type: TorboxSearchApiIdType;
id: string;
season?: string;
episode?: string;
}
interface IdParserDefinition {
type: TorboxSearchApiIdType;
prefixes: string[];
// Regex should have named capture groups:
// - id: the base ID (required)
// - season: the season number (optional)
// - episode: the episode number (optional)
regex: RegExp;
format: (id: string) => string;
}
export class IdParser {
private readonly ID_PARSERS: IdParserDefinition[] = [
{
type: 'imdb_id',
prefixes: ['tt', 'imdb'],
regex: /^(?:tt|imdb)[:-]?(?<id>\d+)(?::(?<season>\d+):(?<episode>\d+))?$/,
format: (id) => `tt${id}`,
},
{
type: 'mal_id',
prefixes: ['mal'],
regex: /^mal[:-]?(?<id>\d+)(?::(?<episode>\d+))?$/,
format: (id) => id,
},
{
type: 'thetvdb_id',
prefixes: ['tvdb'],
regex: /^tvdb[:-]?(?<id>\d+)(?::(?<season>\d+):(?<episode>\d+))?$/,
format: (id) => id,
},
{
type: 'tmdb',
prefixes: ['tmdb'],
regex: /^tmdb[:-]?(?<id>\d+)(?::(?<season>\d+):(?<episode>\d+))?$/,
format: (id) => id,
},
{
type: 'kitsu_id',
prefixes: ['kitsu'],
regex: /^kitsu[:-]?(?<id>\d+)(?::(?<episode>\d+))?$/,
format: (id) => id,
},
{
type: 'anilist_id',
prefixes: ['anilist'],
regex: /^anilist[:-]?(?<id>\d+)(?::(?<episode>\d+))?$/,
format: (id) => id,
},
{
type: 'anidb_id',
prefixes: ['anidb', 'anidb_id', 'anidbid'],
regex: /^(?:anidb|anidb_id|anidbid)[:-]?(?<id>\d+)(?::(?<episode>\d+))?$/,
format: (id) => id,
},
];
constructor(private readonly supportedIdTypes: TorboxSearchApiIdType[]) {}
public readonly supportedPrefixes = this.ID_PARSERS.filter((p) =>
this.supportedIdTypes.includes(p.type)
).flatMap((p) => p.prefixes);
public parse(stremioId: string): ParsedId | null {
for (const parser of this.ID_PARSERS) {
const match = stremioId.match(parser.regex);
if (match?.groups) {
const { id, season, episode } = match.groups;
const parsedId: ParsedId = {
type: parser.type,
id: parser.format(id),
};
if (season) parsedId.season = season;
if (episode) parsedId.episode = episode;
return parsedId;
}
}
return null;
}
}
+48 -11
View File
@@ -53,6 +53,17 @@ export class DB {
return DB.dialect;
}
isSQLite(): boolean {
return this.getDialect() === 'sqlite';
}
getRowsAffected(result: any): number {
if (this.isSQLite()) {
return result.changes || 0;
}
return result.rowCount || 0;
}
async initialise(
uri: string,
dsnModifiers: DSNModifier[] = []
@@ -77,7 +88,6 @@ export class DB {
await this.execute('PRAGMA foreign_keys = ON');
await this.execute('PRAGMA synchronous = OFF');
await this.execute('PRAGMA journal_mode = WAL');
await this.execute('PRAGMA locking_mode = IMMEDIATE');
}
DB.initialised = true;
@@ -204,25 +214,52 @@ export class DB {
},
};
} else if (this.uri.dialect === 'sqlite') {
const db = this.db as Database<any>;
await db.run('BEGIN');
// For sqlite, we manually manage the transaction state.
await (this.db as Database<any>).exec('BEGIN');
let isFinalised = false;
return {
commit: async () => {
await db.run('COMMIT');
if (isFinalised) return;
await (this.db as Database<any>).exec('COMMIT');
isFinalised = true;
},
rollback: async () => {
await db.run('ROLLBACK');
if (isFinalised) return;
await (this.db as Database<any>).exec('ROLLBACK');
isFinalised = true;
},
execute: async (
query: string,
params?: any[]
): Promise<UnifiedQueryResult> => {
const result = await db.all(adaptQuery(query, 'sqlite'), params);
return {
rows: result,
rowCount: result.length || 0,
command: 'SELECT',
};
if (isFinalised) {
throw new Error('Transaction has already been finalised.');
}
const adaptedQuery = adaptQuery(query, 'sqlite');
const command = adaptedQuery.trim().split(' ')[0].toUpperCase();
if (['INSERT', 'UPDATE', 'DELETE'].includes(command)) {
const result = await (this.db as Database<any>).run(
adaptedQuery,
params
);
return {
rows: [],
rowCount: result.changes || 0,
command: command,
};
} else {
const rows = await (this.db as Database<any>).all(
adaptedQuery,
params
);
return {
rows: rows,
rowCount: rows.length,
command: command,
};
}
},
};
}
+25 -24
View File
@@ -1,12 +1,17 @@
import { createLogger } from '../utils';
import { DB } from './db';
const logger = createLogger('db');
const db = DB.getInstance();
// Queue for SQLite transactions
interface QueuedOperation<T> {
operation: () => Promise<T>;
resolve: (value: T | PromiseLike<T>) => void;
reject: (reason?: any) => void;
}
export class TransactionQueue {
private queue: Array<() => Promise<any>> = [];
private queue: Array<QueuedOperation<any>> = [];
private processing = false;
private static instance: TransactionQueue;
@@ -21,38 +26,34 @@ export class TransactionQueue {
async enqueue<T>(operation: () => Promise<T>): Promise<T> {
// If using PostgreSQL, execute directly without queuing
if (db['uri']?.dialect === 'postgres') {
if (db.getDialect() === 'postgres') {
return operation();
}
return new Promise((resolve, reject) => {
this.queue.push(async () => {
try {
const result = await operation();
resolve(result);
} catch (error) {
reject(error);
}
});
return new Promise<T>((resolve, reject) => {
this.queue.push({ operation, resolve, reject });
this.processQueue();
});
}
private async processQueue() {
if (this.processing || this.queue.length === 0) return;
if (this.processing || this.queue.length === 0) {
return;
}
this.processing = true;
while (this.queue.length > 0) {
const operation = this.queue.shift();
if (operation) {
try {
await operation();
} catch (error) {
logger.error('Error processing queued operation:', error);
}
}
}
const { operation, resolve, reject } = this.queue.shift()!;
this.processing = false;
try {
const result = await operation();
resolve(result);
} catch (error) {
logger.error('Error processing queued operation:', error);
reject(error);
} finally {
this.processing = false;
// After finishing one operation, check if there are more in the queue
this.processQueue();
}
}
}
+6
View File
@@ -453,6 +453,12 @@ export const TABLES = {
updated_at TIMESTAMP DEFAULT (CURRENT_TIMESTAMP),
accessed_at TIMESTAMP DEFAULT (CURRENT_TIMESTAMP)
`,
distributed_locks: `
key TEXT PRIMARY KEY,
owner TEXT NOT NULL,
expires_at BIGINT NOT NULL,
result TEXT
`,
};
const strictManifestResourceSchema = z.object({
+178
View File
@@ -0,0 +1,178 @@
import { z } from 'zod';
import { constants, ServiceId } from '../utils';
import { ErrorType, StremThruError } from 'stremthru';
// type ErrorCode = "BAD_GATEWAY" | "BAD_REQUEST" | "CONFLICT" | "FORBIDDEN" | "GONE" | "INTERNAL_SERVER_ERROR" | "METHOD_NOT_ALLOWED" | "NOT_FOUND" | "NOT_IMPLEMENTED" | "PAYMENT_REQUIRED" | "PROXY_AUTHENTICATION_REQUIRED" | "SERVICE_UNAVAILABLE" | "STORE_LIMIT_EXCEEDED" | "STORE_MAGNET_INVALID" | "TOO_MANY_REQUESTS" | "UNAUTHORIZED" | "UNAVAILABLE_FOR_LEGAL_REASONS" | "UNKNOWN" | "UNPROCESSABLE_ENTITY" | "UNSUPPORTED_MEDIA_TYPE";
StremThruError;
type DebridErrorCode =
| 'BAD_GATEWAY'
| 'BAD_REQUEST'
| 'CONFLICT'
| 'FORBIDDEN'
| 'GONE'
| 'INTERNAL_SERVER_ERROR'
| 'METHOD_NOT_ALLOWED'
| 'NOT_FOUND'
| 'NOT_IMPLEMENTED'
| 'PAYMENT_REQUIRED'
| 'PROXY_AUTHENTICATION_REQUIRED'
| 'SERVICE_UNAVAILABLE'
| 'STORE_LIMIT_EXCEEDED'
| 'STORE_MAGNET_INVALID'
| 'TOO_MANY_REQUESTS'
| 'UNAUTHORIZED'
| 'UNAVAILABLE_FOR_LEGAL_REASONS'
| 'UNKNOWN'
| 'UNPROCESSABLE_ENTITY'
| 'UNSUPPORTED_MEDIA_TYPE'
| 'NO_MATCHING_FILE';
type DebridErrorType =
| 'api_error'
| 'store_error'
| 'unknown_error'
| 'upstream_error';
export class DebridError extends Error {
body?: unknown;
code?: DebridErrorCode = 'UNKNOWN';
headers: Record<string, string>;
statusCode: number;
statusText: string;
cause?: unknown;
type?: DebridErrorType = 'unknown_error';
constructor(
message: string,
options: Pick<
DebridError,
'body' | 'code' | 'headers' | 'statusCode' | 'statusText' | 'type'
> & { cause?: unknown }
) {
super(message);
if (Error.captureStackTrace) {
Error.captureStackTrace(this, this.constructor);
}
if (options?.cause) {
this.cause = options.cause;
delete options.cause;
}
if (options.body) {
this.body = options.body;
}
this.headers = options.headers;
this.statusCode = options.statusCode;
this.statusText = options.statusText;
if (options.type) {
this.type = options.type;
}
if (options.code) {
this.code = options.code;
}
}
}
const DebridFileSchema = z.object({
id: z.number().optional(),
name: z.string().optional(),
size: z.number(),
mimeType: z.string().optional(),
link: z.string().optional(),
path: z.string().optional(),
index: z.number().optional(),
});
export type DebridFile = z.infer<typeof DebridFileSchema>;
export interface DebridDownload {
id: string | number;
hash?: string;
name?: string;
size?: number;
status:
| 'cached'
| 'downloaded'
| 'downloading'
| 'failed'
| 'invalid'
| 'processing'
| 'queued'
| 'unknown'
| 'uploading';
files?: DebridFile[];
}
const BasePlaybackInfoSchema = z.object({
// hash: z.string(),
title: z.string().optional(),
metadata: z
.object({
titles: z.array(z.string()),
season: z.number().optional(),
episode: z.number().optional(),
absoluteEpisode: z.number().optional(),
})
.optional(),
file: DebridFileSchema.optional(),
});
const TorrentPlaybackInfoSchema = BasePlaybackInfoSchema.extend({
hash: z.string(),
sources: z.array(z.string()),
// magnet: z.string().optional(),
type: z.literal('torrent'),
});
const UsenetPlaybackInfoSchema = BasePlaybackInfoSchema.extend({
hash: z.string(),
nzb: z.string(),
type: z.literal('usenet'),
});
export const PlaybackInfoSchema = z.discriminatedUnion('type', [
TorrentPlaybackInfoSchema,
UsenetPlaybackInfoSchema,
]);
export const ServiceAuthSchema = z.object({
id: z.enum(constants.BUILTIN_SUPPORTED_SERVICES),
credential: z.string(),
});
export type ServiceAuth = z.infer<typeof ServiceAuthSchema>;
export type PlaybackInfo = z.infer<typeof PlaybackInfoSchema>;
export interface DebridService {
// Common methods
resolve(
playbackInfo: PlaybackInfo,
filename: string
): Promise<string | undefined>;
// Torrent specific methods
checkMagnets(magnets: string[], sid?: string): Promise<DebridDownload[]>;
addMagnet(magnet: string): Promise<DebridDownload>;
generateTorrentLink(link: string, clientIp?: string): Promise<string>;
// Usenet specific methods
checkNzbs?(nzbs: string[]): Promise<DebridDownload[]>;
addNzb?(nzb: string, name: string): Promise<DebridDownload>;
generateUsenetLink?(
downloadId: string,
fileId?: string,
clientIp?: string
): Promise<string>;
// Service info
readonly serviceName: ServiceId;
readonly supportsUsenet: boolean;
}
export type DebridServiceConfig = {
token: string;
clientIp?: string;
};
+31 -1
View File
@@ -1 +1,31 @@
export * from './interface';
export * from './base';
export * from './utils';
export * from './stremthru';
export * from './torbox';
import { ServiceId } from '../utils';
import { DebridService, DebridServiceConfig } from './base';
import { StremThruInterface } from './stremthru';
import { TorboxDebridService } from './torbox';
import { StremThruPreset } from '../presets/stremthru';
export function getDebridService(
serviceName: ServiceId,
token: string,
clientIp?: string
): DebridService {
const config: DebridServiceConfig = {
token,
clientIp,
};
switch (serviceName) {
case 'torbox':
return new TorboxDebridService(config);
default:
if (StremThruPreset.supportedServices.includes(serviceName)) {
return new StremThruInterface({ ...config, serviceName });
}
throw new Error(`Unknown debrid service: ${serviceName}`);
}
}
-355
View File
@@ -1,355 +0,0 @@
import { StremThru, StremThruError } from 'stremthru';
import { constants, Env, ServiceId, createLogger, Cache } from '../utils';
import { TorboxApi } from '@torbox/torbox-api';
import { z } from 'zod';
import { FileParser } from '../parser';
import { findMatchingFileInTorrent, isVideoFile } from './utils';
import { title } from 'process';
const logger = createLogger('debrid');
export type DebridErrorCode = 'NO_MATCHING_FILE';
export class DebridError extends Error {
constructor(
message: string,
public readonly code: DebridErrorCode
) {
super(message);
}
}
interface UsenetDownload {
usenetDownloadId: number;
authId?: string;
hash?: string;
name?: string;
status: 'downloaded' | 'downloading' | 'queued';
files: {
id?: number;
size?: number;
mimeType?: string;
name?: string;
}[];
}
export const getProvider = (
storeName: ServiceId,
storeCredential: string,
clientIp?: string
): StremThru => {
return new StremThru({
baseUrl: Env.BUILTIN_STREMTHRU_URL,
userAgent: Env.DEFAULT_USER_AGENT,
auth: {
store: storeName,
token: storeCredential,
},
clientIp,
timeout: 20000,
});
};
type StoreAuth = {
storeName: ServiceId;
storeCredential: string;
};
export const StoreAuthSchema = z.object({
storeName: z.enum(constants.SERVICES),
storeCredential: z.string(),
});
const TorrentPlaybackInfoSchema = z.object({
parsedId: z.object({
id: z.string(),
type: z.string(),
season: z.string().optional(),
episode: z.string().optional(),
absoluteEpisode: z.string().optional(),
}),
type: z.literal('torrent'),
hash: z.string(),
title: z.string().optional(),
magnet: z.string().optional(),
index: z.number().optional(),
});
const UsenetPlaybackInfoSchema = z.object({
type: z.literal('usenet'),
title: z.string().optional(),
nzb: z.string(),
});
export const PlaybackInfoSchema = z.discriminatedUnion('type', [
TorrentPlaybackInfoSchema,
UsenetPlaybackInfoSchema,
]);
type PlaybackInfo = z.infer<typeof PlaybackInfoSchema>;
export class DebridInterface {
private readonly stremthru: StremThru;
private readonly torboxApi: TorboxApi | undefined;
private static playbackLinkCache = Cache.getInstance<string, string>(
'debrid-link'
);
constructor(
private readonly storeAuth: StoreAuth,
private readonly clientIp?: string
) {
const { storeName, storeCredential } = storeAuth;
this.stremthru = getProvider(storeName, storeCredential, clientIp);
if (storeName === 'torbox') {
this.torboxApi = new TorboxApi({
token: storeCredential,
});
}
}
public async checkMagnets(magnets: string[], sid?: string) {
return await this.stremthru.store.checkMagnet({
magnet: magnets,
sid,
});
}
public async resolve(
playbackInfo: PlaybackInfo,
filename: string
): Promise<string | undefined> {
if (playbackInfo.type === 'torrent') {
logger.debug(`Resolving torrent ${playbackInfo.hash}`);
return await this.resolveTorrent(playbackInfo, filename);
} else {
logger.debug(`Resolving usenet ${playbackInfo.nzb}`);
return await this.resolveUsenet(playbackInfo, filename);
}
}
private async resolveTorrent(
playbackInfo: PlaybackInfo & { type: 'torrent' },
filename: string
): Promise<string | undefined> {
let { hash, index, parsedId } = playbackInfo;
const cacheKey = `${this.storeAuth.storeName}:${hash}:${index ?? 'undefined'}:${this.storeAuth.storeCredential}:${this.clientIp}`;
const cachedLink = await DebridInterface.playbackLinkCache.get(cacheKey);
if (cachedLink) {
logger.debug(`Using cached link for ${hash}`);
return cachedLink;
}
if (index === -1) {
index = undefined;
}
logger.debug(
`Adding magnet to ${this.storeAuth.storeName} for ${playbackInfo.magnet || `magnet:?xt=urn:btih:${hash}`}`
);
const magnet = await this.stremthru.store.addMagnet({
magnet: playbackInfo.magnet || `magnet:?xt=urn:btih:${hash}`,
});
if (magnet.data.status !== 'downloaded') {
return undefined;
}
// parse files first
const files = magnet.data.files.map((file) => {
return {
parsed: FileParser.parse(file.name),
isVideo: isVideoFile(file.name),
index: file.index,
name: file.name,
size: file.size,
link: file.link,
path: file.path,
};
});
const file = findMatchingFileInTorrent(
files,
index,
filename,
undefined,
parsedId.season,
parsedId.episode,
parsedId.absoluteEpisode,
true
);
if (!file || !file.link) {
throw new DebridError('No matching file found', 'NO_MATCHING_FILE');
}
logger.debug(`Found matching file`, {
season: parsedId.season,
episode: parsedId.episode,
absoluteEpisode: parsedId.absoluteEpisode,
chosenFile: file.name,
availableFiles: `[${files.map((file) => file.name).join(', ')}]`,
});
const link = await this.stremthru.store.generateLink({
link: file.link,
clientIp: this.clientIp,
});
const playbackLink = link.data.link;
await DebridInterface.playbackLinkCache.set(
cacheKey,
playbackLink,
60 * 30
);
return playbackLink;
}
private async resolveUsenet(
playbackInfo: PlaybackInfo & { type: 'usenet' },
filename: string
): Promise<string | undefined> {
const { nzb } = playbackInfo;
const cacheKey = `${this.storeAuth.storeName}:${this.storeAuth.storeCredential}:${nzb}:${this.clientIp}`;
const cachedLink = await DebridInterface.playbackLinkCache.get(cacheKey);
if (cachedLink) {
logger.debug(`Using cached link for ${nzb}`);
return cachedLink;
}
if (!this.torboxApi) {
throw new Error('Torbox API not available');
}
const usenetDownload = await this.addUsenetDownload({
link: nzb,
name: filename,
});
if (!usenetDownload || usenetDownload.status !== 'downloaded') {
return undefined;
}
if (!usenetDownload.files.length) {
throw new Error('No files found for usenet download');
}
let fileId: number | undefined;
if (usenetDownload.files.length > 1) {
const files = usenetDownload.files.map((file) => {
return {
parsed: FileParser.parse(file.name ?? ''),
isVideo:
file.mimeType?.includes('video') || isVideoFile(file.name ?? ''),
index: file.id ?? 0,
name: file.name ?? '',
size: file.size ?? 0,
};
});
const file = findMatchingFileInTorrent(files);
if (!file) {
throw new Error('No matching file found');
}
logger.debug(`Found matching file`, {
chosenFile: file.name,
chosenIndex: file.index,
availableFiles: `[${files.map((file) => file.name).join(', ')}]`,
});
fileId = file.index;
}
const link = await this.torboxApi.usenet.requestDownloadLink('v1', {
usenetId: usenetDownload.usenetDownloadId.toString(),
fileId: fileId ? fileId.toString() : undefined,
userIp: this.clientIp,
redirect: 'false',
token: this.storeAuth.storeCredential,
});
logger.debug(
`Requested usenet download link for ${nzb}: ${link.data?.data} ${link.metadata.status}`
);
const playbackLink = link.data?.data;
if (playbackLink) {
await DebridInterface.playbackLinkCache.set(
cacheKey,
playbackLink,
60 * 30
);
}
return playbackLink;
}
private async addUsenetDownload(params: {
link: string;
name: string;
}): Promise<UsenetDownload | undefined> {
if (!this.torboxApi) {
throw new Error('Torbox API not available');
}
const { link, name } = params;
const res = await this.torboxApi.usenet.createUsenetDownload('v1', {
link,
name,
});
if (!res.data?.data?.usenetdownloadId) {
throw new Error(`Usenet download failed: ${res.data?.detail}`);
}
logger.debug(`Created usenet download for ${link}: ${res.data?.detail}`);
if (
res.data?.detail &&
!res.data.detail.includes('Using cached download.')
) {
logger.debug(
`Usenet download detected to not be cached, returning undefined`
);
return undefined;
}
let state: UsenetDownload['status'] = 'queued';
const nzb = await this.torboxApi?.usenet.getUsenetList('v1', {
id: res.data?.data?.usenetdownloadId?.toString(),
});
if (!nzb?.data?.data || nzb?.data?.error || nzb.data.success === false) {
throw new Error(
`Failed to get usenet list: ${nzb?.data?.error || 'Unknown error'}${nzb?.data?.detail ? '- ' + nzb.data.detail : ''}`
);
} else if (Array.isArray(nzb.data.data)) {
throw new Error(`Unexpected response format for usenet download`);
}
const usenetDownload = nzb.data.data;
if (usenetDownload.downloadFinished && usenetDownload.downloadPresent) {
state = 'downloaded';
} else if (usenetDownload.progress && usenetDownload.progress > 0) {
state = 'downloading';
}
const files: UsenetDownload['files'] = [];
for (const file of usenetDownload.files ?? []) {
files.push({
id: file.id,
mimeType: file.mimetype,
name: file.shortName ?? file.name,
size: file.size,
});
}
logger.debug(`Found matching usenet download`, {
usenetDownloadId: usenetDownload.id,
status: usenetDownload.downloadState,
state: state,
});
return {
usenetDownloadId: usenetDownload.id ?? res.data.data.usenetdownloadId,
authId: usenetDownload.authId ?? res.data.data.authId,
hash: usenetDownload.hash ?? res.data.data.hash,
name: usenetDownload.name ?? undefined,
status: state,
files,
};
}
}
+283
View File
@@ -0,0 +1,283 @@
import { StremThru, StremThruError } from 'stremthru';
import { Env, ServiceId, createLogger, getSimpleTextHash } from '../utils';
import { selectFileInTorrentOrNZB, Torrent } from './utils';
import {
DebridService,
DebridServiceConfig,
DebridDownload,
PlaybackInfo,
DebridError,
} from './base';
import { Cache } from '../utils';
import { StremThruServiceId } from '../presets/stremthru';
import { PTT } from '../parser';
import { ParseResult } from 'go-ptt';
const logger = createLogger('debrid:stremthru');
function convertStremThruError(error: StremThruError): DebridError {
return new DebridError(error.message, {
statusCode: error.statusCode,
statusText: error.statusText,
code: error.code,
headers: error.headers,
body: error.body,
cause: 'cause' in error ? error.cause : undefined,
});
}
export class StremThruInterface implements DebridService {
private readonly stremthru: StremThru;
private static playbackLinkCache = Cache.getInstance<string, string>(
'st:link'
);
private static checkCache = Cache.getInstance<string, DebridDownload>(
'st:instant-check'
);
readonly supportsUsenet = false;
readonly serviceName: ServiceId;
constructor(
private readonly config: DebridServiceConfig & {
serviceName: StremThruServiceId;
}
) {
this.serviceName = config.serviceName;
this.stremthru = new StremThru({
baseUrl: Env.BUILTIN_STREMTHRU_URL,
userAgent: Env.DEFAULT_USER_AGENT,
auth: {
store: config.serviceName,
token: config.token,
},
clientIp: config.clientIp,
timeout: 20000,
});
}
public async checkMagnets(
magnets: string[],
sid?: string
): Promise<DebridDownload[]> {
const cachedResults: DebridDownload[] = [];
const magnetsToCheck: string[] = [];
for (const magnet of magnets) {
const cacheKey = getSimpleTextHash(magnet);
const cached = await StremThruInterface.checkCache.get(cacheKey);
if (cached) {
cachedResults.push(cached);
} else {
magnetsToCheck.push(magnet);
}
}
if (magnetsToCheck.length > 0) {
let newResults: DebridDownload[] = [];
const BATCH_SIZE = 500;
// Split magnetsToCheck into batches of 500
const batches: string[][] = [];
for (let i = 0; i < magnetsToCheck.length; i += BATCH_SIZE) {
batches.push(magnetsToCheck.slice(i, i + BATCH_SIZE));
}
try {
// Perform all batch requests in parallel
const batchResults = await Promise.all(
batches.map(async (batch) => {
const result = await this.stremthru.store.checkMagnet({
magnet: batch,
sid,
});
return result.data.items;
})
);
// Flatten all items from all batches
const allItems = batchResults.flat();
for (const item of allItems) {
const download: DebridDownload = {
id: -1,
hash: item.hash,
status: item.status,
size: item.files.reduce((acc, file) => acc + file.size, 0),
files: item.files.map((file) => ({
name: file.name,
size: file.size,
index: file.index,
})),
};
newResults.push(download);
StremThruInterface.checkCache.set(
getSimpleTextHash(item.hash),
download,
Env.BUILTIN_DEBRID_INSTANT_AVAILABILITY_CACHE_TTL
);
}
} catch (error) {
if (error instanceof StremThruError) {
throw convertStremThruError(error);
}
throw error;
}
return [...cachedResults, ...newResults];
}
return cachedResults;
}
public async addMagnet(magnet: string): Promise<DebridDownload> {
try {
const result = await this.stremthru.store.addMagnet({
magnet,
});
return {
id: result.data.id,
status: result.data.status,
hash: result.data.hash,
size: result.data.files.reduce((acc, file) => acc + file.size, 0),
files: result.data.files.map((file) => ({
name: file.name,
size: file.size,
link: file.link,
path: file.path,
index: file.index,
})),
};
} catch (error) {
throw error instanceof StremThruError
? convertStremThruError(error)
: error;
}
}
public async generateTorrentLink(
link: string,
clientIp?: string
): Promise<string> {
try {
const result = await this.stremthru.store.generateLink({
link,
clientIp,
});
return result.data.link;
} catch (error) {
throw error instanceof StremThruError
? convertStremThruError(error)
: error;
}
}
public async resolve(
playbackInfo: PlaybackInfo,
filename: string
): Promise<string | undefined> {
if (playbackInfo.type === 'usenet') {
throw new DebridError('StremThru does not support usenet operations', {
statusCode: 400,
statusText: 'StremThru does not support usenet operations',
code: 'NOT_IMPLEMENTED',
headers: {},
body: playbackInfo,
});
}
const { hash, file: chosenFile, metadata } = playbackInfo;
const cacheKey = `${this.serviceName}:${this.config.token}:${this.config.clientIp}:${JSON.stringify(playbackInfo)}`;
const cachedLink = await StremThruInterface.playbackLinkCache.get(cacheKey);
let magnet = `magnet:?xt=urn:btih:${hash}`;
if (playbackInfo.sources.length > 0) {
magnet += `&tr=${playbackInfo.sources.join('&tr=')}`;
}
if (cachedLink) {
logger.debug(`Using cached link for ${hash}`);
return cachedLink;
}
logger.debug(`Adding magnet to ${this.serviceName} for ${magnet}`);
const magnetDownload = await this.addMagnet(magnet);
logger.debug(`Magnet download added for ${magnet}`, {
status: magnetDownload.status,
id: magnetDownload.id,
});
if (magnetDownload.status !== 'downloaded') {
return undefined;
}
if (!magnetDownload.files?.length) {
throw new DebridError('No files found for magnet download', {
statusCode: 400,
statusText: 'No files found for magnet download',
code: 'NO_MATCHING_FILE',
headers: {},
body: magnetDownload,
});
}
const torrent: Torrent = {
title: magnetDownload.name || playbackInfo.title || '',
hash: hash,
size: magnetDownload.size || 0,
type: 'torrent',
sources: playbackInfo.sources,
};
const allStrings: string[] = [];
allStrings.push(magnetDownload.name ?? '');
allStrings.push(...magnetDownload.files.map((file) => file.name ?? ''));
const parseResults = await PTT.parse(allStrings);
const parsedFiles = new Map<string, ParseResult>();
for (const [index, result] of parseResults.entries()) {
if (result) {
parsedFiles.set(allStrings[index], result);
}
}
const file = await selectFileInTorrentOrNZB(
torrent,
magnetDownload,
parsedFiles,
metadata,
{
chosenFilename: chosenFile?.name,
chosenIndex: chosenFile?.index,
}
);
if (!file?.link) {
throw new DebridError('No matching file found', {
statusCode: 400,
statusText: 'No matching file found',
code: 'NO_MATCHING_FILE',
headers: {},
body: file,
});
}
logger.debug(`Found matching file`, {
season: metadata?.season,
episode: metadata?.episode,
absoluteEpisode: metadata?.absoluteEpisode,
chosenFile: file.name,
availableFiles: `[${magnetDownload.files.map((file) => file.name).join(', ')}]`,
});
const playbackLink = await this.generateTorrentLink(
file.link,
this.config.clientIp
);
await StremThruInterface.playbackLinkCache.set(
cacheKey,
playbackLink,
Env.BUILTIN_DEBRID_PLAYBACK_LINK_CACHE_TTL
);
return playbackLink;
}
}
+448
View File
@@ -0,0 +1,448 @@
import { TorboxApi } from '@torbox/torbox-api';
import { Env, ServiceId, createLogger, getSimpleTextHash } from '../utils';
import { PTT } from '../parser';
import { selectFileInTorrentOrNZB } from './utils';
import {
DebridService,
DebridServiceConfig,
DebridDownload,
PlaybackInfo,
DebridError,
} from './base';
import { Cache } from '../utils';
import { StremThruInterface } from './stremthru';
import { StremThruError } from 'stremthru';
import { ParseResult } from 'go-ptt';
const logger = createLogger('debrid:torbox');
export class TorboxDebridService implements DebridService {
private readonly apiVersion = 'v1';
private readonly torboxApi: TorboxApi;
private readonly stremthru: StremThruInterface;
private static playbackLinkCache = Cache.getInstance<string, string>(
'tb:link'
);
private static instantAvailabilityCache = Cache.getInstance<
string,
DebridDownload
>('tb:instant-availability');
readonly supportsUsenet = true;
readonly serviceName: ServiceId = 'torbox';
constructor(private readonly config: DebridServiceConfig) {
this.torboxApi = new TorboxApi({
token: config.token,
});
this.stremthru = new StremThruInterface({
...config,
serviceName: this.serviceName,
});
}
public async checkMagnets(magnets: string[], sid?: string) {
try {
return this.stremthru.checkMagnets(magnets, sid);
} catch (error) {
if (error instanceof StremThruError) {
throw new DebridError(error.message, {
statusCode: error.statusCode,
statusText: error.statusText,
code: error.code,
headers: error.headers,
body: error.body,
cause: 'cause' in error ? error.cause : undefined,
});
}
throw error;
}
}
public async addMagnet(magnet: string): Promise<DebridDownload> {
try {
return this.stremthru.addMagnet(magnet);
} catch (error) {
if (error instanceof StremThruError) {
throw new DebridError(error.message, {
statusCode: error.statusCode,
statusText: error.statusText,
code: error.code,
headers: error.headers,
body: error.body,
cause: 'cause' in error ? error.cause : undefined,
});
}
throw error;
}
}
public async generateTorrentLink(
link: string,
clientIp?: string
): Promise<string> {
try {
return this.stremthru.generateTorrentLink(link, clientIp);
} catch (error) {
if (error instanceof StremThruError) {
throw new DebridError(error.message, {
statusCode: error.statusCode,
statusText: error.statusText,
code: error.code,
headers: error.headers,
body: error.body,
cause: 'cause' in error ? error.cause : undefined,
});
}
throw error;
}
}
public async checkNzbs(hashes: string[]): Promise<DebridDownload[]> {
const cachedResults: DebridDownload[] = [];
const hashesToCheck: string[] = [];
for (const hash of hashes) {
const cacheKey = getSimpleTextHash(hash);
const cached =
await TorboxDebridService.instantAvailabilityCache.get(cacheKey);
if (cached) {
cachedResults.push(cached);
} else {
hashesToCheck.push(hash);
}
}
if (hashesToCheck.length > 0) {
let newResults: DebridDownload[] = [];
const BATCH_SIZE = 100;
const batches: string[][] = [];
for (let i = 0; i < hashesToCheck.length; i += BATCH_SIZE) {
batches.push(hashesToCheck.slice(i, i + BATCH_SIZE));
}
const batchResults = await Promise.all(
batches.map(async (batch) => {
const result =
await this.torboxApi.usenet.getUsenetCachedAvailability(
this.apiVersion,
{
hash: batch.join(','),
format: 'list',
}
);
if (!result.data?.success) {
throw new DebridError(`Failed to check instant availability`, {
statusCode: result.metadata.status,
statusText: result.metadata.statusText,
code: 'UNKNOWN',
headers: result.metadata.headers,
body: result.data,
});
}
if (!Array.isArray(result.data.data)) {
throw new DebridError(
'Invalid response from Torbox API. Expected array, got object',
{
statusCode: result.metadata.status,
statusText: result.metadata.statusText,
code: 'UNKNOWN',
headers: result.metadata.headers,
body: result.data,
}
);
}
return result.data.data;
})
);
const allItems = batchResults.flat();
for (const item of allItems) {
const download: DebridDownload = {
id: -1,
hash: item.hash,
status: 'cached',
size: item.size,
};
// cachedResults.push(download);
newResults.push(download);
if (item.hash) {
TorboxDebridService.instantAvailabilityCache.set(
getSimpleTextHash(item.hash),
download,
Env.BUILTIN_DEBRID_INSTANT_AVAILABILITY_CACHE_TTL
);
}
}
return [...cachedResults, ...newResults];
}
return cachedResults;
// const data = await this.torboxApi.usenet.getUsenetCachedAvailability(
// this.apiVersion,
// {
// hash: hashes.join(','),
// format: 'list',
// }
// );
// if (!data.data?.success) {
// throw new DebridError(`Failed to check instant availability`, {
// statusCode: data.metadata.status,
// statusText: data.metadata.statusText,
// code: 'UNKNOWN',
// headers: data.metadata.headers,
// body: data.data,
// cause: data.data,
// type: 'api_error',
// });
// }
// if (!Array.isArray(data.data.data)) {
// throw new DebridError(
// 'Invalid response from Torbox API. Expected array, got object',
// {
// statusCode: data.metadata.status,
// statusText: data.metadata.statusText,
// code: 'UNKNOWN',
// headers: data.metadata.headers,
// body: data.data,
// cause: data.data,
// type: 'api_error',
// }
// );
// }
// return data.data.data.map((item) => ({
// id: -1,
// status: 'cached',
// name: item.name,
// size: item.size,
// }));
}
public async addNzb(nzb: string, name: string): Promise<DebridDownload> {
const res = await this.torboxApi.usenet.createUsenetDownload(
this.apiVersion,
{
link: nzb,
name,
}
);
if (!res.data?.data?.usenetdownloadId) {
throw new DebridError(`Usenet download failed: ${res.data?.detail}`, {
statusCode: res.metadata.status,
statusText: res.metadata.statusText,
code: 'UNKNOWN',
headers: res.metadata.headers,
body: res.data,
cause: res.data,
type: 'api_error',
});
}
const nzbInfo = await this.torboxApi.usenet.getUsenetList(this.apiVersion, {
id: res.data.data.usenetdownloadId.toString(),
});
if (
!nzbInfo?.data?.data ||
nzbInfo?.data?.error ||
nzbInfo.data.success === false
) {
throw new DebridError(
`Failed to get usenet list: ${nzbInfo?.data?.error || 'Unknown error'}${nzbInfo?.data?.detail ? '- ' + nzbInfo.data.detail : ''}`,
{
statusCode: nzbInfo.metadata.status,
statusText: nzbInfo.metadata.statusText,
code: 'UNKNOWN',
headers: nzbInfo.metadata.headers,
body: nzbInfo.data,
cause: nzbInfo.data,
type: 'api_error',
}
);
}
if (Array.isArray(nzbInfo.data.data)) {
throw new DebridError('Unexpected response format for usenet download', {
statusCode: nzbInfo.metadata.status,
statusText: nzbInfo.metadata.statusText,
code: 'UNKNOWN',
headers: nzbInfo.metadata.headers,
body: nzbInfo.data,
cause: nzbInfo.data,
type: 'api_error',
});
}
const usenetDownload = nzbInfo.data.data;
let status: DebridDownload['status'] = 'queued';
if (usenetDownload.downloadFinished && usenetDownload.downloadPresent) {
status = 'downloaded';
} else if (usenetDownload.progress && usenetDownload.progress > 0) {
status = 'downloading';
}
return {
id: usenetDownload.id ?? res.data.data.usenetdownloadId,
hash: usenetDownload.hash ?? res.data.data.hash,
name: usenetDownload.name ?? undefined,
status,
files: (usenetDownload.files ?? []).map((file) => ({
id: file.id,
mimeType: file.mimetype,
name: file.shortName ?? file.name ?? '',
size: file.size ?? 0,
})),
};
}
public async generateUsenetLink(
downloadId: string,
fileId?: string,
clientIp?: string
): Promise<string> {
const link = await this.torboxApi.usenet.requestDownloadLink(
this.apiVersion,
{
usenetId: downloadId,
fileId: fileId,
userIp: clientIp,
redirect: 'false',
token: this.config.token,
}
);
if (!link.data?.data) {
throw new DebridError('Failed to generate usenet download link', {
statusCode: link.metadata.status,
statusText: link.metadata.statusText,
code: 'UNKNOWN',
headers: link.metadata.headers,
body: link.data,
cause: link.data,
type: 'api_error',
});
}
return link.data.data;
}
public async resolve(
playbackInfo: PlaybackInfo,
filename: string
): Promise<string | undefined> {
if (playbackInfo.type === 'torrent') {
return this.stremthru.resolve(playbackInfo, filename);
}
const { nzb, file: chosenFile, metadata, title, hash } = playbackInfo;
const cacheKey = `${this.serviceName}:${this.config.token}:${this.config.clientIp}:${JSON.stringify(playbackInfo)}`;
const cachedLink =
await TorboxDebridService.playbackLinkCache.get(cacheKey);
if (cachedLink) {
logger.debug(`Using cached link for ${nzb}`);
return cachedLink;
}
logger.debug(`Adding usenet download for ${nzb}`, {
hash,
});
const usenetDownload = await this.addNzb(nzb, filename);
logger.debug(`Usenet download added for ${nzb}`, {
status: usenetDownload.status,
id: usenetDownload.id,
});
if (usenetDownload.status !== 'downloaded') {
return undefined;
}
if (!usenetDownload.files?.length) {
throw new DebridError('No files found for usenet download', {
statusCode: 400,
statusText: 'No files found for usenet download',
code: 'NO_MATCHING_FILE',
headers: {},
body: usenetDownload,
type: 'api_error',
});
}
let fileId: number | undefined;
if (usenetDownload.files.length > 1) {
const nzbInfo = {
type: 'usenet' as const,
nzb: nzb,
hash: hash,
title: title || usenetDownload.name,
file: chosenFile,
metadata: metadata,
size: usenetDownload.size || 0,
};
const allStrings: string[] = [];
allStrings.push(usenetDownload.name ?? '');
allStrings.push(...usenetDownload.files.map((file) => file.name ?? ''));
const parseResults = await PTT.parse(allStrings);
const parsedFiles = new Map<string, ParseResult>();
for (const [index, result] of parseResults.entries()) {
if (result) {
parsedFiles.set(allStrings[index], result);
}
}
const file = await selectFileInTorrentOrNZB(
nzbInfo,
usenetDownload,
parsedFiles,
metadata,
{
chosenFilename: chosenFile?.name,
chosenIndex: chosenFile?.index,
}
);
if (!file) {
throw new DebridError('No matching file found', {
statusCode: 400,
statusText: 'No matching file found',
code: 'NO_MATCHING_FILE',
headers: {},
body: file,
type: 'api_error',
});
}
logger.debug(`Found matching file`, {
chosenFile: file.name,
chosenIndex: file.id,
availableFiles: `[${usenetDownload.files.map((file) => file.name).join(', ')}]`,
});
fileId = file.id;
}
const playbackLink = await this.generateUsenetLink(
usenetDownload.id.toString(),
fileId?.toString(),
this.config.clientIp
);
await TorboxDebridService.playbackLinkCache.set(
cacheKey,
playbackLink,
Env.BUILTIN_DEBRID_INSTANT_AVAILABILITY_CACHE_TTL
);
return playbackLink;
}
}
+230 -79
View File
@@ -1,100 +1,247 @@
import { ParsedFile } from '../db/schemas';
import { createLogger } from '../utils';
import { z } from 'zod';
import { constants, createLogger } from '../utils';
import { BuiltinServiceId } from '../utils';
import { DebridFile, DebridDownload } from './base';
import { PTT } from '../parser';
import { normaliseTitle, titleMatch } from '../parser/utils';
const logger = createLogger('debrid');
interface FileWithParsedInfo {
name: string;
export const BuiltinDebridServices = z.array(
z.object({
id: z.enum(constants.BUILTIN_SUPPORTED_SERVICES),
credential: z.string(),
})
);
export type BuiltinDebridServices = z.infer<typeof BuiltinDebridServices>;
interface BaseFile {
title?: string;
size: number;
index: number;
parsed: ParsedFile;
isVideo: boolean;
path?: string;
link?: string;
index?: number;
indexer?: string;
seeders?: number;
age?: string;
}
export function findMatchingFileInTorrent(
files: FileWithParsedInfo[],
chosenIndex?: number,
requestedFilename?: string,
requestedTitles?: string[],
season?: string,
episode?: string,
absoluteEpisode?: string,
expectLinks?: boolean
): FileWithParsedInfo | null {
let chosenFile = null;
export interface Torrent extends BaseFile {
type: 'torrent';
downloadUrl?: string;
sources: string[];
hash: string;
files?: DebridFile[];
// magnet?: string;
}
for (const file of files) {
if (!file.isVideo || file.name?.includes('sample')) {
continue;
export interface UnprocessedTorrent extends BaseFile {
type: 'torrent';
hash?: string;
downloadUrl?: string;
sources: string[];
}
export interface NZB extends BaseFile {
type: 'usenet';
hash: string;
nzb: string;
}
export interface TorrentWithSelectedFile extends Torrent {
file: DebridFile;
service?: {
id: BuiltinServiceId;
cached: boolean;
owned: boolean;
};
}
export interface NZBWithSelectedFile extends NZB {
file: DebridFile;
service?: {
id: BuiltinServiceId;
cached: boolean;
owned: boolean;
};
}
// helpers
export const isSeasonWrong = (
parsed: { seasons?: number[] },
metadata?: { season?: number }
) => {
if (
parsed.seasons?.length &&
metadata?.season &&
!parsed.seasons.includes(metadata.season)
) {
return true;
}
return false;
};
export const isEpisodeWrong = (
parsed: { episodes?: number[] },
metadata?: { episode?: number; absoluteEpisode?: number }
) => {
if (
parsed.episodes?.length &&
metadata?.episode &&
!(
parsed.episodes.includes(metadata.episode) ||
(metadata.absoluteEpisode &&
parsed.episodes.includes(metadata.absoluteEpisode))
)
) {
return true;
}
return false;
};
export const isTitleWrong = (
parsed: { title?: string },
metadata?: { titles?: string[] }
) => {
if (
parsed.title &&
metadata?.titles &&
!titleMatch(
normaliseTitle(parsed.title),
metadata.titles.map(normaliseTitle),
{ threshold: 0.8 }
)
) {
return true;
}
return false;
};
export async function selectFileInTorrentOrNZB(
torrentOrNZB: Torrent | NZB,
debridDownload: DebridDownload,
parsedFiles: Map<
string,
{
title?: string;
seasons?: number[];
episodes?: number[];
}
>,
metadata?: {
titles: string[];
season?: number;
episode?: number;
absoluteEpisode?: number;
},
options?: {
chosenFilename?: string;
chosenIndex?: number;
}
): Promise<DebridFile | undefined> {
if (!debridDownload.files?.length) {
return {
name: torrentOrNZB.title,
size: torrentOrNZB.size,
index: -1,
};
}
const isVideo = debridDownload.files.map((file) => isVideoFile(file));
// Create a scoring system for each file
const fileScores = debridDownload.files.map((file, index) => {
let score = 0;
const parsed = parsedFiles.get(file.name ?? '');
if (!parsed) {
logger.warn(`Parsed file not found for ${file.name}`);
}
if (season || episode || absoluteEpisode) {
if (
(requestedTitles && file.parsed.title
? requestedTitles.some(
(title) =>
file.parsed.title!.toLowerCase() === title.toLowerCase()
)
: true) &&
((absoluteEpisode &&
!file.parsed.season &&
file.parsed.episode === Number(absoluteEpisode)) ||
(season && !episode && file.parsed.season === Number(season)) ||
(episode && !season && file.parsed.episode === Number(episode)) ||
(season &&
episode &&
file.parsed.season === Number(season) &&
file.parsed.episode === Number(episode)))
) {
chosenFile = file;
break;
}
} else {
if (requestedTitles) {
if (
requestedTitles.some(
(title) => file.parsed.title?.toLowerCase() === title.toLowerCase()
)
) {
chosenFile = file;
break;
}
}
// Base score from video file status (highest priority)
if (isVideo[index]) {
score += 1000;
}
}
if (chosenFile) {
return chosenFile;
}
if (requestedFilename) {
const requestedFile = files.find(
(file) => file.name.toLowerCase() === requestedFilename.toLowerCase()
);
if (requestedFile) {
return requestedFile;
// Season/Episode matching (second highest priority)
if (parsed && !isSeasonWrong(parsed, metadata)) {
score += 500;
}
if (parsed && !isEpisodeWrong(parsed, metadata)) {
score += 500;
}
}
if (chosenIndex) {
const fileIdx = files.find((file) => file.index === chosenIndex);
if (fileIdx) return fileIdx;
}
// Title matching (third priority)
if (parsed && !isTitleWrong(parsed, metadata)) {
score += 100;
}
if (files.length > 0 && (!expectLinks || files.some((file) => file.link))) {
return files
.filter((file) => !expectLinks || file.link)
.reduce((largest, current) =>
current.size > largest.size ? current : largest
// Size based score (lowest priority but still relevant)
// We normalize the size to be between 0 and 50 points
const files = debridDownload.files || [];
const maxSize =
torrentOrNZB.size || files.reduce((max, f) => Math.max(max, f.size), 0);
score += maxSize > 0 ? (file.size / maxSize) * 50 : 0;
// Small boost for chosen index/filename if provided
if (options?.chosenIndex === index) {
score += 25;
}
if (
options?.chosenFilename &&
torrentOrNZB.title?.includes(options.chosenFilename)
) {
score += 25;
}
return {
file,
score,
index,
};
});
// Sort by score descending
fileScores.sort((a, b) => b.score - a.score);
// Select the best matching file
const bestMatch = fileScores[0];
// return bestMatch.file;
const parsedFile = parsedFiles.get(bestMatch.file.name ?? '');
const parsedTitle = parsedFiles.get(torrentOrNZB.title ?? '');
if (metadata && parsedFile && parsedTitle) {
// if (
// !isSeasonWrong(parsed, metadata) &&
// !isSeasonWrong(parsedTorrentOrNZB, metadata)
// ) {
// logger.debug(
// `Season ${metadata.season} not found in ${torrentOrNZB.title} and ${bestMatch.file.name}, skipping...`
// );
// return undefined;
// }
if (
isEpisodeWrong(parsedFile, metadata) ||
isEpisodeWrong(parsedTitle, metadata)
) {
logger.debug(
`Episode ${metadata.episode} or ${metadata.absoluteEpisode} not found in ${torrentOrNZB.title} and ${bestMatch.file.name}, skipping...`
);
return undefined;
}
// if (
// !titleMatchHelper(parsed, metadata) &&
// !titleMatchHelper(parsedTorrentOrNZB, metadata)
// ) {
// logger.debug(
// `Title ${torrentOrNZB.title} and ${bestMatch.file.name} does not match ${metadata.titles.join(', ')}, skipping...`
// );
// return undefined;
// }
}
return null;
return bestMatch.file;
}
export function isVideoFile(filename: string): boolean {
export function isVideoFile(file: DebridFile): boolean {
const videoExtensions = [
'.3g2',
'.3gp',
@@ -136,5 +283,9 @@ export function isVideoFile(filename: string): boolean {
'.m3u8',
'.m2ts',
];
return videoExtensions.some((ext) => filename.endsWith(ext));
return (
file.mimeType?.includes('video') ||
videoExtensions.some((ext) => file.name?.endsWith(ext) ?? false)
);
}
+3
View File
@@ -11,5 +11,8 @@ export {
GoogleOAuth,
GDriveAPI,
TorBoxSearchAddonError,
TorznabAddon,
NewznabAddon,
ProwlarrAddon,
} from './builtins';
export { PresetManager } from './presets';
+7 -7
View File
@@ -15,6 +15,7 @@ import {
Cache,
ExtrasParser,
makeUrlLogSafe,
AnimeDatabase,
} from './utils';
import { Wrapper } from './wrapper';
import { PresetManager } from './presets';
@@ -40,7 +41,8 @@ import {
StreamUtils,
} from './streams';
import { getAddonName } from './utils/general';
import { TMDBMetadata, TMDBMetadataResponse } from './metadata/tmdb';
import { TMDBMetadata } from './metadata/tmdb';
import { Metadata } from './metadata/utils';
const logger = createLogger('core');
const shuffleCache = Cache.getInstance<string, MetaPreview[]>('shuffle');
@@ -1163,9 +1165,7 @@ export class AIOStreams {
});
}
private async getMetadata(
id: string
): Promise<TMDBMetadataResponse | undefined> {
private async getMetadata(id: string): Promise<Metadata | undefined> {
try {
const metadata = await new TMDBMetadata({
accessToken: this.userData.tmdbAccessToken,
@@ -1186,7 +1186,7 @@ export class AIOStreams {
private _getNextEpisode(
currentSeason: number,
currentEpisode: number,
metadata?: TMDBMetadataResponse
metadata?: Metadata
): {
season: number;
episode: number;
@@ -1194,7 +1194,7 @@ export class AIOStreams {
let season = currentSeason;
let episode = currentEpisode + 1;
const episodeCount = metadata?.seasons?.find(
(s) => s.season_number === Number(season)
(s) => s.season_number === season
)?.episode_count;
// If we are at the last episode of the season, try to move to the next season
@@ -1237,7 +1237,7 @@ export class AIOStreams {
await this.limiter.limit(
await this.sorter.sort(
processedStreams,
id.startsWith('kitsu') ? 'anime' : type
AnimeDatabase.getInstance().isAnime(id) ? 'anime' : type
)
)
)
+31 -21
View File
@@ -2,6 +2,7 @@ import { z } from 'zod';
import { Cache, makeRequest, Env, TYPES } from '../utils';
import { Wrapper } from '../wrapper';
import { Metadata } from './utils';
import { Meta, MetaSchema } from '../db';
const IMDBSuggestionSchema = z.object({
d: z.array(
@@ -27,12 +28,15 @@ const IMDBSuggestionSchema = z.object({
export class IMDBMetadata {
private readonly titleCache: Cache<string, Metadata>;
private readonly cinemetaCache: Cache<string, Meta>;
private readonly titleCacheTTL = 7 * 24 * 60 * 60;
private readonly cinemetaCacheTTL = 7 * 24 * 60 * 60;
private readonly IMDB_SUGGESTION_API =
'https://v3.sg.media-imdb.com/suggestion/a/';
private readonly CINEMETA_URL = 'https://v3-cinemeta.strem.io/manifest.json';
private readonly CINEMETA_URL = 'https://v3-cinemeta.strem.io';
public constructor() {
this.titleCache = Cache.getInstance('imdb-title');
this.cinemetaCache = Cache.getInstance('cinemeta');
}
public async getTitleAndYear(id: string, type: string): Promise<Metadata> {
@@ -47,9 +51,18 @@ export class IMDBMetadata {
if (!cinemetaData.name || !cinemetaData.year) {
throw new Error('Cinemeta data is missing title or year');
}
let year = Number(cinemetaData.releaseInfo?.toString().split('-')[0]);
let yearEnd = Number(cinemetaData.releaseInfo?.toString().split('-')[1]);
if (isNaN(yearEnd)) {
yearEnd = new Date().getFullYear();
}
if (isNaN(year) && Number.isInteger(Number(cinemetaData.year))) {
year = Number(cinemetaData.year);
}
return {
title: cinemetaData.name,
year: Number(cinemetaData.releaseInfo?.toString().split('-')[0]),
year,
yearEnd,
};
}
}
@@ -78,29 +91,26 @@ export class IMDBMetadata {
}
const title = item.l;
const year = item.y;
this.titleCache.set(key, { title, year }, this.titleCacheTTL);
return { title, year };
let yearEnd: number | undefined = undefined;
const yearString = item.yr;
if (yearString && yearString.includes('-')) {
yearEnd = Number(yearString.split('-')[1]);
}
this.titleCache.set(key, { title, year, yearEnd }, this.titleCacheTTL);
return { title, year, yearEnd };
}
private async getCinemetaData(id: string, type: string) {
const cinemeta = new Wrapper({
instanceId: 'cinemeta',
preset: {
id: 'custom',
type: 'custom',
options: {
id: id,
},
},
manifestUrl: this.CINEMETA_URL,
name: 'Cinemeta',
public async getCinemetaData(id: string, type: string) {
const url = `${this.CINEMETA_URL}/meta/${type}/${id}.json`;
const cached = await this.cinemetaCache.get(url);
if (cached) {
return cached;
}
const response = await makeRequest(url, {
timeout: 1000,
enabled: true,
headers: {
'Content-Type': 'application/json',
},
});
const meta = await cinemeta.getMeta(type, id);
const meta = MetaSchema.parse(((await response.json()) as any).meta);
this.cinemetaCache.set(url, meta, this.cinemetaCacheTTL);
return meta;
}
}
-79
View File
@@ -1,79 +0,0 @@
import { AnimeKitsuPreset } from '../presets/animeKitsu';
import { Cache } from '../utils';
import { Wrapper } from '../wrapper';
import { Metadata } from './utils';
import { IdParser, ParsedId } from '../builtins/utils/id-parser';
import z from 'zod';
// kitsu, mal, anilist, anidb
export class KitsuMetadata {
public constructor() {}
public async getMetadata(id: ParsedId, type: string): Promise<Metadata> {
const [kitsuAddon] = await AnimeKitsuPreset.generateAddons(
{
sortCriteria: { global: [] },
formatter: { id: 'gdrive' },
presets: [],
},
{}
);
if (!['kitsu_id', 'mal_id', 'anilist_id', 'anidb_id'].includes(id.type)) {
throw new Error('Invalid ID type');
}
const meta = await new Wrapper(kitsuAddon).getMeta(
type,
`${id.type.replace('_id', '')}:${id.id}`
);
if (!meta.name || !meta.releaseInfo) {
throw new Error('Kitsu metadata is missing title or year');
}
let season: number | undefined = undefined;
if (meta.videos?.[0]?.imdbSeason) {
season = Number(meta.videos[0].imdbSeason);
}
if (season === undefined) {
const aliasesParse = z.array(z.string()).safeParse((meta as any).aliases);
if (aliasesParse.success) {
const found = aliasesParse.data
.map((alias) => alias.match(/Season (\d+)/i))
.filter((match) => match !== null)
.map((match) => match![1])
.map(Number)
.find((s) => !isNaN(s));
if (found !== undefined) {
season = found;
}
}
}
if (season === undefined && z.string().safeParse(meta.slug).success) {
const seasonMatch = z
.string()
.parse(meta.slug)
.match(/-(\d+)$/);
if (seasonMatch) {
season = Number(seasonMatch[1]);
}
}
return {
title: meta.name,
titles: [meta.name, ...(meta.aliases ? (meta.aliases as string[]) : [])],
year: Number(meta.releaseInfo?.toString().split('-')[0]),
seasons: season
? [
{
season_number: season,
episode_count: meta.videos?.length || 0,
},
]
: undefined,
};
}
}
+214
View File
@@ -0,0 +1,214 @@
import { DistributedLock } from '../utils/distributed-lock';
import { Metadata } from './utils';
import { TMDBMetadata } from './tmdb';
import { getTraktAliases } from './trakt';
import { IMDBMetadata } from './imdb';
import { createLogger, getTimeTakenSincePoint } from '../utils/logger';
import { TYPES } from '../utils/constants';
import { AnimeDatabase, ParsedId } from '../utils';
import { Meta } from '../db/schemas';
const logger = createLogger('metadata-service');
export interface MetadataServiceConfig {
tmdbAccessToken?: string;
tmdbApiKey?: string;
}
export class MetadataService {
private readonly tmdbMetadata: TMDBMetadata;
private readonly lock: DistributedLock;
public constructor(config: MetadataServiceConfig) {
this.tmdbMetadata = new TMDBMetadata({
accessToken: config.tmdbAccessToken,
apiKey: config.tmdbApiKey,
});
this.lock = DistributedLock.getInstance();
}
public async getMetadata(
id: ParsedId,
type: (typeof TYPES)[number]
): Promise<Metadata> {
const { result } = await this.lock.withLock(
`metadata:${id.mediaType}:${id.type}:${id.value}`,
async () => {
const start = Date.now();
const titles: string[] = [];
let year: number | undefined;
let yearEnd: number | undefined;
let seasons:
| {
season_number: number;
episode_count: number;
}[]
| undefined;
// Check anime database first
const animeEntry = AnimeDatabase.getInstance().getEntryById(
id.type,
id.value
);
const tmdbId =
id.type === 'themoviedbId'
? id.value.toString()
: (animeEntry?.mappings?.themoviedbId?.toString() ?? null);
const imdbId =
id.type === 'imdbId'
? id.value.toString()
: (animeEntry?.mappings?.imdbId?.toString() ?? null);
const tvdbId =
id.type === 'thetvdbId'
? id.value.toString()
: (animeEntry?.mappings?.thetvdbId?.toString() ?? null);
if (animeEntry) {
if (animeEntry.imdb?.title) titles.push(animeEntry.imdb.title);
if (animeEntry.trakt?.title) titles.push(animeEntry.trakt.title);
if (animeEntry.title) titles.push(animeEntry.title);
if (animeEntry.synonyms) titles.push(...animeEntry.synonyms);
year = animeEntry.animeSeason?.year ?? undefined;
}
// Setup parallel API requests
const promises = [];
// TMDB metadata
if (tmdbId || imdbId || tvdbId) {
let id = tmdbId
? `tmdb:${tmdbId}`
: (imdbId ?? (tvdbId ? `tvdb:${tvdbId}` : null));
promises.push(this.tmdbMetadata.getMetadata(id!, type));
} else {
promises.push(Promise.resolve(undefined));
}
// Trakt aliases
if (imdbId) {
promises.push(getTraktAliases(id));
} else {
promises.push(Promise.resolve(undefined));
}
// IMDb metadata
if (imdbId) {
promises.push(new IMDBMetadata().getCinemetaData(imdbId, type));
} else {
promises.push(Promise.resolve(undefined));
}
// Execute all promises in parallel
const [tmdbResult, traktResult, imdbResult] = (await Promise.allSettled(
promises
)) as [
PromiseSettledResult<Metadata | undefined>,
PromiseSettledResult<string[] | undefined>,
PromiseSettledResult<Meta | undefined>,
];
// Process TMDB results
if (tmdbResult.status === 'fulfilled' && tmdbResult.value) {
const tmdbMetadata = tmdbResult.value;
if (tmdbMetadata.title) titles.unshift(tmdbMetadata.title);
if (tmdbMetadata.titles) titles.push(...tmdbMetadata.titles);
if (!year && tmdbMetadata.year) year = tmdbMetadata.year;
if (tmdbMetadata.yearEnd) yearEnd = tmdbMetadata.yearEnd;
if (tmdbMetadata.seasons)
seasons = tmdbMetadata.seasons.sort(
(a, b) => a.season_number - b.season_number
);
} else if (tmdbResult.status === 'rejected') {
logger.warn(
`Failed to fetch TMDB metadata for ${id.fullId}: ${tmdbResult.reason}`
);
}
// Process Trakt results
if (traktResult.status === 'fulfilled' && traktResult.value) {
titles.push(...traktResult.value);
} else if (traktResult.status === 'rejected') {
logger.warn(
`Failed to fetch Trakt aliases for ${id.fullId}: ${traktResult.reason}`
);
}
// Process IMDb results
if (imdbResult.status === 'fulfilled' && imdbResult.value) {
const cinemetaData = imdbResult.value;
if (cinemetaData.name) titles.unshift(cinemetaData.name);
if (cinemetaData.releaseInfo && !year) {
const releaseYear = Number(
cinemetaData.releaseInfo.toString().split('-')[0]
);
if (!isNaN(releaseYear)) year = releaseYear;
}
if (cinemetaData.videos) {
const seasonMap = new Map<number, Set<number>>();
for (const video of cinemetaData.videos) {
if (
typeof video.season === 'number' &&
typeof video.episode === 'number'
) {
if (!seasonMap.has(video.season)) {
seasonMap.set(video.season, new Set());
}
seasonMap.get(video.season)!.add(video.episode);
}
}
const imdbSeasons = Array.from(seasonMap.entries()).map(
([season_number, episodes]) => ({
season_number,
episode_count: episodes.size,
})
);
if (imdbSeasons.length) {
seasons = imdbSeasons.sort(
(a, b) => a.season_number - b.season_number
);
}
}
} else if (imdbResult.status === 'rejected') {
logger.warn(
`Failed to fetch IMDb metadata for ${imdbId}: ${imdbResult.reason}`
);
}
// Deduplicate titles, lowercase all before deduplication
const uniqueTitles = [
...new Set(titles.map((title) => title.toLowerCase())),
];
if (!uniqueTitles.length || !year) {
throw new Error(`Could not find metadata for ${id.fullId}`);
}
logger.debug(
`Found metadata for ${id.fullId} in ${getTimeTakenSincePoint(start)}`,
{
title: uniqueTitles[0],
aliases: uniqueTitles.slice(1).length,
year,
yearEnd,
seasons: seasons?.length,
}
);
return {
title: uniqueTitles[0],
titles: uniqueTitles,
year,
yearEnd,
seasons,
};
},
{
timeout: 2500, // metadata does not take long to fetch so keep it low
ttl: 5000,
retryInterval: 100,
}
);
return result;
}
}
+96 -26
View File
@@ -1,5 +1,8 @@
import { Headers } from 'undici';
import { Env, Cache, TYPES, makeRequest } from '../utils';
import { Metadata } from './utils';
import { z } from 'zod';
export type ExternalIdType = 'imdb' | 'tmdb' | 'tvdb';
interface ExternalId {
@@ -18,14 +21,56 @@ const ID_CACHE_TTL = 30 * 24 * 60 * 60; // 30 days
const TITLE_CACHE_TTL = 7 * 24 * 60 * 60; // 7 days
const AUTHORISATION_CACHE_TTL = 2 * 24 * 60 * 60; // 2 days
export interface TMDBMetadataResponse {
titles: string[];
year: string;
seasons?: {
season_number: number;
episode_count: number;
}[];
}
// Zod schemas for API responses
const MovieDetailsSchema = z.object({
id: z.number(),
title: z.string(),
release_date: z.string().optional(),
status: z.string(),
});
const TVDetailsSchema = z.object({
id: z.number(),
name: z.string(),
first_air_date: z.string().optional(),
last_air_date: z.string().optional(),
status: z.string(),
seasons: z.array(
z.object({
season_number: z.number(),
episode_count: z.number(),
})
),
});
const MovieAlternativeTitlesSchema = z.object({
titles: z.array(
z.object({
title: z.string(),
})
),
});
const TVAlternativeTitlesSchema = z.object({
results: z.array(
z.object({
title: z.string(),
})
),
});
const FindResultsSchema = z.object({
movie_results: z.array(
z.object({
id: z.number(),
})
),
tv_results: z.array(
z.object({
id: z.number(),
})
),
});
export class TMDBMetadata {
private readonly TMDB_ID_REGEX = /^(?:tmdb)[-:](\d+)(?::\d+:\d+)?$/;
@@ -35,8 +80,8 @@ export class TMDBMetadata {
string,
string
>('tmdb_id_conversion');
private static readonly metadataCache: Cache<string, TMDBMetadataResponse> =
Cache.getInstance<string, TMDBMetadataResponse>('tmdb_metadata');
private static readonly metadataCache: Cache<string, Metadata> =
Cache.getInstance<string, Metadata>('tmdb_metadata');
private readonly accessToken: string | undefined;
private readonly apiKey: string | undefined;
private static readonly validationCache: Cache<string, boolean> =
@@ -109,9 +154,9 @@ export class TMDBMetadata {
throw new Error(`${response.status} - ${response.statusText}`);
}
const data: any = await response.json();
const data = FindResultsSchema.parse(await response.json());
const results = type === 'movie' ? data.movie_results : data.tv_results;
const meta = results?.[0];
const meta = results[0];
if (!meta) {
throw new Error(`No ${type} metadata found for ID: ${id.value}`);
@@ -123,7 +168,8 @@ export class TMDBMetadata {
return tmdbId;
}
private parseReleaseDate(releaseDate: string): string {
private parseReleaseDate(releaseDate: string | undefined): string {
if (!releaseDate) return '0';
const date = new Date(releaseDate);
return date.getFullYear().toString();
}
@@ -131,7 +177,7 @@ export class TMDBMetadata {
public async getMetadata(
id: string,
type: (typeof TYPES)[number]
): Promise<TMDBMetadataResponse> {
): Promise<Metadata> {
if (!['movie', 'series', 'anime'].includes(type)) {
throw new Error(`Invalid type: ${type}`);
}
@@ -168,18 +214,32 @@ export class TMDBMetadata {
throw new Error(`Failed to fetch details: ${detailsResponse.statusText}`);
}
const detailsData: any = await detailsResponse.json();
const detailsJson = await detailsResponse.json();
const detailsData =
type === 'movie'
? MovieDetailsSchema.parse(detailsJson)
: TVDetailsSchema.parse(detailsJson);
const primaryTitle =
type === 'movie' ? detailsData.title : detailsData.name;
type === 'movie'
? (detailsData as z.infer<typeof MovieDetailsSchema>).title
: (detailsData as z.infer<typeof TVDetailsSchema>).name;
const year = this.parseReleaseDate(
type === 'movie' ? detailsData.release_date : detailsData.first_air_date
type === 'movie'
? (detailsData as z.infer<typeof MovieDetailsSchema>).release_date
: (detailsData as z.infer<typeof TVDetailsSchema>).first_air_date
);
const yearEnd =
type === 'series'
? (detailsData as z.infer<typeof TVDetailsSchema>).last_air_date
? this.parseReleaseDate(
(detailsData as z.infer<typeof TVDetailsSchema>).last_air_date
)
: undefined
: undefined;
const seasons =
type === 'series'
? detailsData.seasons.map((season: any) => ({
season_number: season.season_number,
episode_count: season.episode_count,
}))
? (detailsData as z.infer<typeof TVDetailsSchema>).seasons
: undefined;
// Fetch alternative titles
@@ -201,18 +261,28 @@ export class TMDBMetadata {
);
}
const altTitlesData: any = await altTitlesResponse.json();
const altTitlesJson = await altTitlesResponse.json();
const altTitlesData =
type === 'movie'
? MovieAlternativeTitlesSchema.parse(altTitlesJson)
: TVAlternativeTitlesSchema.parse(altTitlesJson);
const alternativeTitles =
type === 'movie'
? altTitlesData.titles.map((title: any) => title.title)
: altTitlesData.results.map((title: any) => title.title);
? (
altTitlesData as z.infer<typeof MovieAlternativeTitlesSchema>
).titles.map((title) => title.title)
: (
altTitlesData as z.infer<typeof TVAlternativeTitlesSchema>
).results.map((title) => title.title);
// Combine primary title with alternative titles, ensuring no duplicates
const allTitles = [primaryTitle, ...alternativeTitles];
const uniqueTitles = [...new Set(allTitles)];
const metadata: TMDBMetadataResponse = {
const metadata: Metadata = {
title: primaryTitle,
titles: uniqueTitles,
year,
year: Number(year),
yearEnd: yearEnd ? Number(yearEnd) : undefined,
seasons,
};
// Cache the result
+80
View File
@@ -0,0 +1,80 @@
import { z } from 'zod';
import {
AnimeDatabase,
Cache,
createLogger,
makeRequest,
ParsedId,
formatZodError,
Env,
} from '../utils';
const traktAliasCache = Cache.getInstance<string, string[]>('trakt-aliases');
const TRAKT_ALIAS_CACHE_TTL = 7 * 24 * 60 * 60; // 7 days
const TraktAliasSchema = z.array(
z.object({
title: z.string(),
country: z.string(),
})
);
const TRAKT_API_BASE_URL = 'https://api.trakt.tv';
const logger = createLogger('trakt');
export async function getTraktAliases(
parsedId: ParsedId
): Promise<string[] | null> {
const cacheKey = `${parsedId.type}:${parsedId.value}`;
const cachedAliases = await traktAliasCache.get(cacheKey);
if (cachedAliases) {
logger.debug(
`Retrieved ${cachedAliases.length} (cached) Trakt aliases for ${parsedId.value}`
);
return cachedAliases;
}
// need imdb id, trakt id requires authentication.
let imdbId = parsedId.type === 'imdbId' ? parsedId.value : null;
// try to get imdb ID from anime database
const animeEntry = AnimeDatabase.getInstance().getEntryById(
parsedId.type,
parsedId.value
);
imdbId = imdbId ?? animeEntry?.mappings?.imdbId?.toString() ?? null;
if (!imdbId) {
return null;
}
try {
const data = await makeRequest(
`${TRAKT_API_BASE_URL}/${parsedId.mediaType === 'movie' ? 'movies' : 'shows'}/${imdbId}/aliases`,
{
timeout: 1000,
headers: {
'Content-Type': 'application/json',
'trakt-api-version': '1',
'trakt-api-key': Env.TRAKT_CLIENT_ID,
},
}
);
const parsedData = TraktAliasSchema.safeParse(await data.json());
if (!parsedData.success) {
logger.error(
`Failed to parse Trakt aliases: ${formatZodError(parsedData.error)}`
);
return null;
}
const aliases = parsedData.data.map((alias) => alias.title);
traktAliasCache.set(cacheKey, aliases, TRAKT_ALIAS_CACHE_TTL);
logger.debug(
`Retrieved ${aliases.length} Trakt aliases for ${parsedId.value}`
);
return aliases;
} catch (error) {
logger.error(`Failed to retrieve Trakt aliases: ${error}`);
return null;
}
}
+1
View File
@@ -2,6 +2,7 @@ export interface Metadata {
title: string;
titles?: string[];
year: number;
yearEnd?: number;
seasons?: {
season_number: number;
episode_count: number;
+1 -1
View File
@@ -1,5 +1,5 @@
import { PARSE_REGEX } from './regex';
import { ParsedFile } from '../db';
import { ParsedFile } from '../db/schemas';
import ptt from './ptt';
function matchPattern(
+94
View File
@@ -0,0 +1,94 @@
import { ParseResult, PTTServer } from 'go-ptt';
import os from 'os';
import {
Cache,
createLogger,
Env,
getSimpleTextHash,
getTimeTakenSincePoint,
} from '../utils';
const logger = createLogger('parser');
const parseCache = Cache.getInstance<string, (ParseResult | null)[]>(
'parseCache',
undefined,
true
);
class PTT {
private static _pttServer: PTTServer | null = null;
private static _pttConfig: {
network: 'tcp' | 'unix';
address: string;
} =
os.platform() === 'win32'
? {
network: 'tcp',
address: `:${Env.PORT + 1}`,
}
: {
network: 'unix',
address: 'ptt.sock',
};
private constructor() {}
public static async initialise(): Promise<PTTServer> {
if (PTT._pttServer) {
return PTT._pttServer;
}
PTT._pttServer = new PTTServer(PTT._pttConfig);
await PTT._pttServer.start();
logger.debug('PTT server started');
return PTT._pttServer;
}
public static async cleanup(): Promise<void> {
await PTT._pttServer?.stop();
PTT._pttServer = null;
}
public static async parse(titles: string[]): Promise<(ParseResult | null)[]> {
const cacheKey = getSimpleTextHash(titles.join('|'));
const cached = await parseCache.get(cacheKey);
if (cached) {
return cached;
}
if (!PTT._pttServer) {
throw new Error('PTT server not running');
}
if (titles.length === 0) {
return [];
}
const parsedTitles: (ParseResult | null)[] = [];
const startTime = Date.now();
let results: ParseResult[] = [];
try {
results = await PTT._pttServer.parse({
torrent_titles: titles,
normalize: true,
});
} catch (error) {
logger.error(
`Error calling PTT server: ${error}, ${JSON.stringify((error as any).metadata)}`,
error
);
}
logger.debug(
`PTT server parsed ${titles.length} titles in ${getTimeTakenSincePoint(startTime)}`
);
titles.forEach((title, idx) => {
const result = results[idx];
if (result.err) {
logger.error(`Error parsing title ${title} (${idx}): ${result.err}`);
parsedTitles.push(null);
}
parsedTitles.push(result);
});
await parseCache.set(cacheKey, parsedTitles, 60 * 60 * 24);
return parsedTitles;
}
}
export default PTT;
+1
View File
@@ -1,2 +1,3 @@
export { default as FileParser } from './file';
export { default as StreamParser } from './streams';
export { default as PTT } from './go-ptt';
+3 -3
View File
@@ -55,7 +55,7 @@ export const PARSE_REGEX: PARSE_REGEX = {
WEBRip: createRegex('web[ .\\-_]?rip'),
HDRip: createRegex('hd[ .\\-_]?rip|web[ .\\-_]?dl[ .\\-_]?rip'),
'HC HD-Rip': createRegex('hc|hd[ .\\-_]?rip'),
DVDRip: createRegex('dvd[ .\\-_]?(rip|mux|r|full|5|9)'),
DVDRip: createRegex('dvd[ .\\-_]?(rip|mux|r|full|5|9)?'),
HDTV: createRegex(
'(hd|pd)tv|tv[ .\\-_]?rip|hdtv[ .\\-_]?rip|dsr(ip)?|sat[ .\\-_]?rip'
),
@@ -81,7 +81,7 @@ export const PARSE_REGEX: PARSE_REGEX = {
'(d(olby)?[ .\\-_]?d(igital)?[ .\\-_]?(p(lus)?|\\+)(?:[ .\\-_]?(5[ .\\-_]?1|7[ .\\-_]?1))?)|e[ .\\-_]?ac[ .\\-_]?3'
),
DD: createRegex(
'(d(olby)?[ .\\-_]?d(igital)?(?:[ .\\-_]?(5[ .\\-_]?1|7[ .\\-_]?1))?)|(?<!e[ .\\-_]?)ac[ .\\-_]?3'
'(d(olby)?[ .\\-_]?d(igital)?(?:[ .\\-_]?(5[ .\\-_]?1|7[ .\\-_]?1|2[ .\\-_]?0?))?)|(?<!e[ .\\-_]?)ac[ .\\-_]?3'
),
'DTS-HD MA': createRegex('dts[ .\\-_]?hd[ .\\-_]?ma'),
'DTS-HD': createRegex('dts[ .\\-_]?hd(?![ .\\-_]?ma)'),
@@ -93,7 +93,7 @@ export const PARSE_REGEX: PARSE_REGEX = {
FLAC: createRegex('flac(?:[ .\\-_]?(lossless|2\\.0|x[2-4]))?'),
},
audioChannels: {
'2.0': createRegex('(2[ .\\-_]?0)(ch)?'),
'2.0': createRegex('(d(olby)?[ .\\-_]?d(igital)?)?2[ .\\-_]?0?(ch)?'),
'5.1': createRegex(
'(d(olby)?[ .\\-_]?d(igital)?[ .\\-_]?(p(lus)?|\\+)?)?5[ .\\-_]?1(ch)?'
),
+37
View File
@@ -0,0 +1,37 @@
import { extract, FuzzballExtractOptions } from 'fuzzball';
import { createLogger } from '../utils';
const logger = createLogger('parser');
export function titleMatch(
parsedTitle: string,
titles: string[],
options: {
threshold: number;
} & Exclude<FuzzballExtractOptions, 'returnObjects'>
) {
const { threshold, ...extractOptions } = options;
const results = extract(parsedTitle, titles, {
...extractOptions,
returnObjects: true,
});
const highestScore =
results.reduce(
(max: number, result: { choice: string; score: number; key: number }) => {
return Math.max(max, result.score);
},
0
) / 100;
return highestScore >= threshold;
}
export function normaliseTitle(title: string) {
return title
.normalize('NFD')
.replace(/[\u0300-\u036f]/g, '')
.replace(/[^\p{L}\p{N}+]/gu, '')
.toLowerCase();
}
+1 -1
View File
@@ -325,7 +325,7 @@ export class AICompanionPreset extends Preset {
return result.manifest_url;
} catch (error) {
throw new Error(
`Failed to generate manifest URL for AI Search: ${error}`
`Failed to generate manifest URL for AI Companion: ${error}`
);
}
}
+70
View File
@@ -0,0 +1,70 @@
import { Option, UserData } from '../db';
import { Env, constants } from '../utils';
import { baseOptions } from './preset';
import { StremThruPreset } from './stremthru';
import { TorznabPreset } from './torznab';
export class AnimeToshoPreset extends TorznabPreset {
static override get METADATA() {
const supportedResources = [constants.STREAM_RESOURCE];
const options: Option[] = [
...baseOptions(
'AnimeTosho',
supportedResources,
Env.BUILTIN_ANIMETOSHO_TIMEOUT || Env.DEFAULT_TIMEOUT
).filter((option) => option.id !== 'url' && option.id !== 'resources'),
{
id: 'services',
name: 'Services',
description:
'Optionally override the services that are used. If not specified, then the services that are enabled and supported will be used.',
type: 'multi-select',
required: false,
showInNoobMode: false,
options: StremThruPreset.supportedServices.map((service) => ({
value: service,
label: constants.SERVICE_DETAILS[service].name,
})),
default: undefined,
emptyIsUndefined: true,
},
];
return {
ID: 'animetosho',
NAME: 'AnimeTosho',
LOGO: '/assets/animetosho_logo.png',
URL: Env.BUILTIN_ANIMETOSHO_URL,
TIMEOUT: Env.BUILTIN_ANIMETOSHO_TIMEOUT || Env.DEFAULT_TIMEOUT,
USER_AGENT: Env.DEFAULT_USER_AGENT,
SUPPORTED_SERVICES: StremThruPreset.supportedServices,
DESCRIPTION: 'Directly search AnimeTosho.',
OPTIONS: options,
SUPPORTED_STREAM_TYPES: [constants.DEBRID_STREAM_TYPE],
SUPPORTED_RESOURCES: supportedResources,
BUILTIN: true,
};
}
protected static override generateManifestUrl(
userData: UserData,
services: constants.ServiceId[],
options: Record<string, any>
): string {
const animetoshoUrl = this.METADATA.URL;
const config = {
url: animetoshoUrl,
apiPath: '/api',
tmdbAccessToken: userData.tmdbAccessToken,
tmdbApiKey: userData.tmdbApiKey,
services: services.map((service) => ({
id: service,
credential: this.getServiceCredential(service, userData),
})),
};
const configString = this.base64EncodeJSON(config);
return `${Env.INTERNAL_URL}/builtins/torznab/${configString}/manifest.json`;
}
}
+70
View File
@@ -0,0 +1,70 @@
import { ParsedStream, Stream, UserData } from '../db';
import { StreamParser } from '../parser';
import { ServiceId } from '../utils/constants';
import { Preset } from './preset';
import { stremthruSpecialCases } from './stremthru';
export class BuiltinStreamParser extends StreamParser {
override getFolder(stream: Stream): string | undefined {
if (!stream.description) {
return undefined;
}
const folderName = stream.description.split('\n')[0];
return folderName.trim() || undefined;
}
protected getError(
stream: Stream,
currentParsedStream: ParsedStream
): ParsedStream['error'] | undefined {
if (stream.name?.startsWith('[❌]')) {
return {
// title: stream.name.replace('[❌]', ''),
title: this.addon.name,
description: stream.description || 'Unknown error',
};
}
return undefined;
}
protected parseServiceData(
string: string
): ParsedStream['service'] | undefined {
return super.parseServiceData(string.replace('TorBox', ''));
}
protected getInLibrary(
stream: Stream,
currentParsedStream: ParsedStream
): boolean {
return stream.name?.includes('☁️') ?? false;
}
protected get ageRegex(): RegExp | undefined {
return this.getRegexForTextAfterEmojis(['🕒']);
}
protected getStreamType(
stream: Stream,
service: ParsedStream['service'],
currentParsedStream: ParsedStream
): ParsedStream['type'] {
return (stream as any).type === 'usenet' ? 'usenet' : 'debrid';
}
}
export class BuiltinAddonPreset extends Preset {
static override getParser(): typeof StreamParser {
return BuiltinStreamParser;
}
protected static getServiceCredential(
serviceId: ServiceId,
userData: UserData,
specialCases?: Partial<Record<ServiceId, (credentials: any) => any>>
) {
return super.getServiceCredential(serviceId, userData, {
...stremthruSpecialCases,
...specialCases,
});
}
}
+21 -11
View File
@@ -309,17 +309,18 @@ export class MoreLikeThisPreset extends Preset {
default: 'en',
},
{
id: 'streamButtonPlatform',
id: 'streamButtons',
name: 'Stream Button Platform',
description:
'The type of buttons to show as streams on movie/series pages.',
type: 'select',
"This addon provides a set of streams in Stremio that act as buttons for easier access. Disable any you don't need.",
type: 'multi-select',
options: [
{ label: 'Both', value: 'both' },
{ label: 'Web', value: 'web' },
{ label: 'Go to detail page', value: 'details' },
{ label: 'Show recommendations', value: 'recs' },
{ label: 'App', value: 'app' },
{ label: 'Web', value: 'web' },
],
default: 'both',
default: ['details', 'recs', 'app', 'web'],
},
{
id: 'includeTmdbCollection',
@@ -363,7 +364,7 @@ export class MoreLikeThisPreset extends Preset {
return {
ID: 'more-like-this',
NAME: 'More Like This',
LOGO: `https://raw.githubusercontent.com/rama1997/More-Like-This/refs/heads/main/assets/logo.jpg`,
LOGO: `https://raw.githubusercontent.com/rama1997/More-Like-This/refs/heads/main/assets/images/logo.jpg`,
URL: Env.MORE_LIKE_THIS_URL,
TIMEOUT: Env.DEFAULT_MORE_LIKE_THIS_TIMEOUT || Env.DEFAULT_TIMEOUT,
USER_AGENT:
@@ -421,8 +422,10 @@ export class MoreLikeThisPreset extends Preset {
const isKeyValid = (key: string): string =>
typeof key === 'string' && key.length > 0 ? 'true' : '';
const config = {
const streamButtons = options.streamButtons
? options.streamButtons.join(',')
: ['details', 'recs', 'app', 'web'];
const config: Record<string, string> = {
tmdbApiKey: tmdbApiKey ?? '',
validatedTmdbApiKey: isKeyValid(tmdbApiKey ?? ''),
includeTmdbCollection: options.includeTmdbCollection ? 'on' : '',
@@ -436,8 +439,13 @@ export class MoreLikeThisPreset extends Preset {
catalogOrder: 'TMDB,Trakt,Simkl,Gemini+AI,TasteDive,Watchmode',
metadataSource: options.metadataSource,
language: options.language,
streamButtonPlatform: options.streamButtonPlatform,
enableTitleSearching: options.enableTitleSearching ? 'on' : '',
streamDetailEnabled: streamButtons.includes('details') ? 'on' : '',
streamRecEnabled: streamButtons.includes('recs') ? 'on' : '',
streamAppEnabled: streamButtons.includes('app') ? 'on' : '',
streamWebEnabled: streamButtons.includes('web') ? 'on' : '',
combineCatalogs: options.combineCatalogs ? 'on' : '',
streamOrder: streamButtons,
rpdbApiKey: '',
validatedRpdbApiKey: 'true',
forCopy: 'true',
@@ -471,7 +479,9 @@ export class MoreLikeThisPreset extends Preset {
let manifestUrl = response.headers.get('Location');
if (response.status < 300 || response.status >= 400) {
throw new Error(`${response.status} - ${response.statusText}`);
throw new Error(
`${response.status} - ${response.statusText}: ${await response.text()}`
);
}
if (!manifestUrl) {
throw new Error('Manifest URL not present in redirect');
+148
View File
@@ -0,0 +1,148 @@
import { Addon, Option, Stream, UserData } from '../db';
import { Preset, baseOptions } from './preset';
import { Env, RESOURCES, ServiceId, constants } from '../utils';
import { BuiltinAddonPreset } from './builtin';
export class NewznabPreset extends BuiltinAddonPreset {
static override get METADATA() {
const supportedResources = [constants.STREAM_RESOURCE];
const options: Option[] = [
{
id: 'name',
name: 'Name',
description: 'What to call this addon',
type: 'string',
required: true,
default: 'Newznab',
},
{
id: 'newznabUrl',
name: 'Newznab URL',
description: 'Provide the URL to the Newznab endpoint ',
type: 'url',
required: true,
},
{
id: 'apiKey',
name: 'API Key',
description:
'The password for the Newznab API. This is used to authenticate with the Newznab endpoint.',
type: 'password',
required: false,
},
{
id: 'apiPath',
name: 'API Path',
description: 'The path to the Newznab API. Usually /api.',
type: 'string',
required: false,
default: '/api',
},
{
id: 'timeout',
name: 'Timeout',
description: 'The timeout for this addon',
type: 'number',
default: Env.DEFAULT_TIMEOUT,
constraints: {
min: Env.MIN_TIMEOUT,
max: Env.MAX_TIMEOUT,
},
},
{
id: 'forceQuerySearch',
name: 'Force Query Search',
description: 'Force the addon to use the query search parameter',
type: 'boolean',
required: false,
default: false,
},
];
return {
ID: 'newznab',
NAME: 'Newznab',
LOGO: '',
URL: `${Env.INTERNAL_URL}/builtins/newznab`,
TIMEOUT: Env.DEFAULT_TIMEOUT,
USER_AGENT: Env.DEFAULT_USER_AGENT,
SUPPORTED_SERVICES: [constants.TORBOX_SERVICE],
DESCRIPTION:
'Directly search a Newznab instance for results with your services.',
OPTIONS: options,
SUPPORTED_STREAM_TYPES: [constants.USENET_STREAM_TYPE],
SUPPORTED_RESOURCES: supportedResources,
BUILTIN: true,
};
}
static async generateAddons(
userData: UserData,
options: Record<string, any>
): Promise<Addon[]> {
const usableServices = this.getUsableServices(userData, options.services);
if (!usableServices || usableServices.length === 0) {
throw new Error(
`${this.METADATA.NAME} requires at least one usable service, but none were found. Please enable at least one of the following services: ${this.METADATA.SUPPORTED_SERVICES.join(
', '
)}`
);
}
return [
this.generateAddon(
userData,
options,
usableServices.map((service) => service.id)
),
];
}
private static generateAddon(
userData: UserData,
options: Record<string, any>,
services: ServiceId[]
): Addon {
return {
name: options.name || this.METADATA.NAME,
manifestUrl: this.generateManifestUrl(userData, services, options),
enabled: true,
library: options.libraryAddon ?? false,
resources: options.resources || undefined,
timeout: options.timeout || this.METADATA.TIMEOUT,
preset: {
id: '',
type: this.METADATA.ID,
options: options,
},
formatPassthrough:
options.formatPassthrough ?? options.streamPassthrough ?? false,
resultPassthrough: options.resultPassthrough ?? false,
forceToTop: options.forceToTop ?? false,
headers: {
'User-Agent': this.METADATA.USER_AGENT,
},
};
}
protected static generateManifestUrl(
userData: UserData,
services: ServiceId[],
options: Record<string, any>
) {
const config = {
url: options.newznabUrl,
apiPath: options.apiPath,
apiKey: options.apiKey,
tmdbAccessToken: userData.tmdbAccessToken,
tmdbApiKey: userData.tmdbApiKey,
forceQuerySearch: options.forceQuerySearch ?? false,
services: services.map((service) => ({
id: service,
credential: this.getServiceCredential(service, userData),
})),
};
const configString = this.base64EncodeJSON(config);
return `${this.METADATA.URL}/${configString}/manifest.json`;
}
}
+21 -1
View File
@@ -52,16 +52,27 @@ import { ContentDeepDivePreset } from './contentDeepDive';
import { AICompanionPreset } from './aiCompanion';
import { GoogleOAuth } from '../builtins/gdrive/api';
import { TorBoxSearchPreset } from './torboxSearch';
import { TorznabPreset } from './torznab';
import { AStreamPreset } from './aStream';
import { Env } from '../utils/env';
import { ZileanPreset } from './zilean';
import { AnimeToshoPreset } from './animetosho';
import { NewznabPreset } from './newznab';
import { ProwlarrPreset } from './prowlarr';
let PRESET_LIST: string[] = [
'custom',
'torznab',
'newznab',
'aiostreams',
'torrentio',
'comet',
'mediafusion',
'stremthruTorz',
'stremthruStore',
'animetosho',
'zilean',
Env.BUILTIN_PROWLARR_URL && Env.BUILTIN_PROWLARR_API_KEY ? 'prowlarr' : '',
'jackettio',
'peerflix',
'orion',
@@ -109,7 +120,6 @@ let PRESET_LIST: string[] = [
'ai-search',
'more-like-this',
'content-deep-dive',
'aiostreams',
].filter(Boolean);
export class PresetManager {
@@ -236,6 +246,16 @@ export class PresetManager {
return GDrivePreset;
case 'torbox-search':
return TorBoxSearchPreset;
case 'torznab':
return TorznabPreset;
case 'newznab':
return NewznabPreset;
case 'zilean':
return ZileanPreset;
case 'animetosho':
return AnimeToshoPreset;
case 'prowlarr':
return ProwlarrPreset;
default:
throw new Error(`Preset ${id} not found`);
}
+135
View File
@@ -0,0 +1,135 @@
import { Addon, Option, Stream, UserData } from '../db';
import { Preset, baseOptions } from './preset';
import { Env, RESOURCES, ServiceId, constants } from '../utils';
import { StremThruPreset } from './stremthru';
import { BuiltinAddonPreset } from './builtin';
export class ProwlarrPreset extends BuiltinAddonPreset {
static override get METADATA() {
const supportedResources = [constants.STREAM_RESOURCE];
const options: Option[] = [
{
id: 'name',
name: 'Name',
description: 'What to call this addon',
type: 'string',
required: true,
default: 'Prowlarr',
},
{
id: 'timeout',
name: 'Timeout',
description: 'The timeout for this addon',
type: 'number',
default: Env.DEFAULT_TIMEOUT,
constraints: {
min: Env.MIN_TIMEOUT,
max: Env.MAX_TIMEOUT,
},
},
{
id: 'services',
name: 'Services',
description:
'Optionally override the services that are used. If not specified, then the services that are enabled and supported will be used.',
type: 'multi-select',
required: false,
showInNoobMode: false,
options: StremThruPreset.supportedServices.map((service) => ({
value: service,
label: constants.SERVICE_DETAILS[service].name,
})),
default: undefined,
emptyIsUndefined: true,
},
];
return {
ID: 'prowlarr',
NAME: 'Prowlarr',
LOGO: 'https://raw.githubusercontent.com/Prowlarr/Prowlarr/refs/heads/develop/Logo/256.png',
URL: `${Env.INTERNAL_URL}/builtins/prowlarr`,
TIMEOUT: Env.DEFAULT_TIMEOUT,
USER_AGENT: Env.DEFAULT_USER_AGENT,
SUPPORTED_SERVICES: StremThruPreset.supportedServices,
DESCRIPTION:
'Directly search a Prowlarr instance for results with your services.',
OPTIONS: options,
SUPPORTED_STREAM_TYPES: [constants.DEBRID_STREAM_TYPE],
SUPPORTED_RESOURCES: supportedResources,
BUILTIN: true,
};
}
static async generateAddons(
userData: UserData,
options: Record<string, any>
): Promise<Addon[]> {
const usableServices = this.getUsableServices(userData, options.services);
if (
(!usableServices || usableServices.length === 0) &&
!options.enableP2P
) {
throw new Error(
`${this.METADATA.NAME} requires at least one usable service, but none were found. Please enable at least one of the following services: ${this.METADATA.SUPPORTED_SERVICES.join(
', '
)}`
);
}
return [
this.generateAddon(
userData,
options,
usableServices?.map((service) => service.id) || []
),
];
}
private static generateAddon(
userData: UserData,
options: Record<string, any>,
services: ServiceId[]
): Addon {
return {
name: options.name || this.METADATA.NAME,
manifestUrl: this.generateManifestUrl(userData, services, options),
enabled: true,
library: options.libraryAddon ?? false,
resources: options.resources || undefined,
timeout: options.timeout || this.METADATA.TIMEOUT,
preset: {
id: '',
type: this.METADATA.ID,
options: options,
},
formatPassthrough:
options.formatPassthrough ?? options.streamPassthrough ?? false,
resultPassthrough: options.resultPassthrough ?? false,
forceToTop: options.forceToTop ?? false,
headers: {
'User-Agent': this.METADATA.USER_AGENT,
},
};
}
protected static generateManifestUrl(
userData: UserData,
services: ServiceId[],
options: Record<string, any>
) {
const config = {
url: Env.BUILTIN_PROWLARR_URL,
apiKey: Env.BUILTIN_PROWLARR_API_KEY,
indexers: Env.BUILTIN_PROWLARR_INDEXERS || [],
tmdbAccessToken: userData.tmdbAccessToken,
tmdbApiKey: userData.tmdbApiKey,
services: services.map((service) => ({
id: service,
credential: this.getServiceCredential(service, userData),
})),
};
const configString = this.base64EncodeJSON(config);
return `${this.METADATA.URL}/${configString}/manifest.json`;
}
}
+13 -4
View File
@@ -3,6 +3,15 @@ import { StreamParser } from '../parser';
import { constants, ServiceId } from '../utils';
import { Preset } from './preset';
export const stremthruSpecialCases: Partial<
Record<ServiceId, (credentials: any) => any>
> = {
[constants.OFFCLOUD_SERVICE]: (credentials: any) =>
`${credentials.email}:${credentials.password}`,
[constants.PIKPAK_SERVICE]: (credentials: any) =>
`${credentials.email}:${credentials.password}`,
};
export class StremThruStreamParser extends StreamParser {
protected override getIndexer(
stream: Stream,
@@ -59,11 +68,11 @@ export class StremThruPreset extends Preset {
specialCases?: Partial<Record<ServiceId, (credentials: any) => any>>
) {
return super.getServiceCredential(serviceId, userData, {
[constants.OFFCLOUD_SERVICE]: (credentials: any) =>
`${credentials.email}:${credentials.password}`,
[constants.PIKPAK_SERVICE]: (credentials: any) =>
`${credentials.email}:${credentials.password}`,
...stremthruSpecialCases,
...specialCases,
});
}
}
export type StremThruServiceId =
(typeof StremThruPreset.supportedServices)[number];
+164
View File
@@ -0,0 +1,164 @@
import { Addon, Option, Stream, UserData } from '../db';
import { Preset, baseOptions } from './preset';
import { Env, RESOURCES, ServiceId, constants } from '../utils';
import { StremThruPreset } from './stremthru';
import { BuiltinAddonPreset } from './builtin';
export class TorznabPreset extends BuiltinAddonPreset {
static override get METADATA() {
const supportedResources = [constants.STREAM_RESOURCE];
const options: Option[] = [
{
id: 'name',
name: 'Name',
description: 'What to call this addon',
type: 'string',
required: true,
default: 'Torznab',
},
{
id: 'torznabUrl',
name: 'Torznab URL',
description: 'Provide the URL to the Torznab endpoint ',
type: 'url',
required: true,
},
{
id: 'apiKey',
name: 'API Key',
description:
'The password for the Torznab API. This is used to authenticate with the Torznab endpoint.',
type: 'password',
required: false,
},
{
id: 'apiPath',
name: 'API Path',
description: 'The path to the Torznab API. Usually /api.',
type: 'string',
required: false,
default: '/api',
},
{
id: 'timeout',
name: 'Timeout',
description: 'The timeout for this addon',
type: 'number',
default: Env.DEFAULT_TIMEOUT,
constraints: {
min: Env.MIN_TIMEOUT,
max: Env.MAX_TIMEOUT,
},
},
{
id: 'services',
name: 'Services',
description:
'Optionally override the services that are used. If not specified, then the services that are enabled and supported will be used.',
type: 'multi-select',
required: false,
showInNoobMode: false,
options: StremThruPreset.supportedServices.map((service) => ({
value: service,
label: constants.SERVICE_DETAILS[service].name,
})),
default: undefined,
emptyIsUndefined: true,
},
{
id: 'forceQuerySearch',
name: 'Force Query Search',
description: 'Force the addon to use the query search parameter',
type: 'boolean',
required: false,
default: false,
},
];
return {
ID: 'torznab',
NAME: 'Torznab',
LOGO: '',
URL: `${Env.INTERNAL_URL}/builtins/torznab`,
TIMEOUT: Env.DEFAULT_TIMEOUT,
USER_AGENT: Env.DEFAULT_USER_AGENT,
SUPPORTED_SERVICES: StremThruPreset.supportedServices,
DESCRIPTION:
'Directly search a Torznab instance for results with your services.',
OPTIONS: options,
SUPPORTED_STREAM_TYPES: [constants.DEBRID_STREAM_TYPE],
SUPPORTED_RESOURCES: supportedResources,
BUILTIN: true,
};
}
static async generateAddons(
userData: UserData,
options: Record<string, any>
): Promise<Addon[]> {
const usableServices = this.getUsableServices(userData, options.services);
if (!usableServices || usableServices.length === 0) {
throw new Error(
`${this.METADATA.NAME} requires at least one usable service, but none were found. Please enable at least one of the following services: ${this.METADATA.SUPPORTED_SERVICES.join(
', '
)}`
);
}
return [
this.generateAddon(
userData,
options,
usableServices?.map((service) => service.id) || []
),
];
}
private static generateAddon(
userData: UserData,
options: Record<string, any>,
services: ServiceId[]
): Addon {
return {
name: options.name || this.METADATA.NAME,
manifestUrl: this.generateManifestUrl(userData, services, options),
enabled: true,
library: options.libraryAddon ?? false,
resources: options.resources || undefined,
timeout: options.timeout || this.METADATA.TIMEOUT,
preset: {
id: '',
type: this.METADATA.ID,
options: options,
},
formatPassthrough:
options.formatPassthrough ?? options.streamPassthrough ?? false,
resultPassthrough: options.resultPassthrough ?? false,
forceToTop: options.forceToTop ?? false,
headers: {
'User-Agent': this.METADATA.USER_AGENT,
},
};
}
protected static generateManifestUrl(
userData: UserData,
services: ServiceId[],
options: Record<string, any>
) {
const config = {
url: options.torznabUrl,
apiPath: options.apiPath,
apiKey: options.apiKey,
tmdbAccessToken: userData.tmdbAccessToken,
tmdbApiKey: userData.tmdbApiKey,
forceQuerySearch: options.forceQuerySearch ?? false,
services: services.map((service) => ({
id: service,
credential: this.getServiceCredential(service, userData),
})),
};
const configString = this.base64EncodeJSON(config);
return `${this.METADATA.URL}/${configString}/manifest.json`;
}
}
+77
View File
@@ -0,0 +1,77 @@
import { Option, UserData } from '../db';
import { Env, constants } from '../utils';
import { baseOptions } from './preset';
import { StremThruPreset } from './stremthru';
import { TorznabPreset } from './torznab';
export class ZileanPreset extends TorznabPreset {
static override get METADATA() {
const supportedResources = [constants.STREAM_RESOURCE];
const options: Option[] = [
...baseOptions(
'Zilean',
supportedResources,
Env.BUILTIN_ZILEAN_TIMEOUT || Env.DEFAULT_TIMEOUT
).filter((option) => option.id !== 'url' && option.id !== 'resources'),
{
id: 'url',
name: 'URL',
description: 'Optionally override the URL of the Zilean instance',
type: 'url',
required: false,
},
{
id: 'services',
name: 'Services',
description:
'Optionally override the services that are used. If not specified, then the services that are enabled and supported will be used.',
type: 'multi-select',
required: false,
showInNoobMode: false,
options: StremThruPreset.supportedServices.map((service) => ({
value: service,
label: constants.SERVICE_DETAILS[service].name,
})),
default: undefined,
emptyIsUndefined: true,
},
];
return {
ID: 'zilean',
NAME: 'Zilean',
LOGO: '/assets/zilean_logo.jpg',
URL: Env.BUILTIN_ZILEAN_URL,
TIMEOUT: Env.BUILTIN_ZILEAN_TIMEOUT || Env.DEFAULT_TIMEOUT,
USER_AGENT: Env.DEFAULT_USER_AGENT,
SUPPORTED_SERVICES: StremThruPreset.supportedServices,
DESCRIPTION: 'Directly search a Zilean instance.',
OPTIONS: options,
SUPPORTED_STREAM_TYPES: [constants.DEBRID_STREAM_TYPE],
SUPPORTED_RESOURCES: supportedResources,
BUILTIN: true,
};
}
protected static override generateManifestUrl(
userData: UserData,
services: constants.ServiceId[],
options: Record<string, any>
): string {
const zileanUrl = (options.url || this.METADATA.URL).replace(/\/$/, '');
const config = {
url: `${zileanUrl}/torznab`,
apiPath: '/api',
tmdbAccessToken: userData.tmdbAccessToken,
tmdbApiKey: userData.tmdbApiKey,
services: services.map((service) => ({
id: service,
credential: this.getServiceCredential(service, userData),
})),
};
const configString = this.base64EncodeJSON(config);
return `${Env.INTERNAL_URL}/builtins/torznab/${configString}/manifest.json`;
}
}
+13 -4
View File
@@ -1,5 +1,10 @@
import { Addon, ParsedStream, UserData } from '../db/schemas';
import { constants, createLogger, getTimeTakenSincePoint } from '../utils';
import {
AnimeDatabase,
constants,
createLogger,
getTimeTakenSincePoint,
} from '../utils';
import { Wrapper } from '../wrapper';
import { GroupConditionEvaluator } from '../parser/streamExpression';
import { getAddonName } from '../utils/general';
@@ -46,6 +51,10 @@ class StreamFetcher {
}[] = [];
let allStreams: ParsedStream[] = [];
const start = Date.now();
let queryType = type;
if (AnimeDatabase.getInstance().isAnime(id)) {
queryType = 'anime';
}
// Helper function to fetch streams from an addon and log summary
const fetchFromAddon = async (addon: Addon) => {
@@ -147,7 +156,7 @@ class StreamFetcher {
await this.precompute.precompute(filteredStreams);
logger.info(
`Group processing finished. Filtered to ${filteredStreams.length} streams in ${getTimeTakenSincePoint(groupStart)}`
`Finished fetching from group in ${getTimeTakenSincePoint(groupStart)}`
);
return {
totalTime: Date.now() - groupStart,
@@ -211,7 +220,7 @@ class StreamFetcher {
allStreams,
previousGroupTimeTaken,
totalTimeTaken,
type
queryType
);
const shouldFetchNext = await evaluator.evaluate(
nextGroup.condition
@@ -233,7 +242,7 @@ class StreamFetcher {
allStreams,
previousGroupTimeTaken,
totalTimeTaken,
type
queryType
);
const shouldFetch = await evaluator.evaluate(group.condition);
+111 -44
View File
@@ -4,6 +4,8 @@ import {
FeatureControl,
getTimeTakenSincePoint,
constants,
AnimeDatabase,
IdParser,
} from '../utils';
import { TYPES } from '../utils/constants';
import { compileRegex } from '../utils/regex';
@@ -12,7 +14,11 @@ import { safeRegexTest } from '../utils/regex';
import { StreamType } from '../utils/constants';
import { StreamSelector } from '../parser/streamExpression';
import StreamUtils from './utils';
import { TMDBMetadata, TMDBMetadataResponse } from '../metadata/tmdb';
import { MetadataService } from '../metadata/service';
import { Metadata } from '../metadata/utils';
import { titleMatch } from '../parser/utils';
import { partial_ratio } from 'fuzzball';
import { calculateAbsoluteEpisode } from '../builtins/utils/general';
const logger = createLogger('filterer');
@@ -32,7 +38,15 @@ class StreamFilterer {
total: number;
details: Record<string, number>;
}
const isAnime = id.startsWith('kitsu');
const parsedId = IdParser.parse(id, type);
let isAnime = id.startsWith('kitsu');
if (AnimeDatabase.getInstance().isAnime(id)) {
logger.debug(
`Determined that ${id} is anime based on an existing entry in the anime database`
);
isAnime = true;
}
const skipReasons: Record<string, SkipReason> = {
titleMatching: { total: 0, details: {} },
yearMatching: { total: 0, details: {} },
@@ -92,17 +106,51 @@ class StreamFilterer {
...(this.userData.includedRegexPatterns ?? []),
]);
let requestedMetadata: TMDBMetadataResponse | undefined;
let requestedMetadata:
| (Metadata & { absoluteEpisode?: number })
| undefined;
if (
(this.userData.titleMatching?.enabled ||
this.userData.yearMatching?.enabled) &&
this.userData.yearMatching?.enabled ||
this.userData.seasonEpisodeMatching?.enabled) &&
TYPES.includes(type as any)
) {
try {
requestedMetadata = await new TMDBMetadata({
accessToken: this.userData.tmdbAccessToken,
apiKey: this.userData.tmdbApiKey,
}).getMetadata(id, type as any);
if (!parsedId) {
throw new Error(`Invalid ID: ${id}`);
}
const animeEntry = AnimeDatabase.getInstance().getEntryById(
parsedId.type,
parsedId.value
);
if (animeEntry && !parsedId.season) {
parsedId.season =
animeEntry.imdb?.fromImdbSeason?.toString() ??
animeEntry.trakt?.season?.toString();
}
requestedMetadata = await new MetadataService({
tmdbAccessToken: this.userData.tmdbAccessToken,
tmdbApiKey: this.userData.tmdbApiKey,
}).getMetadata(parsedId, type as any);
if (
isAnime &&
parsedId.season &&
parsedId.episode &&
requestedMetadata.seasons
) {
const seasons = requestedMetadata.seasons.map(
({ season_number, episode_count }) => ({
number: season_number.toString(),
episodes: episode_count,
})
);
logger.debug(
`Calculating absolute episode with current season and episode: ${parsedId.season}, ${parsedId.episode} and seasons: ${JSON.stringify(seasons)}`
);
requestedMetadata.absoluteEpisode = Number(
calculateAbsoluteEpisode(parsedId.season, parsedId.episode, seasons)
);
}
logger.info(`Fetched metadata for ${id}`, requestedMetadata);
} catch (error) {
logger.warn(
@@ -127,7 +175,11 @@ class StreamFilterer {
if (!titleMatchingOptions || !titleMatchingOptions.enabled) {
return true;
}
if (!requestedMetadata || requestedMetadata.titles.length === 0) {
if (
!requestedMetadata ||
!requestedMetadata.titles ||
requestedMetadata.titles.length === 0
) {
return true;
}
@@ -153,12 +205,21 @@ class StreamFilterer {
}
if (titleMatchingOptions.mode === 'exact') {
return requestedMetadata?.titles.some(
(title) => normaliseTitle(title) === normaliseTitle(streamTitle)
return titleMatch(
normaliseTitle(streamTitle),
requestedMetadata.titles.map(normaliseTitle),
{
threshold: 0.85,
}
);
} else {
return requestedMetadata?.titles.some((title) =>
normaliseTitle(streamTitle).includes(normaliseTitle(title))
return titleMatch(
normaliseTitle(streamTitle),
requestedMetadata.titles.map(normaliseTitle),
{
threshold: 0.85,
scorer: partial_ratio,
}
);
}
};
@@ -193,33 +254,40 @@ class StreamFilterer {
}
const streamYear = stream.parsedFile?.year;
if (!streamYear && type === 'movie') {
// only filter out movies without a year as series results usually don't include a year
return false;
}
if (!streamYear) {
// non-movie results without a year can be kept in
return true;
// if no year is present, filter out if its a movie, keep otherwise
return type === 'movie' ? false : true;
}
// streamYear can be a string like "2004" or "2012-2020"
const requestedYear = Number(requestedMetadata.year);
let streamYearRange: [number, number] | undefined;
if (streamYear && streamYear.includes('-')) {
// Calculate the requested year range
let requestedYearRange: [number, number] = [
requestedMetadata.year,
requestedMetadata.year,
];
if (requestedMetadata.yearEnd) {
requestedYearRange[1] = requestedMetadata.yearEnd;
}
// Calculate the stream year range
let streamYearRange: [number, number];
if (streamYear.includes('-')) {
const [min, max] = streamYear.split('-').map(Number);
streamYearRange = [min, max];
} else {
streamYearRange = [Number(streamYear), Number(streamYear)];
const yearNum = Number(streamYear);
streamYearRange = [yearNum, yearNum];
}
let tolerance = yearMatchingOptions.tolerance ?? 1;
streamYearRange[0] = streamYearRange[0] - tolerance;
streamYearRange[1] = streamYearRange[1] + tolerance;
// Apply tolerance to the stream year range
const tolerance = yearMatchingOptions.tolerance ?? 1;
streamYearRange[0] -= tolerance;
streamYearRange[1] += tolerance;
// requested year should be within the stream year range
return (
requestedYear >= streamYearRange[0] &&
requestedYear <= streamYearRange[1]
);
// If the requested year range and stream year range overlap, accept the stream
const [requestedStart, requestedEnd] = requestedYearRange;
const [streamStart, streamEnd] = streamYearRange;
return requestedStart <= streamEnd && requestedEnd >= streamStart;
};
const performSeasonEpisodeMatch = (stream: ParsedStream) => {
@@ -231,17 +299,13 @@ class StreamFilterer {
return true;
}
// parse the id to get the season and episode
const seasonEpisodeRegex = /:(\d+):(\d+)$/;
const match = id.match(seasonEpisodeRegex);
if (!match || !match[1] || !match[2]) {
// only if both season and episode are present, we can filter
return true;
}
const requestedSeason = parseInt(match[1]);
const requestedEpisode = parseInt(match[2]);
if (!parsedId) return true;
const requestedSeason = Number.isInteger(Number(parsedId.season))
? Number(parsedId.season)
: undefined;
const requestedEpisode = Number.isInteger(Number(parsedId.episode))
? Number(parsedId.episode)
: undefined;
if (
seasonEpisodeMatchingOptions.requestTypes?.length &&
@@ -270,11 +334,14 @@ class StreamFilterer {
return false;
}
// is requested episode present
// is the present episode incorrect (does not match either the requested episode or absolute episode if present)
if (
requestedEpisode &&
stream.parsedFile?.episode &&
stream.parsedFile.episode !== requestedEpisode
stream.parsedFile.episode !== requestedEpisode &&
(requestedMetadata?.absoluteEpisode
? stream.parsedFile.episode !== requestedMetadata.absoluteEpisode
: true)
) {
return false;
}
+858
View File
@@ -0,0 +1,858 @@
import path from 'path';
import fs from 'fs/promises';
import { z, ZodError } from 'zod';
import { getDataFolder } from './general';
import { makeRequest } from './http';
import { createLogger, getTimeTakenSincePoint } from './logger';
import { IdParser, IdType, ID_TYPES } from './id-parser';
import { formatZodError } from './config';
import { DistributedLock, Env } from '.';
import { createWriteStream } from 'fs';
const logger = createLogger('anime-database');
// --- Constants for Data Sources ---
const ANIME_DATABASE_PATH = path.join(getDataFolder(), 'anime-database');
const DATA_SOURCES = {
fribbMappings: {
name: 'Fribb Mappings',
url: 'https://raw.githubusercontent.com/Fribb/anime-lists/refs/heads/master/anime-list-full.json',
filePath: path.join(ANIME_DATABASE_PATH, 'fribb-mappings.json'),
etagPath: path.join(ANIME_DATABASE_PATH, 'fribb-mappings.etag'),
loader: 'loadFribbMappings',
refreshInterval: Env.ANIME_DB_FRIBB_MAPPINGS_REFRESH_INTERVAL,
dataKey: 'fribbMappingsById',
},
manami: {
name: 'Manami DB',
url: 'https://github.com/manami-project/anime-offline-database/releases/download/latest/anime-offline-database.json',
filePath: path.join(ANIME_DATABASE_PATH, 'manami-db.json'),
etagPath: path.join(ANIME_DATABASE_PATH, 'manami-db.etag'),
loader: 'loadManamiDb',
refreshInterval: Env.ANIME_DB_MANAMI_DB_REFRESH_INTERVAL,
dataKey: 'manamiById',
},
kitsuImdb: {
name: 'Kitsu IMDB Mapping',
url: 'https://raw.githubusercontent.com/TheBeastLT/stremio-kitsu-anime/master/static/data/imdb_mapping.json',
filePath: path.join(ANIME_DATABASE_PATH, 'kitsu-imdb-mapping.json'),
etagPath: path.join(ANIME_DATABASE_PATH, 'kitsu-imdb-mapping.etag'),
loader: 'loadKitsuImdbMapping',
refreshInterval: Env.ANIME_DB_KITSU_IMDB_MAPPING_REFRESH_INTERVAL,
dataKey: 'kitsuById',
},
anitraktMovies: {
name: 'Extended Anitrakt Movies',
url: 'https://github.com/rensetsu/db.trakt.extended-anitrakt/releases/download/latest/movies_ex.json',
filePath: path.join(ANIME_DATABASE_PATH, 'anitrakt-movies-ex.json'),
etagPath: path.join(ANIME_DATABASE_PATH, 'anitrakt-movies-ex.etag'),
loader: 'loadExtendedAnitraktMovies',
refreshInterval: Env.ANIME_DB_EXTENDED_ANITRAKT_MOVIES_REFRESH_INTERVAL,
dataKey: 'extendedAnitraktMoviesById',
},
anitraktTv: {
name: 'Extended Anitrakt TV',
url: 'https://github.com/rensetsu/db.trakt.extended-anitrakt/releases/download/latest/tv_ex.json',
filePath: path.join(ANIME_DATABASE_PATH, 'anitrakt-tv-ex.json'),
etagPath: path.join(ANIME_DATABASE_PATH, 'anitrakt-tv-ex.etag'),
loader: 'loadExtendedAnitraktTv',
refreshInterval: Env.ANIME_DB_EXTENDED_ANITRAKT_TV_REFRESH_INTERVAL,
dataKey: 'extendedAnitraktTvById',
},
} as const;
const extractIdFromUrl: {
[K in
| 'anidbId'
| 'anilistId'
| 'animePlanetId'
| 'animecountdownId'
| 'anisearchId'
| 'imdbId'
| 'kitsuId'
| 'livechartId'
| 'malId'
| 'notifyMoeId'
| 'simklId'
| 'themoviedbId'
| 'thetvdbId']?: (url: string) => string | null;
} = {
anidbId: (url: string) => {
const match = url.match(/anidb\.net\/anime\/(\d+)/);
return match ? match[1] : null;
},
anilistId: (url: string) => {
const match = url.match(/anilist\.co\/anime\/(\d+)/);
return match ? match[1] : null;
},
animePlanetId: (url: string) => {
const match = url.match(/anime-planet\.com\/anime\/(\w+)/);
return match ? match[1] : null;
},
animecountdownId: (url: string) => {
const match = url.match(/animecountdown\.com\/(\d+)/);
return match ? match[1] : null;
},
anisearchId: (url: string) => {
const match = url.match(/anisearch\.com\/anime\/(\d+)/);
return match ? match[1] : null;
},
kitsuId: (url: string) => {
const match = url.match(/kitsu\.app\/anime\/(\d+)/);
return match ? match[1] : null;
},
livechartId: (url: string) => {
const match = url.match(/livechart\.me\/anime\/(\d+)/);
return match ? match[1] : null;
},
malId: (url: string) => {
const match = url.match(/myanimelist\.net\/anime\/(\d+)/);
return match ? match[1] : null;
},
notifyMoeId: (url: string) => {
const match = url.match(/notify\.moe\/anime\/(\w+)/);
return match ? match[1] : null;
},
simklId: (url: string) => {
const match = url.match(/simkl\.com\/anime\/(\d+)/);
return match ? match[1] : null;
},
};
// --- Zod Schemas and Types ---
const AnimeType = z.enum(['TV', 'SPECIAL', 'OVA', 'MOVIE', 'ONA', 'UNKNOWN']);
const AnimeStatus = z.enum([
'CURRENT',
'FINISHED',
'UPCOMING',
'UNKNOWN',
'ONGOING',
]);
const AnimeSeason = z.enum(['WINTER', 'SPRING', 'SUMMER', 'FALL', 'UNDEFINED']);
// Schema for Fribb's Mappings
const MappingEntrySchema = z
.object({
'anime-planet_id': z.union([z.string(), z.number()]).optional(),
animecountdown_id: z.number().optional(),
anidb_id: z.number().optional(),
anilist_id: z.number().optional(),
anisearch_id: z.number().optional(),
imdb_id: z.string().optional().nullable(),
kitsu_id: z.number().optional(),
livechart_id: z.number().optional(),
mal_id: z.number().optional(),
'notify.moe_id': z.string().optional(),
simkl_id: z.number().optional(),
themoviedb_id: z.coerce.number().optional(),
thetvdb_id: z.number().optional().nullable(),
trakt_id: z.number().optional(),
type: AnimeType,
})
.transform((data) => ({
animePlanetId: data['anime-planet_id'],
animecountdownId: data['animecountdown_id'],
anidbId: data['anidb_id'],
anilistId: data['anilist_id'],
anisearchId: data['anisearch_id'],
imdbId: data['imdb_id'],
kitsuId: data['kitsu_id'],
livechartId: data['livechart_id'],
malId: data['mal_id'],
notifyMoeId: data['notify.moe_id'],
simklId: data['simkl_id'],
themoviedbId: data['themoviedb_id'],
thetvdbId: data['thetvdb_id'],
traktId: data['trakt_id'],
type: data['type'],
}));
type MappingEntry = z.infer<typeof MappingEntrySchema>;
// Schema for Manami's Database
const ManamiEntrySchema = z.object({
sources: z.array(z.url()),
title: z.string(),
type: AnimeType,
episodes: z.number(),
status: AnimeStatus,
animeSeason: z.object({
season: AnimeSeason,
year: z.number().nullable(),
}),
picture: z.url().nullable(),
thumbnail: z.url().nullable(),
duration: z
.object({
value: z.number(),
unit: z.enum(['SECONDS']),
})
.nullable(),
score: z
.object({
arithmeticGeometricMean: z.number(),
arithmeticMean: z.number(),
median: z.number(),
})
.nullable(),
synonyms: z.array(z.string()),
studios: z.array(z.string()),
producers: z.array(z.string()),
relatedAnime: z.array(z.url()),
tags: z.array(z.string()),
});
type ManamiEntry = z.infer<typeof ManamiEntrySchema>;
const KitsuEntrySchema = z
.object({
fanartLogoId: z.coerce.number().optional(),
tvdb_id: z.coerce.number().optional(),
imdb_id: z.string().optional(),
title: z.string().optional(),
fromSeason: z.number().optional(),
fromEpisode: z.number().optional(),
})
.transform((data) => ({
fanartLogoId: data.fanartLogoId,
tvdbId: data.tvdb_id,
imdbId: data.imdb_id,
title: data.title,
fromSeason: data.fromSeason,
fromEpisode: data.fromEpisode,
}));
type KitsuEntry = z.infer<typeof KitsuEntrySchema>;
const ExtendedAnitraktMovieEntrySchema = z
.object({
myanimelist: z.object({
title: z.string(),
id: z.number(),
}),
trakt: z.object({
title: z.string(),
id: z.number(),
slug: z.string(),
type: z.literal('movies'),
}),
release_year: z.number(),
externals: z.object({
tmdb: z.number().optional().nullable(),
imdb: z.string().optional().nullable(),
letterboxd: z
.object({
slug: z.string().nullable(),
lid: z.string().nullable(),
uid: z.number().nullable(),
})
.nullable(),
}),
})
.transform((data) => ({
myanimelist: data.myanimelist,
trakt: data.trakt,
releaseYear: data.release_year,
externals: data.externals,
}));
type ExtendedAnitraktMovieEntry = z.infer<
typeof ExtendedAnitraktMovieEntrySchema
>;
const ExtendedAnitraktTvEntrySchema = z
.object({
myanimelist: z.object({
title: z.string(),
id: z.number(),
}),
trakt: z.object({
title: z.string(),
id: z.number(),
slug: z.string(),
type: z.literal('shows'),
is_split_cour: z.boolean(),
season: z
.object({
id: z.number(),
number: z.number(),
externals: z.object({
tvdb: z.number().nullable(),
tmdb: z.number().nullable(),
imdb: z.string().optional().nullable(),
}),
})
.nullable(),
}),
release_year: z.number(),
externals: z.object({
tvdb: z.number().optional().nullable(),
tmdb: z.number().optional().nullable(),
imdb: z.string().optional().nullable(),
}),
})
.transform((data) => ({
myanimelist: data.myanimelist,
trakt: {
title: data.trakt.title,
id: data.trakt.id,
slug: data.trakt.slug,
type: data.trakt.type,
isSplitCour: data.trakt.is_split_cour,
season: data.trakt.season,
},
releaseYear: data.release_year,
externals: data.externals,
}));
type ExtendedAnitraktTvEntry = z.infer<typeof ExtendedAnitraktTvEntrySchema>;
type MappingIdMap = Map<IdType, Map<string | number, MappingEntry>>;
type ManamiIdMap = Map<IdType, Map<string | number, ManamiEntry>>;
type KitsuIdMap = Map<number, KitsuEntry>;
type ExtendedAnitraktMoviesIdMap = Map<number, ExtendedAnitraktMovieEntry>;
type ExtendedAnitraktTvIdMap = Map<number, ExtendedAnitraktTvEntry>;
export class AnimeDatabase {
private static instance: AnimeDatabase;
private isInitialised = false;
// Data storage
// private mappingsById: MappingIdMap = new Map();
// private manamiById: ManamiIdMap = new Map();
// private kitsuById: KitsuIdMap = new Map();
// private extendedAnitraktMoviesById: ExtendedAnitraktMoviesIdMap = new Map();
// private extendedAnitraktTvById: ExtendedAnitraktTvIdMap = new Map();
private dataStore: {
fribbMappingsById: MappingIdMap;
manamiById: ManamiIdMap;
kitsuById: KitsuIdMap;
extendedAnitraktMoviesById: ExtendedAnitraktMoviesIdMap;
extendedAnitraktTvById: ExtendedAnitraktTvIdMap;
} = {
fribbMappingsById: new Map(),
manamiById: new Map(),
kitsuById: new Map(),
extendedAnitraktMoviesById: new Map(),
extendedAnitraktTvById: new Map(),
};
// Refresh timers
private refreshTimers: NodeJS.Timeout[] = [];
private constructor() {}
public static getInstance(): AnimeDatabase {
if (!this.instance) {
this.instance = new AnimeDatabase();
}
return this.instance;
}
public async initialise(): Promise<void> {
if (this.isInitialised) {
logger.warn('AnimeDatabase is already initialised.');
return;
}
logger.info('Starting initial refresh of all anime data sources...');
// Perform initial fetch for all datasets concurrently
const refreshPromises = Object.values(DATA_SOURCES).map((dataSource) =>
this.refreshDataSource(dataSource)
);
await Promise.all(refreshPromises);
this.setupAllRefreshIntervals();
this.isInitialised = true;
logger.info('AnimeDatabase initialised successfully.');
}
// --- Public Methods for Data Access ---
public isAnime(id: string): boolean {
const parsedId = IdParser.parse(id, 'unknown');
if (parsedId && this.getEntryById(parsedId.type, parsedId.value) !== null) {
return true;
}
return false;
}
public getEntryById(idType: IdType, idValue: string | number) {
const getFromMap = <T>(map: Map<any, T> | undefined, key: any) =>
map?.get(key) || map?.get(key.toString()) || map?.get(Number(key));
let mappings = getFromMap(
this.dataStore.fribbMappingsById.get(idType),
idValue
);
let details = getFromMap(this.dataStore.manamiById.get(idType), idValue);
// If no direct match for details, try finding via mappings
if (!details && mappings) {
logger.debug('No direct match for details, searching via mappings...');
for (const [type, id] of Object.entries(mappings)) {
if (id && type !== idType) {
details = getFromMap(
this.dataStore.manamiById.get(type as IdType),
id
);
if (details) break;
}
}
}
const malId =
mappings?.malId ?? (idType === 'malId' ? Number(idValue) : null);
const kitsuId =
mappings?.kitsuId ?? (idType === 'kitsuId' ? Number(idValue) : null);
const kitsuEntry = kitsuId ? this.dataStore.kitsuById.get(kitsuId) : null;
const tvAnitraktEntry = malId
? this.dataStore.extendedAnitraktTvById.get(malId)
: null;
const movieAnitraktEntry = malId
? this.dataStore.extendedAnitraktMoviesById.get(malId)
: null;
if (
!details &&
!mappings &&
!kitsuEntry &&
!tvAnitraktEntry &&
!movieAnitraktEntry
) {
return null;
}
// Merge data from all sources
const finalMappings = {
...mappings,
imdbId:
mappings?.imdbId ??
kitsuEntry?.imdbId ??
movieAnitraktEntry?.externals?.imdb ??
tvAnitraktEntry?.externals?.imdb,
kitsuId: mappings?.kitsuId ?? kitsuId,
malId: mappings?.malId ?? malId,
themoviedbId:
mappings?.themoviedbId ??
movieAnitraktEntry?.externals?.tmdb ??
tvAnitraktEntry?.externals?.tmdb,
thetvdbId:
kitsuEntry?.tvdbId ??
mappings?.thetvdbId ??
tvAnitraktEntry?.externals?.tvdb,
traktId:
mappings?.traktId ??
tvAnitraktEntry?.trakt?.id ??
movieAnitraktEntry?.trakt?.id,
};
return {
mappings: finalMappings,
imdb: kitsuEntry
? {
fromImdbSeason: kitsuEntry.fromSeason,
fromImdbEpisode: kitsuEntry.fromEpisode,
title: kitsuEntry.title,
}
: null,
fanart: kitsuEntry?.fanartLogoId
? { logoId: kitsuEntry.fanartLogoId }
: null,
trakt: tvAnitraktEntry?.trakt
? {
title: tvAnitraktEntry.trakt.title,
slug: tvAnitraktEntry.trakt.slug,
isSplitCour: tvAnitraktEntry.trakt.isSplitCour,
season: tvAnitraktEntry.trakt.season ?? null,
}
: movieAnitraktEntry?.trakt
? {
title: movieAnitraktEntry.trakt.title,
slug: movieAnitraktEntry.trakt.slug,
}
: null,
...details,
};
}
// --- Refresh Interval Configuration ---
private setupAllRefreshIntervals(): void {
this.refreshTimers.forEach(clearInterval);
this.refreshTimers = [];
for (const source of Object.values(DATA_SOURCES)) {
const timer = setInterval(
() =>
this.refreshDataSource(source).catch((e) =>
logger.error(`[${source.name}] Failed to auto-refresh: ${e}`)
),
source.refreshInterval
);
this.refreshTimers.push(timer);
logger.info(
`[${source.name}] Set auto-refresh interval to ${source.refreshInterval}ms`
);
}
}
// --- Private Refresh and Load Methods ---
private async refreshDataSource(
source: (typeof DATA_SOURCES)[keyof typeof DATA_SOURCES]
): Promise<void> {
const lockKey = `anime-datasource-refresh-${source.dataKey}`;
const lockOptions = { timeout: 10000, ttl: 12000 };
let lockAcquired = false;
let firstError: any = null;
for (let attempt = 0; attempt < 2 && !lockAcquired; attempt++) {
try {
await DistributedLock.getInstance().withLock(
lockKey,
async () => {
try {
const remoteEtag = await this.fetchRemoteEtag(source.url);
const localEtag = await this.readLocalFile(source.etagPath);
const isDbMissing = !(await this.fileExists(source.filePath));
const isOutOfDate =
!remoteEtag || !localEtag || remoteEtag !== localEtag;
if (isDbMissing || isOutOfDate) {
logger.info(
`[${source.name}] Source is missing or out of date. Downloading...`
);
await this.downloadFile(
source.url,
source.filePath,
source.etagPath,
remoteEtag
);
} else {
logger.info(`[${source.name}] Source is up to date.`);
}
} catch (error) {
logger.error(
`[${source.name}] Failed to refresh: ${error}. Will retry on next cycle.`
);
}
},
lockOptions
);
lockAcquired = true;
// Dynamically call the correct loader
// the lock is only needed to avoid multiple instances trying to download files at the same time.
// since the loader needs to be called per instance as its in memory, it needs to be called outside of the lock.
await this[source.loader]();
} catch (error) {
if (attempt === 0) {
// First attempt failed, wait for ttl before retrying
logger.error(`[${source.name}] Failed to refresh: ${error}`);
firstError = error;
logger.warn(
`[${source.name}] Lock acquisition failed, will retry after ${lockOptions.ttl}ms`
);
await new Promise((resolve) => setTimeout(resolve, lockOptions.ttl));
} else {
// Second attempt failed, log and exit
logger.error(`[${source.name}] Will retry on next cycle.`);
}
}
}
}
private async loadFribbMappings(): Promise<void> {
const start = Date.now();
const fileContents = await this.readLocalFile(
DATA_SOURCES.fribbMappings.filePath
);
if (!fileContents)
throw new Error(DATA_SOURCES.fribbMappings.name + ' file not found');
const data = JSON.parse(fileContents);
if (!Array.isArray(data))
throw new Error(
DATA_SOURCES.fribbMappings.name + ' data must be an array'
);
const validEntries = this.validateEntries(data, MappingEntrySchema);
const newMappingsById: MappingIdMap = new Map();
for (const idType of ID_TYPES) {
newMappingsById.set(idType, new Map());
}
for (const entry of validEntries) {
for (const idType of ID_TYPES) {
const idValue = entry[idType];
if (idValue !== undefined && idValue !== null) {
const existingEntry = newMappingsById.get(idType)?.get(idValue);
if (!existingEntry) {
newMappingsById.get(idType)?.set(idValue, entry);
}
}
}
}
this.dataStore.fribbMappingsById = newMappingsById;
logger.info(
`[${DATA_SOURCES.fribbMappings.name}] Loaded and indexed ${validEntries.length} valid entries in ${getTimeTakenSincePoint(start)}`
);
}
private async loadManamiDb(): Promise<void> {
const start = Date.now();
const fileContents = await this.readLocalFile(DATA_SOURCES.manami.filePath);
if (!fileContents)
throw new Error(DATA_SOURCES.manami.name + ' file not found');
const data = JSON.parse(fileContents);
if (!Array.isArray(data.data))
throw new Error(DATA_SOURCES.manami.name + ' data must be an array');
const validEntries = this.validateEntries(data.data, ManamiEntrySchema);
const newManamiById: ManamiIdMap = new Map();
const idTypes = Object.keys(extractIdFromUrl) as Exclude<
IdType,
'traktId'
>[];
for (const idType of idTypes) {
newManamiById.set(idType, new Map());
}
for (const entry of validEntries) {
for (const sourceUrl of entry.sources) {
for (const idType of idTypes) {
const idExtractor = extractIdFromUrl[idType];
if (idExtractor) {
const idValue = idExtractor(sourceUrl);
if (idValue) {
const existingEntry = newManamiById.get(idType)?.get(idValue);
if (!existingEntry) {
newManamiById.get(idType)?.set(idValue, entry);
}
}
}
}
}
}
this.dataStore.manamiById = newManamiById;
logger.info(
`[${DATA_SOURCES.manami.name}] Loaded and indexed ${validEntries.length} valid entries in ${getTimeTakenSincePoint(start)}`
);
}
private async loadKitsuImdbMapping(): Promise<void> {
const start = Date.now();
const fileContents = await this.readLocalFile(
DATA_SOURCES.kitsuImdb.filePath
);
if (!fileContents)
throw new Error(DATA_SOURCES.kitsuImdb.name + ' file not found');
const data = JSON.parse(fileContents);
// Validate each entry using the schema
this.dataStore.kitsuById = new Map();
for (const [kitsuId, kitsuEntry] of Object.entries(data)) {
const parsed = KitsuEntrySchema.safeParse(kitsuEntry);
if (parsed.success) {
this.dataStore.kitsuById.set(Number(kitsuId), parsed.data);
} else {
logger.warn(
`[${DATA_SOURCES.kitsuImdb.name}] Skipping invalid entry for kitsuId ${kitsuId}: ${formatZodError(parsed.error)}`
);
}
}
logger.info(
`[${DATA_SOURCES.kitsuImdb.name}] Loaded and indexed ${this.dataStore.kitsuById.size} valid entries in ${getTimeTakenSincePoint(start)}`
);
}
private async loadExtendedAnitraktMovies(): Promise<void> {
const start = Date.now();
const fileContents = await this.readLocalFile(
DATA_SOURCES.anitraktMovies.filePath
);
if (!fileContents)
throw new Error(DATA_SOURCES.anitraktMovies.name + ' file not found');
const data = JSON.parse(fileContents);
if (!Array.isArray(data))
throw new Error(
DATA_SOURCES.anitraktMovies.name + ' data must be an array'
);
const validEntries = this.validateEntries(
data,
ExtendedAnitraktMovieEntrySchema
);
const newExtendedAnitraktMoviesById: ExtendedAnitraktMoviesIdMap =
new Map();
for (const entry of validEntries) {
newExtendedAnitraktMoviesById.set(entry.myanimelist.id, entry);
}
this.dataStore.extendedAnitraktMoviesById = newExtendedAnitraktMoviesById;
logger.info(
`[${DATA_SOURCES.anitraktMovies.name}] Loaded and indexed ${validEntries.length} valid entries in ${getTimeTakenSincePoint(start)}`
);
}
private async loadExtendedAnitraktTv(): Promise<void> {
const start = Date.now();
const fileContents = await this.readLocalFile(
DATA_SOURCES.anitraktTv.filePath
);
if (!fileContents)
throw new Error(DATA_SOURCES.anitraktTv.name + ' file not found');
const data = JSON.parse(fileContents);
if (!Array.isArray(data))
throw new Error(DATA_SOURCES.anitraktTv.name + ' data must be an array');
const validEntries = this.validateEntries(
data,
ExtendedAnitraktTvEntrySchema
);
const newExtendedAnitraktTvById: ExtendedAnitraktTvIdMap = new Map();
for (const entry of validEntries) {
newExtendedAnitraktTvById.set(entry.myanimelist.id, entry);
}
this.dataStore.extendedAnitraktTvById = newExtendedAnitraktTvById;
logger.info(
`[${DATA_SOURCES.anitraktTv.name}] Loaded and indexed ${validEntries.length} valid entries in ${getTimeTakenSincePoint(start)}`
);
}
// --- Generic File and Network Helpers ---
private validateEntries<T extends z.ZodTypeAny>(
entries: unknown[],
schema: T
): z.infer<T>[] {
const validEntries: z.infer<T>[] = [];
for (const entry of entries) {
const parsed = schema.safeParse(entry);
if (parsed.success) {
validEntries.push(parsed.data);
} else {
logger.warn(`Skipping invalid entry: ${formatZodError(parsed.error)}`);
logger.verbose(`Invalid entry: ${JSON.stringify(entry, null, 2)}`);
}
}
return validEntries;
}
private async fetchRemoteEtag(url: string): Promise<string | null> {
try {
const response = await makeRequest(url, {
method: 'HEAD',
timeout: 15000,
});
return response.headers.get('etag');
} catch (error) {
logger.warn(`Failed to fetch remote etag for ${url}: ${error}`);
return null;
}
}
private async readLocalFile(filePath: string): Promise<string | null> {
try {
return await fs.readFile(filePath, 'utf8');
} catch (error) {
return null; // Gracefully handle file not existing
}
}
private async downloadFile(
url: string,
filePath: string,
etagPath: string,
remoteEtag: string | null
): Promise<void> {
const startTime = Date.now();
const response = await makeRequest(url, { method: 'GET', timeout: 90000 });
if (!response.ok) throw new Error(`Download failed: ${response.status}`);
// Stream the response directly to file for large files
await fs.mkdir(ANIME_DATABASE_PATH, { recursive: true });
// Create a write stream for the file
const fileStream = createWriteStream(filePath);
// Pipe the response body to the file using Node.js streams
await new Promise<void>((resolve, reject) => {
if (!response.body) {
reject(new Error('No response body to stream'));
return;
}
const reader = response.body.getReader();
const stream = new ReadableStream({
async start(controller) {
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
controller.enqueue(value);
}
controller.close();
} catch (error) {
controller.error(error);
}
},
});
// Pipe the stream to the file
stream
.pipeTo(
new WritableStream({
write(chunk) {
return new Promise((resolve, reject) => {
fileStream.write(chunk, (error) => {
if (error) reject(error);
else resolve();
});
});
},
close() {
fileStream.end();
},
})
)
.then(resolve)
.catch(reject);
// Handle stream errors
fileStream.on('error', reject);
});
// Write the etag if present
const etag = remoteEtag ?? response.headers.get('etag');
if (etag) {
await fs.writeFile(etagPath, etag);
}
logger.info(
`Downloaded ${path.basename(filePath)} in ${getTimeTakenSincePoint(startTime)}`
);
}
private async fileExists(filePath: string): Promise<boolean> {
try {
await fs.access(filePath);
return true;
} catch {
return false;
}
}
}
+8
View File
@@ -238,6 +238,10 @@ export class Cache<K, V> {
return cachedValue as ReturnType<T>;
}
const result = await fn(...args);
// do not cache empty arrays
if (Array.isArray(result) && result.length === 0) {
return result as ReturnType<T>;
}
await this.set(key, result, ttl);
return result;
}
@@ -276,4 +280,8 @@ export class Cache<K, V> {
async waitUntilReady(): Promise<void> {
return this.backend.waitUntilReady();
}
getType(): 'memory' | 'redis' {
return this.backend instanceof MemoryCacheBackend ? 'memory' : 'redis';
}
}
+14 -8
View File
@@ -415,7 +415,7 @@ export async function validateConfig(
}
}
await validateRegexes(config);
await validateRegexes(config, skipErrorsFromAddonsOrProxies);
await new AIOStreams(ensureDecrypted(config), {
skipFailedAddons: skipErrorsFromAddonsOrProxies,
@@ -490,7 +490,7 @@ export function applyMigrations(config: UserData): UserData {
return config;
}
async function validateRegexes(config: UserData) {
async function validateRegexes(config: UserData, skipErrors: boolean = false) {
const excludedRegexes = config.excludedRegexPatterns;
const includedRegexes = config.includedRegexPatterns;
const requiredRegexes = config.requiredRegexPatterns;
@@ -511,14 +511,20 @@ async function validateRegexes(config: UserData) {
allowedPatterns.includes(regex)
);
if (allowedRegexes.length === 0) {
throw new Error(
'You do not have permission to use regex filters, please remove them from your config'
);
if (!skipErrors) {
throw new Error(
'You do not have permission to use regex filters, please remove them from your config'
);
}
return;
}
if (allowedRegexes.length !== regexes.length) {
throw new Error(
`You are only permitted to use specific regex patterns, you have ${regexes.length - allowedRegexes.length} / ${regexes.length} regexes that are not allowed. Please remove them from your config.`
);
if (!skipErrors) {
throw new Error(
`You are only permitted to use specific regex patterns, you have ${regexes.length - allowedRegexes.length} / ${regexes.length} regexes that are not allowed. Please remove them from your config.`
);
}
return;
}
}
+14 -1
View File
@@ -1,4 +1,4 @@
import { Option } from '../db';
import { Option } from '../db/schemas';
export enum ErrorCode {
// User API
@@ -207,7 +207,20 @@ const SERVICES = [
EASYNEWS_SERVICE,
] as const;
export const BUILTIN_SUPPORTED_SERVICES = [
REALDEBRID_SERVICE,
DEBRIDLINK_SERVICE,
PREMIUMIZE_SERVICE,
ALLDEBRID_SERVICE,
TORBOX_SERVICE,
EASYDEBRID_SERVICE,
DEBRIDER_SERVICE,
PIKPAK_SERVICE,
OFFCLOUD_SERVICE,
] as const;
export type ServiceId = (typeof SERVICES)[number];
export type BuiltinServiceId = (typeof BUILTIN_SUPPORTED_SERVICES)[number];
export const MEDIAFLOW_SERVICE = 'mediaflow' as const;
export const STREMTHRU_SERVICE = 'stremthru' as const;
+263
View File
@@ -0,0 +1,263 @@
import { createLogger } from './logger';
import { DB } from '../db/db';
import { Cache } from './cache';
import { RedisClientType } from 'redis';
import { Env } from './env';
import { REDIS_PREFIX } from './constants';
import { TransactionQueue } from '../db/queue';
const logger = createLogger('distributed-lock');
const lockPrefix = `${REDIS_PREFIX}lock:`;
export interface LockOptions {
timeout?: number;
ttl?: number;
retryInterval?: number;
}
export interface LockResult<T> {
result: T;
cached: boolean;
}
interface StoredResult<T> {
value?: T;
error?: string;
}
export class DistributedLock {
private static instance: DistributedLock;
private redis: RedisClientType | null = null;
private subRedis: RedisClientType | null = null;
private initialised = false;
private initialisePromise: Promise<void> | null = null;
private constructor() {}
static getInstance(): DistributedLock {
if (!this.instance) {
this.instance = new DistributedLock();
}
return this.instance;
}
async initialise(): Promise<void> {
if (this.initialised) return;
if (this.initialisePromise) {
await this.initialisePromise;
return;
}
this.initialisePromise = (async () => {
if (Env.REDIS_URI) {
this.redis = Cache.getRedisClient();
this.subRedis = this.redis.duplicate();
await this.subRedis.connect();
logger.debug('DistributedLock initialised with Redis backend.');
} else {
logger.debug('DistributedLock initialised with SQL backend.');
}
this.initialised = true;
})();
await this.initialisePromise;
}
async withLock<T>(
key: string,
fn: () => Promise<T>,
options: LockOptions = {}
): Promise<LockResult<T>> {
await this.initialise();
return this.redis
? this.withRedisLock(key, fn, options)
: this.withSqlLock(key, fn, options);
}
private async withRedisLock<T>(
key: string,
fn: () => Promise<T>,
options: LockOptions
): Promise<LockResult<T>> {
const { timeout = 30000, ttl = 60000 } = options;
const owner = Math.random().toString(36).substring(2);
const redisKey = `${lockPrefix}${key}`;
const doneChannel = `${redisKey}:done`;
const acquireLock = async (): Promise<boolean> => {
const result = await this.redis!.set(redisKey, owner, {
PX: ttl,
NX: true,
});
return result === 'OK';
};
if (await acquireLock()) {
logger.debug(`Redis lock acquired for key: ${key}`);
let result: T;
try {
result = await fn();
const storedResult: StoredResult<T> = { value: result };
await this.redis!.publish(doneChannel, JSON.stringify(storedResult));
} catch (e: any) {
const errorResult: StoredResult<T> = { error: e.message || 'Error' };
await this.redis!.publish(doneChannel, JSON.stringify(errorResult));
throw e;
} finally {
if ((await this.redis!.get(redisKey)) === owner) {
logger.debug(`Releasing redis lock for key: ${key}`);
await this.redis!.del(redisKey);
}
}
return { result, cached: false };
}
return new Promise((resolve, reject) => {
let timeoutId: NodeJS.Timeout;
const subscriber = (message: string) => {
clearTimeout(timeoutId);
this.subRedis!.unsubscribe(doneChannel);
const storedResult: StoredResult<T> = JSON.parse(message);
if (storedResult.error) {
logger.warn(
`Received error result for key: ${key} from lock holder.`
);
reject(new Error(storedResult.error));
} else {
logger.debug(`Received cached result for key: ${key} via pub/sub.`);
resolve({ result: storedResult.value!, cached: true });
}
};
this.subRedis!.subscribe(doneChannel, subscriber).catch(reject);
timeoutId = setTimeout(() => {
this.subRedis!.unsubscribe(doneChannel);
const errorMessage = `Timed out waiting for redis lock on key: ${key}`;
logger.error(errorMessage);
reject(new Error(errorMessage));
}, timeout);
});
}
private async withSqlLock<T>(
key: string,
fn: () => Promise<T>,
options: LockOptions
): Promise<LockResult<T>> {
const db = DB.getInstance();
const { timeout = 30000, ttl = 60000 } = options;
const { retryInterval = db.isSQLite() ? 250 : 100 } = options;
const owner = Math.random().toString(36).substring(2);
const expiresAt = new Date(Date.now() + ttl);
const tryAcquireLock = async () => {
return TransactionQueue.getInstance().enqueue(async () => {
const tx = await db.begin();
try {
await tx.execute(
`DELETE FROM distributed_locks WHERE expires_at < ?`,
[new Date()]
);
const existing = await tx.execute(
`SELECT * FROM distributed_locks WHERE key = ?`,
[key]
);
if (existing.rows.length === 0) {
await tx.execute(
`INSERT INTO distributed_locks (key, owner, expires_at) VALUES (?, ?, ?)`,
[key, owner, expiresAt]
);
await tx.commit();
logger.debug(`SQL lock acquired for key: ${key}`);
return true;
}
await tx.rollback();
return false;
} catch (e) {
await tx.rollback();
throw e;
}
});
};
if (await tryAcquireLock()) {
let result: T;
try {
result = await fn();
// Atomically update the result using the transaction queue
await TransactionQueue.getInstance().enqueue(async () => {
const tx = await db.begin();
try {
await tx.execute(
`UPDATE distributed_locks SET result = ? WHERE key = ? AND owner = ?`,
[JSON.stringify({ value: result }), key, owner]
);
await tx.commit();
} catch (e) {
await tx.rollback();
throw e;
}
});
} catch (e: any) {
// Atomically update the error using the transaction queue
await TransactionQueue.getInstance().enqueue(async () => {
const tx = await db.begin();
try {
await tx.execute(
`UPDATE distributed_locks SET result = ? WHERE key = ? AND owner = ?`,
[JSON.stringify({ error: e.message || 'Error' }), key, owner]
);
await tx.commit();
} catch (err) {
await tx.rollback();
// We throw the original error, but log the transaction error
logger.error(
`Failed to write error result to lock for key ${key}:`,
err
);
}
});
throw e;
} finally {
// This setTimeout is now safe because the update operation is complete.
setTimeout(() => {
logger.debug(`Releasing SQL lock for key: ${key}`);
db.execute(
`DELETE FROM distributed_locks WHERE key = ? AND owner = ?`,
[key, owner]
);
}, 2000);
}
return { result, cached: false };
}
const startTime = Date.now();
while (Date.now() - startTime < timeout) {
const lock = await db.query(
`SELECT result FROM distributed_locks WHERE key = ?`,
[key]
);
if (lock.length > 0 && lock[0].result) {
const storedResult: StoredResult<T> = JSON.parse(lock[0].result);
if (storedResult.error) {
logger.warn(`Polled error result for key: ${key} from SQL lock.`);
throw new Error(storedResult.error);
}
return { result: storedResult.value!, cached: true };
}
await new Promise((res) => setTimeout(res, retryInterval));
}
const errorMessage = `Timed out waiting for SQL lock on key: ${key}`;
logger.error(errorMessage);
throw new Error(errorMessage);
}
async close(): Promise<void> {
if (this.subRedis) {
await this.subRedis.quit();
}
this.initialised = false;
this.initialisePromise = null;
}
}
+126 -19
View File
@@ -92,9 +92,12 @@ const namedRegexes = makeValidator((x) => {
const removeTrailingSlash = (x: string) =>
x.endsWith('/') ? x.slice(0, -1) : x;
const presetUrls = makeExactValidator<readonly string[]>((x) => {
const urlOrUrlList = makeExactValidator<readonly string[]>((x) => {
if (!x) {
return [];
}
if (typeof x !== 'string') {
throw new EnvError('Preset URLs must be a string or an array of strings');
throw new EnvError('List of URLs must be a string or an array of strings');
}
const validateUrl = (x: string) => {
try {
@@ -108,7 +111,7 @@ const presetUrls = makeExactValidator<readonly string[]>((x) => {
const urls = JSON.parse(x);
if (!Array.isArray(urls) || urls.some((x) => !validateUrl(x))) {
throw new EnvError(
'Preset URLs must be an array of URLs or a single URL'
'List of URLs must be an array of URLs or a single URL'
);
}
return Object.freeze(urls.map(removeTrailingSlash));
@@ -329,13 +332,13 @@ export const Env = cleanEnv(process.env, {
default: 500,
desc: 'Redis timeout for the addon',
}),
ADDON_PROXY: url({
ADDON_PROXY: urlOrUrlList({
default: undefined,
desc: 'Proxy URL for the addon',
}),
ADDON_PROXY_CONFIG: str({
default: undefined,
desc: 'Proxy config for the addon in format of comma separated hostname:boolean',
desc: 'Proxy config for the addon in format of comma separated hostname:(boolean|number). If you have multiple proxies, use a number to specify the index of the proxy. (starts from 0)',
}),
REQUEST_URL_MAPPINGS: urlMappings({
default: undefined,
@@ -357,6 +360,10 @@ export const Env = cleanEnv(process.env, {
default: undefined,
desc: 'TMDB API Key. Used for fetching metadata for the strict title matching option.',
}),
TRAKT_CLIENT_ID: str({
default: undefined,
desc: 'Trakt Client ID. Used for fetching Trakt aliases.',
}),
PROVIDE_STREAM_DATA: boolOrList<boolean | string[] | undefined>({
default: undefined,
desc: 'Provide stream data to the client in stream responses. Required for users to wrap this addon within another AIOStreams instance.',
@@ -369,6 +376,26 @@ export const Env = cleanEnv(process.env, {
default: true,
desc: 'Enable the search API. If true, the search API will be enabled.',
}),
ANIME_DB_FRIBB_MAPPINGS_REFRESH_INTERVAL: num({
default: 24 * 60 * 60 * 1000, // 24 hours
desc: 'Interval for refreshing the anime mappings in milliseconds',
}),
ANIME_DB_MANAMI_DB_REFRESH_INTERVAL: num({
default: 7 * 24 * 60 * 60 * 1000, // 7 days
desc: 'Interval for refreshing the Manami anime offline database in milliseconds',
}),
ANIME_DB_KITSU_IMDB_MAPPING_REFRESH_INTERVAL: num({
default: 24 * 60 * 60 * 1000, // 24 hours
desc: 'Interval for refreshing the Kitsu IMDB mapping in milliseconds',
}),
ANIME_DB_EXTENDED_ANITRAKT_MOVIES_REFRESH_INTERVAL: num({
default: 24 * 60 * 60 * 1000, // 24 hours
desc: 'Interval for refreshing the Extended Anitrakt Movies in milliseconds',
}),
ANIME_DB_EXTENDED_ANITRAKT_TV_REFRESH_INTERVAL: num({
default: 24 * 60 * 60 * 1000, // 24 hours
desc: 'Interval for refreshing the Extended Anitrakt TV in milliseconds',
}),
// logging settings
LOG_SENSITIVE_INFO: bool({
default: false,
@@ -805,7 +832,7 @@ export const Env = cleanEnv(process.env, {
desc: 'AIOStreams user agent',
}),
COMET_URL: presetUrls({
COMET_URL: urlOrUrlList({
default: ['https://comet.elfhosted.com'],
desc: 'Comet URL',
}),
@@ -832,7 +859,7 @@ export const Env = cleanEnv(process.env, {
}),
// MediaFusion settings
MEDIAFUSION_URL: presetUrls({
MEDIAFUSION_URL: urlOrUrlList({
default: ['https://mediafusion.elfhosted.com'],
desc: 'MediaFusion URL',
}),
@@ -858,7 +885,7 @@ export const Env = cleanEnv(process.env, {
}),
// Jackettio settings
JACKETTIO_URL: presetUrls({
JACKETTIO_URL: urlOrUrlList({
default: ['https://jackettio.elfhosted.com'],
desc: 'Jackettio URL',
}),
@@ -1061,8 +1088,8 @@ export const Env = cleanEnv(process.env, {
}),
// StremThru Store settings
STREMTHRU_STORE_URL: presetUrls({
default: ['https://stremthru.elfhosted.com/stremio/store'],
STREMTHRU_STORE_URL: urlOrUrlList({
default: ['https://stremthru.13377001.xyz/'],
desc: 'StremThru Store URL',
}),
DEFAULT_STREMTHRU_STORE_TIMEOUT: num({
@@ -1088,8 +1115,8 @@ export const Env = cleanEnv(process.env, {
}),
// StremThru Torz settings
STREMTHRU_TORZ_URL: presetUrls({
default: ['https://stremthru.elfhosted.com/stremio/torz'],
STREMTHRU_TORZ_URL: urlOrUrlList({
default: ['https://stremthru.13377001.xyz/stremio/torz'],
desc: 'StremThru Torz URL',
}),
DEFAULT_STREMTHRU_TORZ_TIMEOUT: num({
@@ -1522,7 +1549,14 @@ export const Env = cleanEnv(process.env, {
default: 'https://stremthru.13377001.xyz',
desc: 'Builtin StremThru URL',
}),
BUILTIN_DEBRID_INSTANT_AVAILABILITY_CACHE_TTL: num({
default: 60 * 30, // 30 minutes
desc: 'Builtin Debrid instant availability cache TTL',
}),
BUILTIN_DEBRID_PLAYBACK_LINK_CACHE_TTL: num({
default: 60 * 60, // 1 hour
desc: 'Builtin Debrid playback link cache TTL',
}),
BUILTIN_GDRIVE_CLIENT_ID: str({
default: undefined,
desc: 'Builtin GDrive client ID',
@@ -1557,22 +1591,87 @@ export const Env = cleanEnv(process.env, {
desc: 'Builtin TorBox Search search API timeout',
}),
BUILTIN_TORBOX_SEARCH_SEARCH_API_CACHE_TTL: num({
default: 1 * 60 * 60, // 1 hour
default: 7 * 24 * 60 * 60, // 7 days
desc: 'Builtin TorBox Search search API cache TTL',
}),
BUILTIN_TORBOX_SEARCH_METADATA_CACHE_TTL: num({
default: 7 * 24 * 60 * 60, // 7 days
default: 14 * 24 * 60 * 60, // 14 days
desc: 'Builtin TorBox Search metadata cache TTL',
}),
BUILTIN_TORBOX_SEARCH_INSTANT_AVAILABILITY_CACHE_TTL: num({
default: 15 * 60, // 15 minutes
desc: 'Builtin TorBox Search instant availability cache TTL',
}),
BUILTIN_TORBOX_SEARCH_CACHE_PER_USER_SEARCH_ENGINE: bool({
default: false,
desc: 'Whether to cache results separately for every user that is using their own search engines.',
}),
BUILTIN_TORZNAB_SEARCH_TIMEOUT: num({
default: 30000, // 30 seconds
desc: 'Builtin Torznab Search timeout',
}),
BUILTIN_TORZNAB_SEARCH_CACHE_TTL: num({
default: 7 * 24 * 60 * 60, // 7 days
desc: 'Builtin Torznab Search cache TTL',
}),
BUILTIN_TORZNAB_CAPABILITIES_CACHE_TTL: num({
default: 14 * 24 * 60 * 60, // 14 days
desc: 'Builtin Torznab Capabilities cache TTL',
}),
BUILTIN_NEWZNAB_SEARCH_TIMEOUT: num({
default: 30000, // 30 seconds
desc: 'Builtin Newznab Search timeout',
}),
BUILTIN_NEWZNAB_SEARCH_CACHE_TTL: num({
default: 7 * 24 * 60 * 60, // 7 days
desc: 'Builtin Newznab Search cache TTL',
}),
BUILTIN_NEWZNAB_CAPABILITIES_CACHE_TTL: num({
default: 14 * 24 * 60 * 60, // 14 days
desc: 'Builtin Newznab Capabilities cache TTL',
}),
BUILTIN_ZILEAN_URL: url({
default: 'https://zilean.elfhosted.com',
desc: 'Builtin Zilean URL',
}),
BUILTIN_ZILEAN_TIMEOUT: num({
default: undefined,
desc: 'Builtin Zilean timeout',
}),
BUILTIN_ANIMETOSHO_URL: url({
default: 'https://feed.animetosho.org',
desc: 'Builtin AnimeTosho URL',
}),
BUILTIN_ANIMETOSHO_TIMEOUT: num({
default: undefined,
desc: 'Builtin AnimeTosho timeout',
}),
BUILTIN_PROWLARR_URL: url({
default: undefined,
desc: 'Builtin Prowlarr URL',
}),
BUILTIN_PROWLARR_API_KEY: str({
default: undefined,
desc: 'Builtin Prowlarr API Key',
}),
BUILTIN_PROWLARR_INDEXERS: commaSeparated({
default: undefined,
desc: 'Comma separated list of prowlarr indexers to use.',
}),
BUILTIN_PROWLARR_SEARCH_TIMEOUT: num({
default: 30000, // 30 seconds
desc: 'Builtin Prowlarr Search timeout',
}),
BUILTIN_PROWLARR_SEARCH_CACHE_TTL: num({
default: 7 * 24 * 60 * 60, // 7 days
desc: 'Builtin Prowlarr Search cache TTL',
}),
BUILTIN_PROWLARR_INDEXERS_CACHE_TTL: num({
default: 14 * 24 * 60 * 60, // 14 days
desc: 'Builtin Prowlarr Indexers cache TTL',
}),
// Rate limiting settings
DISABLE_RATE_LIMITS: bool({
default: false,
@@ -1615,6 +1714,14 @@ export const Env = cleanEnv(process.env, {
CATALOG_API_RATE_LIMIT_MAX_REQUESTS: num({
default: 5, // allow 100 requests per IP per minute
}),
ANIME_API_RATE_LIMIT_WINDOW: num({
default: 60, // 1 minute
desc: 'Time window for mappings API rate limiting in seconds',
}),
ANIME_API_RATE_LIMIT_MAX_REQUESTS: num({
default: 120,
desc: 'Maximum number of requests allowed per IP within the time window',
}),
STREMIO_STREAM_RATE_LIMIT_WINDOW: num({
default: 15, // 1 minute
desc: 'Time window for Stremio stream rate limiting in seconds',
+11
View File
@@ -1,5 +1,16 @@
import { Addon } from '../db/schemas';
import { parseConnectionURI } from '../db/utils';
import { Env } from './env';
import path from 'path';
export function getAddonName(addon: Addon): string {
return `${addon.name}${addon.displayIdentifier || addon.identifier ? ` ${addon.displayIdentifier || addon.identifier}` : ''}`;
}
export function getDataFolder(): string {
const databaseURI = parseConnectionURI(Env.DATABASE_URI);
if (databaseURI.dialect === 'sqlite') {
return path.dirname(databaseURI.filename);
}
return path.join(process.cwd(), 'data');
}
+53 -14
View File
@@ -73,7 +73,7 @@ export async function makeRequest(url: string, options: RequestOptions) {
}
}
const useProxy = shouldProxy(urlObj);
const { useProxy, proxyIndex } = shouldProxy(urlObj);
const headers = new Headers(options.headers);
if (options.forwardIp) {
for (const header of HEADERS_FOR_IP_FORWARDING) {
@@ -114,7 +114,7 @@ export async function makeRequest(url: string, options: RequestOptions) {
await urlCount.set(key, 1, Env.RECURSION_THRESHOLD_WINDOW);
}
logger.debug(
`Making a ${useProxy ? 'proxied' : 'direct'} request to ${makeUrlLogSafe(
`Making a ${useProxy ? 'proxied' : 'direct'}${proxyIndex !== -1 ? ` (proxy ${proxyIndex + 1})` : ''} request to ${makeUrlLogSafe(
urlObj.toString()
)} with forwarded ip ${maskSensitiveInfo(options.forwardIp ?? 'none')} and headers ${maskSensitiveInfo(JSON.stringify(Object.fromEntries(headers)))}`
);
@@ -123,7 +123,9 @@ export async function makeRequest(url: string, options: RequestOptions) {
method: options.method,
body: options.body,
headers: headers,
dispatcher: useProxy ? getProxyAgent(Env.ADDON_PROXY!) : undefined,
dispatcher: useProxy
? getProxyAgent(Env.ADDON_PROXY![proxyIndex])
: undefined,
signal: AbortSignal.timeout(options.timeout),
});
@@ -140,42 +142,79 @@ function getProxyAgent(proxyUrl: string) {
type: 5,
port: parseInt(proxyUrlObj.port),
host: proxyUrlObj.hostname,
userId: proxyUrlObj.username || undefined,
password: proxyUrlObj.password || undefined,
});
} else {
return new ProxyAgent(proxyUrl);
}
}
function shouldProxy(url: URL) {
let shouldProxy = false;
function shouldProxy(url: URL): {
useProxy: boolean;
proxyIndex: number;
} {
let useProxy = false;
let hostname = url.hostname;
let proxyIndex = -1;
if (!Env.ADDON_PROXY) {
return false;
if (!Env.ADDON_PROXY || Env.ADDON_PROXY.length === 0) {
return { useProxy: false, proxyIndex };
}
shouldProxy = true;
if (hostname === 'localhost') {
return { useProxy: false, proxyIndex };
}
useProxy = true;
if (Env.ADDON_PROXY_CONFIG) {
for (const rule of Env.ADDON_PROXY_CONFIG.split(',')) {
const [ruleHostname, ruleShouldProxy] = rule.split(':');
if (['true', 'false'].includes(ruleShouldProxy) === false) {
const [ruleHostname, ruleProxyIndexOrBool] = rule.split(':');
if (
['true', 'false'].includes(ruleProxyIndexOrBool) === false &&
isNaN(parseInt(ruleProxyIndexOrBool))
) {
logger.error(`Invalid proxy config: ${rule}`);
continue;
}
if (ruleHostname === '*') {
shouldProxy = !(ruleShouldProxy === 'false');
useProxy = !(ruleProxyIndexOrBool === 'false');
proxyIndex = Number.isInteger(parseInt(ruleProxyIndexOrBool))
? parseInt(ruleProxyIndexOrBool)
: ruleProxyIndexOrBool === 'true'
? 0
: -1;
if (useProxy) {
return { useProxy, proxyIndex };
}
} else if (ruleHostname.startsWith('*')) {
if (hostname.endsWith(ruleHostname.slice(1))) {
shouldProxy = !(ruleShouldProxy === 'false');
useProxy = !(ruleProxyIndexOrBool === 'false');
proxyIndex = Number.isInteger(parseInt(ruleProxyIndexOrBool))
? parseInt(ruleProxyIndexOrBool)
: ruleProxyIndexOrBool === 'true'
? 0
: -1;
}
}
if (hostname === ruleHostname) {
shouldProxy = !(ruleShouldProxy === 'false');
useProxy = !(ruleProxyIndexOrBool === 'false');
proxyIndex = Number.isInteger(parseInt(ruleProxyIndexOrBool))
? parseInt(ruleProxyIndexOrBool)
: ruleProxyIndexOrBool === 'true'
? 0
: -1;
}
}
}
return shouldProxy;
if (useProxy && Env.ADDON_PROXY[proxyIndex] === undefined) {
logger.error(`Invalid proxy index: ${proxyIndex}, does not exist`);
return { useProxy: false, proxyIndex: -1 };
}
return { useProxy, proxyIndex };
}
function domainHasUserAgent(url: URL) {
+174
View File
@@ -0,0 +1,174 @@
export const ID_TYPES = [
'animePlanetId',
'animecountdownId',
'anidbId',
'anilistId',
'anisearchId',
'imdbId',
'kitsuId',
'livechartId',
'malId',
'notifyMoeId',
'simklId',
'themoviedbId',
'thetvdbId',
'traktId',
] as const;
export type IdType = (typeof ID_TYPES)[number];
export type ExternalIdType =
| 'anime-planet_id'
| 'animecountdown_id'
| 'anidb_id'
| 'anilist_id'
| 'anisearch_id'
| 'imdb_id'
| 'kitsu_id'
| 'livechart_id'
| 'mal_id'
| 'notify.moe_id'
| 'simkl_id'
| 'themoviedb_id'
| 'thetvdb_id';
export interface ParsedId {
type: IdType;
value: string | number;
fullId: string;
externalType: ExternalIdType;
mediaType: string;
season?: string;
episode?: string;
}
interface IdParserDefinition {
type: IdType;
externalType: ExternalIdType;
prefixes: string[];
// Regex should have named capture groups:
// - id: the base ID (required)
// - season: the season number (optional)
// - episode: the episode number (optional)
regex: RegExp;
format: (id: string) => string | number;
}
export class IdParser {
private static readonly ID_PARSERS: IdParserDefinition[] = [
{
type: 'imdbId',
externalType: 'imdb_id',
prefixes: ['tt', 'imdb'],
regex: /^(?:tt|imdb)[:-]?(?<id>\d+)(?::(?<season>\d+):(?<episode>\d+))?$/,
format: (id) => `tt${id}`,
},
{
type: 'malId',
externalType: 'mal_id',
prefixes: ['mal'],
regex: /^mal[:-]?(?<id>\d+)(?::(?<episode>\d+))?$/,
format: (id) => Number(id),
},
{
type: 'thetvdbId',
externalType: 'thetvdb_id',
prefixes: ['tvdb'],
regex: /^tvdb[:-]?(?<id>\d+)(?::(?<season>\d+):(?<episode>\d+))?$/,
format: (id) => Number(id),
},
{
type: 'themoviedbId',
externalType: 'themoviedb_id',
prefixes: ['tmdb'],
regex: /^tmdb[:-]?(?<id>\d+)(?::(?<season>\d+):(?<episode>\d+))?$/,
format: (id) => Number(id),
},
{
type: 'kitsuId',
externalType: 'kitsu_id',
prefixes: ['kitsu'],
regex: /^kitsu[:-]?(?<id>\d+)(?::(?<episode>\d+))?$/,
format: (id) => Number(id),
},
{
type: 'anilistId',
externalType: 'anilist_id',
prefixes: ['anilist'],
regex: /^anilist[:-]?(?<id>\d+)(?::(?<episode>\d+))?$/,
format: (id) => Number(id),
},
{
type: 'anidbId',
externalType: 'anidb_id',
prefixes: ['anidb', 'anidb_id', 'anidbid'],
regex: /^(?:anidb|anidb_id|anidbid)[:-]?(?<id>\d+)(?::(?<episode>\d+))?$/,
format: (id) => Number(id),
},
{
type: 'animePlanetId',
externalType: 'anime-planet_id',
prefixes: ['animeplanet', 'ap'],
regex: /^(?:animeplanet|ap)[:-]?(?<id>\d+)$/,
format: (id) => Number(id),
},
{
type: 'animecountdownId',
externalType: 'animecountdown_id',
prefixes: ['acd'],
regex: /^acd[:-]?(?<id>\d+)$/,
format: (id) => Number(id),
},
{
type: 'anisearchId',
externalType: 'anisearch_id',
prefixes: ['anisearch'],
regex: /^anisearch[:-]?(?<id>\d+)$/,
format: (id) => Number(id),
},
{
type: 'notifyMoeId',
externalType: 'notify.moe_id',
prefixes: ['notifymoe', 'nm'],
regex: /^(?:notifymoe|nm)[:-]?(?<id>[a-zA-Z0-9]+)$/,
format: (id) => id,
},
{
type: 'simklId',
externalType: 'simkl_id',
prefixes: ['simkl'],
regex: /^simkl[:-]?(?<id>\d+)$/,
format: (id) => Number(id),
},
];
constructor() {}
public static getPrefixes(types: IdType[]): string[] {
return IdParser.ID_PARSERS.filter((p) => types.includes(p.type)).flatMap(
(p) => p.prefixes
);
}
public static parse(stremioId: string, mediaType: string): ParsedId | null {
for (const parser of IdParser.ID_PARSERS) {
const match = stremioId.match(parser.regex);
if (match?.groups) {
const { id, season, episode } = match.groups;
const parsedId: ParsedId = {
type: parser.type,
value: parser.format(id),
fullId: stremioId,
externalType: parser.externalType,
mediaType,
};
if (season) parsedId.season = season;
if (episode) parsedId.episode = episode;
return parsedId;
}
}
return null;
}
}
+3
View File
@@ -13,3 +13,6 @@ export * from './dsu';
export * from './startup';
export * from './extras';
export * from './rpdb';
export * from './distributed-lock';
export * from './id-parser';
export * from './anime-database';
+6
View File
@@ -39,6 +39,12 @@ const moduleMap: { [key: string]: string } = {
gdrive: '☁️ GDRIVE',
'torbox-search': '🔍 TORBOX SEARCH',
debrid: '🔗 DEBRID',
'distributed-lock': '🔒 DISTRIBUTED LOCK',
'anime-database': '🔍 ANIME DATABASE',
torznab: '🔍 TORZNAB',
newznab: '🔍 NEWZNAB',
'metadata-service': '🔍 METADATA',
torrent: '👤 TORRENT',
};
// Define colors for each log level using full names
+66 -34
View File
@@ -2,21 +2,14 @@ import { Cache } from './cache';
import { makeRequest } from './http';
import { RPDBIsValidResponse } from '../db/schemas';
import { Env } from './env';
export type IdType = 'imdb' | 'tmdb' | 'tvdb';
interface Id {
type: IdType;
value: string;
}
import { IdParser } from './id-parser';
import { AnimeDatabase } from './anime-database';
const apiKeyValidationCache = Cache.getInstance<string, boolean>('rpdbApiKey');
const posterCheckCache = Cache.getInstance<string, string>('rpdbPosterCheck');
export class RPDB {
private readonly apiKey: string;
private readonly TMDB_ID_REGEX = /^(?:tmdb)[-:](\d+)(?::\d+:\d+)?$/;
private readonly TVDB_ID_REGEX = /^(?:tvdb)[-:](\d+)(?::\d+:\d+)?$/;
private readonly IMDB_ID_REGEX = /^(?:tt)(\d+)(?::\d+:\d+)?$/;
constructor(apiKey: string) {
this.apiKey = apiKey;
if (!this.apiKey) {
@@ -64,20 +57,59 @@ export class RPDB {
id: string,
checkExists: boolean = true
): Promise<string | null> {
const parsedId = this.getParsedId(id, type);
const parsedId = IdParser.parse(id, type);
if (!parsedId) return null;
let idType: 'tmdb' | 'imdb' | 'tvdb';
let idValue: string;
switch (parsedId.type) {
case 'themoviedbId':
idType = 'tmdb';
idValue = `${type}-${parsedId.value}`;
break;
case 'imdbId':
idType = 'imdb';
idValue = parsedId.value.toString();
break;
case 'thetvdbId':
if (type === 'movie') return null; // tvdb not supported for movies
idType = 'tvdb';
idValue = parsedId.value.toString();
break;
default: {
// Try to map unsupported id types
const entry = AnimeDatabase.getInstance().getEntryById(
parsedId.type,
parsedId.value
);
if (!entry) return null;
if (entry.mappings?.thetvdbId && type === 'series') {
idType = 'tvdb';
idValue = `${entry.mappings.thetvdbId}`;
} else if (entry.mappings?.themoviedbId) {
idType = 'tmdb';
idValue = `${type}-${entry.mappings.themoviedbId}`;
} else if (entry.mappings?.imdbId) {
idType = 'imdb';
idValue = `tt${entry.mappings.imdbId}`;
} else {
return null;
}
break;
}
}
const cacheKey = `${type}-${id}-${this.apiKey}`;
const cached = await posterCheckCache.get(cacheKey);
if (cached) {
return cached;
}
if (!parsedId) {
if (!idType || !idValue) {
return null;
}
if (parsedId.type === 'tvdb' && type === 'movie') {
// rpdb doesnt seem to support tvdb for movies
return null;
}
const posterUrl = `https://api.ratingposterdb.com/${this.apiKey}/${parsedId.type}/poster-default/${parsedId.value}.jpg?fallback=true`;
const posterUrl = `https://api.ratingposterdb.com/${this.apiKey}/${idType}/poster-default/${idValue}.jpg?fallback=true`;
if (!checkExists) {
return posterUrl;
}
@@ -97,22 +129,22 @@ export class RPDB {
return posterUrl;
}
private getParsedId(id: string, type: string): Id | null {
if (this.TMDB_ID_REGEX.test(id)) {
const match = id.match(this.TMDB_ID_REGEX);
if (['movie', 'series'].includes(type)) {
return match ? { type: 'tmdb', value: `${type}-${match[1]}` } : null;
}
return null;
}
if (this.IMDB_ID_REGEX.test(id)) {
const match = id.match(this.IMDB_ID_REGEX);
return match ? { type: 'imdb', value: `tt${match[1]}` } : null;
}
if (this.TVDB_ID_REGEX.test(id)) {
const match = id.match(this.TVDB_ID_REGEX);
return match ? { type: 'tvdb', value: match[1] } : null;
}
return null;
}
// private getParsedId(id: string, type: string): Id | null {
// if (this.TMDB_ID_REGEX.test(id)) {
// const match = id.match(this.TMDB_ID_REGEX);
// if (['movie', 'series'].includes(type)) {
// return match ? { type: 'tmdb', value: `${type}-${match[1]}` } : null;
// }
// return null;
// }
// if (this.IMDB_ID_REGEX.test(id)) {
// const match = id.match(this.IMDB_ID_REGEX);
// return match ? { type: 'imdb', value: `tt${match[1]}` } : null;
// }
// if (this.TVDB_ID_REGEX.test(id)) {
// const match = id.match(this.TVDB_ID_REGEX);
// return match ? { type: 'tvdb', value: match[1] } : null;
// }
// return null;
// }
}
+119 -13
View File
@@ -6,10 +6,17 @@ const logger = createLogger('startup');
const formatDuration = (seconds: number): string => {
if (seconds < 60) return `${seconds}s`;
if (seconds < 3600) return `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
const hours = Math.floor(seconds / 3600);
const days = Math.floor(seconds / 86400);
const hours = Math.floor((seconds % 86400) / 3600);
const minutes = Math.floor((seconds % 3600) / 60);
const secs = seconds % 60;
return `${hours}h ${minutes}m ${secs}s`;
// return `${days}d ${hours}h ${minutes}m ${secs}s`;
let result = '';
if (days > 0) result += `${days}d `;
if (hours > 0) result += `${hours}h `;
if (minutes > 0) result += `${minutes}m `;
if (secs > 0) result += `${secs}s`;
return result;
};
const formatMilliseconds = (ms: number): string => {
@@ -84,7 +91,7 @@ const logStartupInfo = () => {
logKeyValue('Port:', Env.PORT.toString());
logKeyValue('Base URL:', Env.BASE_URL || 'Not configured');
if (Env.ADDON_PROXY) {
logKeyValue('Proxy URL:', Env.ADDON_PROXY);
logKeyValue('Proxy URL:', Env.ADDON_PROXY.join(', '));
}
if (Env.ADDON_PROXY_CONFIG) {
logKeyValue('Proxy Config:', Env.ADDON_PROXY_CONFIG);
@@ -233,6 +240,10 @@ const logStartupInfo = () => {
'Catalog API:',
`${Env.CATALOG_API_RATE_LIMIT_MAX_REQUESTS}/${formatDuration(Env.CATALOG_API_RATE_LIMIT_WINDOW)}`
);
logKeyValue(
'Anime API:',
`${Env.ANIME_API_RATE_LIMIT_MAX_REQUESTS}/${formatDuration(Env.ANIME_API_RATE_LIMIT_WINDOW)}`
);
logKeyValue(
'Stremio Stream:',
`${Env.STREMIO_STREAM_RATE_LIMIT_MAX_REQUESTS}/${formatDuration(Env.STREMIO_STREAM_RATE_LIMIT_WINDOW)}`
@@ -503,22 +514,116 @@ const logStartupInfo = () => {
'TMDB Access Token:',
Env.TMDB_ACCESS_TOKEN ? '✅ Configured' : '❌ None'
);
logKeyValue(
'TMDB API Key:',
Env.TMDB_API_KEY ? '✅ Configured' : '❌ None'
);
logKeyValue(
'Trakt Client ID:',
Env.TRAKT_CLIENT_ID ? '✅ Configured' : '❌ None'
);
});
logSection('BUILT-IN ADDONS', '🔧', () => {
// Torznab
logKeyValue('Torznab:', '');
logKeyValue(
'Search Timeout:',
formatMilliseconds(Env.BUILTIN_TORZNAB_SEARCH_TIMEOUT),
' '
);
logKeyValue(
'Search Cache TTL:',
formatDuration(Env.BUILTIN_TORZNAB_SEARCH_CACHE_TTL),
' '
);
logKeyValue(
'Capabilities Cache TTL:',
formatDuration(Env.BUILTIN_TORZNAB_CAPABILITIES_CACHE_TTL),
' '
);
// Newznab
logKeyValue('Newznab:', '');
logKeyValue(
'Search Timeout:',
formatMilliseconds(Env.BUILTIN_NEWZNAB_SEARCH_TIMEOUT),
' '
);
logKeyValue(
'Search Cache TTL:',
formatDuration(Env.BUILTIN_NEWZNAB_SEARCH_CACHE_TTL),
' '
);
logKeyValue(
'Capabilities Cache TTL:',
formatDuration(Env.BUILTIN_NEWZNAB_CAPABILITIES_CACHE_TTL),
' '
);
// Prowlarr
const prowlarrSearchEnabled =
Env.BUILTIN_PROWLARR_URL && Env.BUILTIN_PROWLARR_API_KEY;
logKeyValue(
'Prowlarr:',
prowlarrSearchEnabled
? '✅ Enabled'
: '❌ Disabled (Set PROWLARR_URL, PROWLARR_API_KEY)'
);
if (prowlarrSearchEnabled) {
logKeyValue(' URL:', Env.BUILTIN_PROWLARR_URL!, ' ');
logKeyValue(
' API Key:',
Env.BUILTIN_PROWLARR_API_KEY ? '✅ Configured' : '❌ None',
' '
);
logKeyValue(
' Indexers:',
Env.BUILTIN_PROWLARR_INDEXERS ? '✅ Configured' : '❌ None',
' '
);
logKeyValue(
' Search Timeout:',
formatMilliseconds(Env.BUILTIN_PROWLARR_SEARCH_TIMEOUT),
' '
);
logKeyValue(
' Search Cache TTL:',
formatDuration(Env.BUILTIN_PROWLARR_SEARCH_CACHE_TTL),
' '
);
logKeyValue(
' Indexers Cache TTL:',
formatDuration(Env.BUILTIN_PROWLARR_INDEXERS_CACHE_TTL),
' '
);
}
// animetosho
logKeyValue('AnimeTosho:', Env.BUILTIN_ANIMETOSHO_URL);
if (Env.BUILTIN_ANIMETOSHO_TIMEOUT) {
logKeyValue(
' Timeout:',
formatMilliseconds(Env.BUILTIN_ANIMETOSHO_TIMEOUT),
' '
);
}
logKeyValue('Zilean', Env.BUILTIN_ZILEAN_URL);
if (Env.BUILTIN_ZILEAN_TIMEOUT) {
logKeyValue(
' Timeout:',
formatMilliseconds(Env.BUILTIN_ZILEAN_TIMEOUT),
' '
);
}
const torboxSearchEnabled = Env.BASE_URL;
logKeyValue(
'Torbox Search:',
torboxSearchEnabled ? '✅ Enabled' : '❌ Disabled (Set BASE_URL)'
);
if (torboxSearchEnabled) {
if (Env.BUILTIN_TORBOX_SEARCH_INSTANT_AVAILABILITY_CACHE_TTL)
logKeyValue(
' Instant Availability Cache TTL:',
formatDuration(
Env.BUILTIN_TORBOX_SEARCH_INSTANT_AVAILABILITY_CACHE_TTL
)
);
logKeyValue(
' Metadata Cache TTL:',
formatDuration(Env.BUILTIN_TORBOX_SEARCH_METADATA_CACHE_TTL)
@@ -1335,7 +1440,6 @@ const logStartupInfo = () => {
// Additional Features
const features: string[] = [];
if (Env.TMDB_ACCESS_TOKEN) features.push('TMDB Integration');
if (Env.CUSTOM_HTML) features.push('Custom HTML');
if (Env.ENCRYPT_MEDIAFLOW_URLS) features.push('Encrypt MediaFlow URLs');
if (Env.ENCRYPT_STREMTHRU_URLS) features.push('Encrypt StremThru URLs');
@@ -1357,7 +1461,9 @@ const logStartupInfo = () => {
logKeyValue('Pruning :', '❌ DISABLED');
}
});
};
const logStartupFooter = () => {
// Footer
logger.info(
'╔═══════════════════════════════════════════════════════════════╗'
@@ -1366,7 +1472,7 @@ const logStartupInfo = () => {
'║ 🎬 AIOStreams Ready! ║'
);
logger.info(
'║ All systems initialized successfully ║'
'║ All systems initialised successfully ║'
);
logger.info(
'╚═══════════════════════════════════════════════════════════════╝'
@@ -1374,4 +1480,4 @@ const logStartupInfo = () => {
logger.info('');
};
export { logStartupInfo };
export { logStartupInfo, logStartupFooter };
+112
View File
@@ -0,0 +1,112 @@
import { Torrent, UnprocessedTorrent, DebridFile } from '../debrid';
import {
extractInfoHashFromMagnet,
extractTrackersFromMagnet,
} from '../builtins/utils/debrid';
import { createLogger } from './logger';
import { Cache } from './cache';
import { makeRequest } from './http';
const logger = createLogger('torrent');
interface TorrentMetadata {
hash: string;
files: DebridFile[];
sources: string[];
}
export class TorrentClient {
static readonly #metadataCache = Cache.getInstance<string, TorrentMetadata>(
'torrent-metadata'
);
private constructor() {}
static async getMetadata(
torrent: UnprocessedTorrent
): Promise<TorrentMetadata | undefined> {
// If we have hash and don't need full metadata, return early
if (torrent.hash) {
return {
hash: torrent.hash,
files: [], // Empty files array since we don't need metadata
sources: torrent.sources || [],
};
}
// If we don't have a download URL, we can't proceed
if (!torrent.downloadUrl) {
logger.debug(
`No download URL available for torrent with hash ${torrent.hash}`
);
return undefined;
}
// Try to get from cache if we have a download URL
if (torrent.downloadUrl) {
const cachedMetadata = await this.#metadataCache.get(torrent.downloadUrl);
if (cachedMetadata) {
return cachedMetadata;
}
}
try {
const metadata = await this.#fetchMetadata(torrent);
if (metadata && torrent.downloadUrl) {
await this.#metadataCache.set(
torrent.downloadUrl,
metadata,
5 * 60 * 1000
);
}
return metadata;
} catch (error) {
logger.error(`Failed to fetch metadata for torrent: ${error}`);
if (torrent.hash) {
// If we have a hash but metadata fetch failed, return basic info
return {
hash: torrent.hash,
files: [],
sources: torrent.sources || [],
};
}
return undefined;
}
}
static async #fetchMetadata(
torrent: UnprocessedTorrent
): Promise<TorrentMetadata> {
if (!torrent.downloadUrl) {
throw new Error('Download URL must be provided');
}
const response = await makeRequest(torrent.downloadUrl, {
timeout: 500,
rawOptions: {
redirect: 'manual',
},
});
// Handle redirects
if (response.status === 302 || response.status === 301) {
const redirectUrl = response.headers.get('Location');
if (!redirectUrl) {
throw new Error('Redirect location not found');
}
if (redirectUrl.startsWith('magnet:')) {
const hash = extractInfoHashFromMagnet(redirectUrl);
const sources = extractTrackersFromMagnet(redirectUrl);
if (!hash) throw new Error('Invalid magnet URL');
return {
hash,
files: [],
sources,
};
}
}
throw new Error('Unsupported torrent');
}
}
+5 -1
View File
@@ -9,9 +9,10 @@
"lint": "next lint"
},
"dependencies": {
"@aiostreams/core": "^0.0.0",
"@aiostreams/core": "workspace:*",
"@dnd-kit/core": "^6.3.1",
"@dnd-kit/modifiers": "^9.0.0",
"@dnd-kit/utilities": "^3.2.2",
"@dnd-kit/sortable": "^10.0.0",
"@headlessui/react": "^2.2.4",
"@headlessui/tailwindcss": "^0.2.2",
@@ -26,6 +27,7 @@
"@radix-ui/react-switch": "^1.2.5",
"@radix-ui/react-tabs": "^1.1.12",
"@radix-ui/react-tooltip": "^1.2.7",
"@radix-ui/react-visually-hidden": "^1.2.1",
"@tailwindcss/forms": "^0.5.10",
"@tailwindcss/typography": "^0.5.16",
"@zag-js/number-input": "^1.13.1",
@@ -35,6 +37,8 @@
"cmdk": "^1.1.1",
"lucide-react": "^0.511.0",
"motion": "^12.23.9",
"framer-motion": "^12.1.4",
"fast-deep-equal": "^3.0.0",
"next": "15.3.2",
"next-themes": "^0.4.6",
"react": "^19.0.0",
Binary file not shown.

After

Width:  |  Height:  |  Size: 86 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 46 KiB

+4 -1
View File
@@ -27,7 +27,10 @@ import { ModeProvider } from '@/context/mode';
function ErrorOverlay({ error }: { error: string | null }) {
return (
<LoadingOverlay showSpinner={false}>
<LuffyError title="Something went wrong!" showRefreshButton>
<LuffyError
title="Something went wrong!"
reset={() => window.location.reload()}
>
<p>{error}</p>
</LuffyError>
</LoadingOverlay>
@@ -10,7 +10,7 @@ import { Textarea } from '../ui/textarea';
import { Select } from '../ui/select';
import { Switch } from '../ui/switch';
import { TextInput } from '../ui/text-input';
import FileParser from '@aiostreams/core/src/parser/file';
import FileParser from '../../../../core/src/parser/file';
import { UserConfigAPI } from '@/services/api';
import { SNIPPETS } from '../../../../core/src/utils/constants';
import { Modal } from '@/components/ui/modal';
+5 -3
View File
@@ -31,9 +31,11 @@ export function StatusProvider({ children }: { children: ReactNode }) {
useEffect(() => {
fetch(`${baseUrl}/status`)
.then((res) => {
if (!res.ok) throw new Error('Failed to fetch status');
return res.json();
.then(async (res) => {
const data = await res.json();
if (!res.ok)
throw new Error(data.error?.message || 'Failed to fetch status');
return data;
})
.then((data) => setStatus(data.data))
.catch((err) => setError(err.message))
+5 -18
View File
@@ -1,12 +1,8 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"target": "ES2021",
"lib": [
"dom",
"dom.iterable",
"esnext"
],
"target": "ES2022",
"lib": ["dom", "dom.iterable", "esnext"],
"allowJs": true,
"skipLibCheck": true,
"strict": true,
@@ -19,9 +15,7 @@
"jsx": "preserve",
"incremental": true,
"paths": {
"@/*": [
"./src/*"
]
"@/*": ["./src/*"]
},
"plugins": [
{
@@ -34,13 +28,6 @@
"path": "../server"
}
],
"include": [
"**/*.ts",
"**/*.tsx",
"next-env.d.ts",
".next/types/**/*.ts"
],
"exclude": [
"node_modules"
]
"include": ["**/*.ts", "**/*.tsx", "next-env.d.ts", ".next/types/**/*.ts"],
"exclude": ["node_modules"]
}
+3 -2
View File
@@ -10,10 +10,11 @@
},
"description": "Combine all your streams into one addon and display them with consistent formatting, sorting, and filtering.",
"dependencies": {
"@aiostreams/core": "^0.0.0",
"@aiostreams/core": "workspace:*",
"express": "^5.1.0",
"express-rate-limit": "^8.0.1",
"rate-limit-redis": "^4.2.2"
"rate-limit-redis": "^4.2.2",
"zod": "^4.1.5"
},
"devDependencies": {
"@types/express": "^5.0.1",
+14 -4
View File
@@ -1,5 +1,4 @@
import express, { Request, Response } from 'express';
import express, { Request, Response, Express } from 'express';
import {
userApi,
healthApi,
@@ -10,6 +9,7 @@ import {
gdriveApi,
debridApi,
searchApi,
animeApi,
} from './routes/api';
import {
configure,
@@ -21,7 +21,13 @@ import {
addonCatalog,
alias,
} from './routes/stremio';
import { gdrive, torboxSearch } from './routes/builtins';
import {
gdrive,
torboxSearch,
torznab,
newznab,
prowlarr,
} from './routes/builtins';
import {
ipMiddleware,
loggerMiddleware,
@@ -38,7 +44,7 @@ import { StremioTransformer } from '@aiostreams/core';
import { createResponse } from './utils/responses';
import path from 'path';
import fs from 'fs';
const app = express();
const app: Express = express();
const logger = createLogger('server');
export const frontendRoot = path.join(__dirname, '../../frontend/out');
@@ -78,6 +84,7 @@ apiRouter.use('/debrid', debridApi);
if (Env.ENABLE_SEARCH_API) {
apiRouter.use('/search', searchApi);
}
apiRouter.use('/anime', animeApi);
app.use(`/api/v${constants.API_VERSION}`, apiRouter);
// Stremio Routes
@@ -115,6 +122,9 @@ const builtinsRouter = express.Router();
builtinsRouter.use(internalMiddleware);
builtinsRouter.use('/gdrive', gdrive);
builtinsRouter.use('/torbox-search', torboxSearch);
builtinsRouter.use('/torznab', torznab);
builtinsRouter.use('/newznab', newznab);
builtinsRouter.use('/prowlarr', prowlarr);
app.use('/builtins', builtinsRouter);
app.get(
+10 -1
View File
@@ -24,6 +24,15 @@ const isIpInRange = (ip: string, range: string) => {
return ip === range;
};
const isPrivateIp = (ip?: string) => {
if (!ip) {
return false;
}
return /^(10\.|(::ffff)?127\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|::1)/.test(
ip
);
};
export const ipMiddleware = (
req: Request,
res: Response,
@@ -52,7 +61,7 @@ export const ipMiddleware = (
req.get('CF-Connecting-IP') ||
ip
: ip;
req.userIp = userIp;
req.userIp = isPrivateIp(userIp) ? undefined : userIp;
req.requestIp = requestIp;
next();
};
@@ -81,6 +81,12 @@ const catalogApiRateLimiter = createRateLimiter(
'catalog-api'
);
const animeApiRateLimiter = createRateLimiter(
Env.ANIME_API_RATE_LIMIT_WINDOW * 1000,
Env.ANIME_API_RATE_LIMIT_MAX_REQUESTS,
'anime-api'
);
const stremioStreamRateLimiter = createRateLimiter(
Env.STREMIO_STREAM_RATE_LIMIT_WINDOW * 1000,
Env.STREMIO_STREAM_RATE_LIMIT_MAX_REQUESTS,
@@ -122,6 +128,7 @@ export {
streamApiRateLimiter,
formatApiRateLimiter,
catalogApiRateLimiter,
animeApiRateLimiter,
stremioStreamRateLimiter,
stremioCatalogRateLimiter,
stremioManifestRateLimiter,
+63
View File
@@ -0,0 +1,63 @@
import { Router, Request, Response, NextFunction } from 'express';
import { createResponse } from '../../utils/responses';
import {
APIError,
constants,
createLogger,
formatZodError,
AnimeDatabase,
IdType,
ID_TYPES,
} from '@aiostreams/core';
import { z, ZodError } from 'zod';
import { animeApiRateLimiter } from '../../middlewares/ratelimit';
const router: Router = Router();
const logger = createLogger('server');
router.use(animeApiRateLimiter);
router.get('/', async (req: Request, res: Response, next: NextFunction) => {
let idType: IdType;
let idValue: string | number;
try {
const { idType: idTypeParam, idValue: idValueParam } = z
.object({
idType: z.enum(ID_TYPES),
idValue: z.union([z.string(), z.number()]),
})
.parse(req.query);
idType = idTypeParam;
idValue = idValueParam;
} catch (error: any) {
if (error instanceof ZodError) {
next(
new APIError(
constants.ErrorCode.BAD_REQUEST,
400,
formatZodError(error)
)
);
return;
}
next(new APIError(constants.ErrorCode.BAD_REQUEST, error.message));
return;
}
try {
const mappingEntry = AnimeDatabase.getInstance().getEntryById(
idType,
idValue
);
res
.status(200)
.json(
createResponse({ success: true, detail: 'OK', data: mappingEntry })
);
} catch (error: any) {
logger.error(`Mapping check failed: ${error.message}`);
next(
new APIError(constants.ErrorCode.INTERNAL_SERVER_ERROR, error.message)
);
}
});
export default router;
+1 -1
View File
@@ -10,7 +10,7 @@ import {
constants,
} from '@aiostreams/core';
import { catalogApiRateLimiter } from '../../middlewares/ratelimit';
const router = Router();
const router: Router = Router();
const logger = createLogger('server');
router.use(catalogApiRateLimiter);
+15 -21
View File
@@ -6,13 +6,12 @@ import {
formatZodError,
} from '@aiostreams/core';
import {
DebridInterface,
DebridError,
PlaybackInfoSchema,
StoreAuthSchema,
getDebridService,
ServiceAuthSchema,
} from '@aiostreams/core';
import { ZodError } from 'zod';
import { StremThruError } from 'stremthru';
import {
STATIC_DOWNLOAD_FAILED,
STATIC_DOWNLOADING,
@@ -23,7 +22,7 @@ import {
STATIC_UNAUTHORIZED,
STATIC_NO_MATCHING_FILE,
} from '../../app';
const router = Router();
const router: Router = Router();
const logger = createLogger('server');
// block HEAD requests
@@ -36,7 +35,7 @@ router.use((req: Request, res: Response, next: NextFunction) => {
});
router.get(
'/resolve/:encodedStoreAuth/:encodedPlaybackInfo/:filename',
'/playback/:encodedStoreAuth/:encodedPlaybackInfo/:filename',
async (req: Request, res: Response, next: NextFunction) => {
try {
const { encodedStoreAuth, encodedPlaybackInfo, filename } = req.params;
@@ -51,22 +50,25 @@ router.get(
JSON.parse(Buffer.from(encodedPlaybackInfo, 'base64').toString('utf-8'))
);
const storeAuth = StoreAuthSchema.parse(
const storeAuth = ServiceAuthSchema.parse(
JSON.parse(Buffer.from(encodedStoreAuth, 'base64').toString('utf-8'))
);
const debridInterface = new DebridInterface(storeAuth, req.userIp);
const debridInterface = getDebridService(
storeAuth.id,
storeAuth.credential,
req.userIp
);
let streamUrl: string | undefined;
try {
streamUrl = await debridInterface.resolve(playbackInfo, filename);
} catch (error: any) {
let staticFile: string = STATIC_INTERNAL_SERVER_ERROR;
if (error instanceof StremThruError) {
if (error instanceof DebridError) {
logger.error(
`Got StremThru error during debrid resolve: ${error.code}: ${error.message}`
`Got Debrid error during debrid resolve: ${error.code}: ${error.message}`
);
switch (error.code) {
case 'UNAVAILABLE_FOR_LEGAL_REASONS':
staticFile = STATIC_UNAVAILABLE_FOR_LEGAL_REASONS;
@@ -85,14 +87,6 @@ router.get(
case 'STORE_MAGNET_INVALID':
staticFile = STATIC_DOWNLOAD_FAILED;
break;
default:
break;
}
} else if (error instanceof DebridError) {
logger.error(
`Got Debrid error during debrid resolve: ${error.code}: ${error.message}`
);
switch (error.code) {
case 'NO_MATCHING_FILE':
staticFile = STATIC_NO_MATCHING_FILE;
break;
@@ -116,9 +110,6 @@ router.get(
res.status(307).redirect(streamUrl);
} catch (error: any) {
logger.error(
`Got unexpected error during debrid resolve: ${error.message}`
);
if (error instanceof APIError) {
next(error);
} else if (error instanceof ZodError) {
@@ -130,6 +121,9 @@ router.get(
)
);
} else {
logger.error(
`Got unexpected error during debrid resolve: ${error.message}`
);
next(
new APIError(
constants.ErrorCode.INTERNAL_SERVER_ERROR,
+18 -4
View File
@@ -1,6 +1,11 @@
import { Router, Request, Response, NextFunction } from 'express';
import { createResponse } from '../../utils/responses';
import { createLogger, UserData, UserDataSchema } from '@aiostreams/core';
import {
createLogger,
UserData,
UserDataSchema,
formatZodError,
} from '@aiostreams/core';
import {
createFormatter,
ParsedStreamSchema,
@@ -8,7 +13,8 @@ import {
} from '@aiostreams/core';
import * as constants from '@aiostreams/core';
import { formatApiRateLimiter } from '../../middlewares/ratelimit';
const router = Router();
const router: Router = Router();
router.use(formatApiRateLimiter);
@@ -23,7 +29,11 @@ router.post('/', (req: Request, res: Response) => {
} = ParsedStreamSchema.safeParse(stream);
if (!streamSuccess) {
logger.error('Invalid stream', { error: streamError });
throw new APIError(constants.ErrorCode.FORMAT_INVALID_STREAM);
throw new APIError(
constants.ErrorCode.FORMAT_INVALID_STREAM,
400,
formatZodError(streamError)
);
}
const {
success: userDataSuccess,
@@ -32,7 +42,11 @@ router.post('/', (req: Request, res: Response) => {
} = UserDataSchema.safeParse(userData);
if (!userDataSuccess) {
logger.error('Invalid user data', { error: userDataError });
throw new APIError(constants.ErrorCode.FORMAT_INVALID_FORMATTER);
throw new APIError(
constants.ErrorCode.FORMAT_INVALID_FORMATTER,
400,
formatZodError(userDataError)
);
}
const formattedStream = createFormatter(userDataData).format(streamData);
res
+1 -1
View File
@@ -2,7 +2,7 @@ import { Router, Request, Response, NextFunction } from 'express';
import { createResponse } from '../../utils/responses';
import { APIError, constants, createLogger, GDriveAPI } from '@aiostreams/core';
import { GoogleOAuth } from '@aiostreams/core';
const router = Router();
const router: Router = Router();
const logger = createLogger('server');
router.post('/', async (req: Request, res: Response, next: NextFunction) => {
+1 -1
View File
@@ -6,7 +6,7 @@ import {
createLogger,
UserRepository,
} from '@aiostreams/core';
const router = Router();
const router: Router = Router();
const logger = createLogger('server');
router.get('/', async (req: Request, res: Response, next: NextFunction) => {
+1
View File
@@ -7,3 +7,4 @@ export { default as rpdbApi } from './rpdb';
export { default as gdriveApi } from './gdrive';
export { default as debridApi } from './debrid';
export { default as searchApi } from './search';
export { default as animeApi } from './anime';
+3 -2
View File
@@ -9,7 +9,7 @@ import { RPDB } from '@aiostreams/core';
import { createResponse } from '../../utils/responses';
import { z } from 'zod';
const router = Router();
const router: Router = Router();
const logger = createLogger('server');
const searchParams = z.object({
@@ -38,13 +38,14 @@ router.get('/', async (req: Request, res: Response, next: NextFunction) => {
const { id, type, fallback, apiKey } = data;
const rpdb = new RPDB(apiKey);
const posterUrl = (await rpdb.getPosterUrl(type, id)) || fallback;
if (!(posterUrl && fallback)) {
if (!posterUrl) {
res.status(404).json(
createResponse({
success: false,
detail: 'Not found',
})
);
return;
}
res.redirect(301, posterUrl!);
} catch (error: any) {
+1 -1
View File
@@ -15,7 +15,7 @@ import { createLogger } from '@aiostreams/core';
import { ApiTransformer, ApiSearchResponseData } from '@aiostreams/core';
import { ApiResponse, createResponse } from '../../utils/responses';
import { z, ZodError } from 'zod';
const router = Router();
const router: Router = Router();
const logger = createLogger('server');
+1 -1
View File
@@ -10,7 +10,7 @@ import { encryptString } from '@aiostreams/core';
import { FeatureControl } from '@aiostreams/core';
import { createResponse } from '../../utils/responses';
const router = Router();
const router: Router = Router();
const statusInfo = async (): Promise<StatusResponse> => {
const userCount = await UserRepository.getUserCount();
+1 -1
View File
@@ -8,7 +8,7 @@ import {
} from '@aiostreams/core';
import { userApiRateLimiter } from '../../middlewares/ratelimit';
import { createResponse } from '../../utils/responses';
const router = Router();
const router: Router = Router();
const logger = createLogger('server');
@@ -2,12 +2,10 @@ import { Router, Request, Response, NextFunction } from 'express';
import { AIOStreams, AIOStreamResponse, GDriveAddon } from '@aiostreams/core';
import { stremioStreamRateLimiter } from '../../middlewares/ratelimit';
import { createLogger } from '@aiostreams/core';
const router = Router();
const router: Router = Router();
const logger = createLogger('server');
router.use(stremioStreamRateLimiter);
router.get(
'{/:encodedConfig}/manifest.json',
async (req: Request, res: Response, next: NextFunction) => {
@@ -1,2 +1,5 @@
export { default as gdrive } from './gdrive';
export { default as torboxSearch } from './torbox-search';
export { default as torznab } from './torznab';
export { default as newznab } from './newznab';
export { default as prowlarr } from './prowlarr';
@@ -0,0 +1,45 @@
import { Router, Request, Response, NextFunction } from 'express';
import { NewznabAddon } from '@aiostreams/core';
import { createLogger } from '@aiostreams/core';
const router: Router = Router();
const logger = createLogger('server');
router.get(
'/:encodedConfig/manifest.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig } = req.params;
const config = encodedConfig
? JSON.parse(Buffer.from(encodedConfig, 'base64').toString('utf-8'))
: undefined;
try {
const manifest = new NewznabAddon(config, req.userIp).getManifest();
res.json(manifest);
} catch (error) {
next(error);
}
}
);
router.get(
'/:encodedConfig/stream/:type/:id.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig, type, id } = req.params;
const config = JSON.parse(
Buffer.from(encodedConfig, 'base64').toString('utf-8')
);
try {
const addon = new NewznabAddon(config, req.userIp);
const streams = await addon.getStreams(type, id);
res.json({
streams: streams,
});
} catch (error) {
next(error);
}
}
);
export default router;
@@ -0,0 +1,46 @@
import { Router, Request, Response, NextFunction } from 'express';
import { AIOStreams, AIOStreamResponse, ProwlarrAddon } from '@aiostreams/core';
import { stremioStreamRateLimiter } from '../../middlewares/ratelimit';
import { createLogger } from '@aiostreams/core';
const router: Router = Router();
const logger = createLogger('server');
router.get(
'/:encodedConfig/manifest.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig } = req.params;
const config = encodedConfig
? JSON.parse(Buffer.from(encodedConfig, 'base64').toString('utf-8'))
: undefined;
try {
const manifest = new ProwlarrAddon(config, req.userIp).getManifest();
res.json(manifest);
} catch (error) {
next(error);
}
}
);
router.get(
'/:encodedConfig/stream/:type/:id.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig, type, id } = req.params;
const config = JSON.parse(
Buffer.from(encodedConfig, 'base64').toString('utf-8')
);
try {
const addon = new ProwlarrAddon(config, req.userIp);
const streams = await addon.getStreams(type, id);
res.json({
streams: streams,
});
} catch (error) {
next(error);
}
}
);
export default router;
@@ -6,7 +6,7 @@ import {
TorBoxSearchAddonError,
} from '@aiostreams/core';
import { createResponse } from '../../utils/responses';
const router = Router();
const router: Router = Router();
const logger = createLogger('builtins:torbox-search');

Some files were not shown because too many files have changed in this diff Show More