From dc50c6c70b94df7cc0124bbc8b2f96df01011b38 Mon Sep 17 00:00:00 2001 From: Viren070 Date: Tue, 17 Jun 2025 20:03:08 +0100 Subject: [PATCH] feat: add webstreamr, improve parsing of nuviostream results, validate tmdb access token, always check for languages --- packages/core/src/db/schemas.ts | 4 +- packages/core/src/parser/regex.ts | 6 +- packages/core/src/parser/streams.ts | 15 +- packages/core/src/presets/nuviostreams.ts | 73 +++++-- packages/core/src/presets/presetManager.ts | 4 + packages/core/src/presets/webstreamr.ts | 191 ++++++++++++++++++ packages/core/src/utils/config.ts | 7 +- packages/core/src/utils/env.ts | 13 ++ packages/core/src/utils/languages.ts | 2 +- packages/core/src/utils/metadata.ts | 28 ++- .../frontend/src/components/menu/addons.tsx | 7 +- 11 files changed, 323 insertions(+), 27 deletions(-) create mode 100644 packages/core/src/presets/webstreamr.ts diff --git a/packages/core/src/db/schemas.ts b/packages/core/src/db/schemas.ts index 305d8e0f..bc07be19 100644 --- a/packages/core/src/db/schemas.ts +++ b/packages/core/src/db/schemas.ts @@ -766,7 +766,7 @@ const PresetMetadataSchema = z.object({ disabled: z.boolean(), }) .optional(), - LOGO: z.string(), + LOGO: z.string().optional(), DESCRIPTION: z.string(), URL: z.string(), TIMEOUT: z.number(), @@ -780,7 +780,7 @@ const PresetMetadataSchema = z.object({ const PresetMinimalMetadataSchema = z.object({ ID: z.string(), NAME: z.string(), - LOGO: z.string(), + LOGO: z.string().optional(), DESCRIPTION: z.string(), URL: z.string(), DISABLED: z diff --git a/packages/core/src/parser/regex.ts b/packages/core/src/parser/regex.ts index 5492ad10..ccdee0a2 100644 --- a/packages/core/src/parser/regex.ts +++ b/packages/core/src/parser/regex.ts @@ -52,11 +52,11 @@ export const PARSE_REGEX: PARSE_REGEX = { '1080p': createRegex( '(bd|hd|m)?(1080(p|i)?)|f(ull)?[ .\\-_]?hd|1920\s?x\s?(\d{3,4})' ), - '720p': createRegex('(bd|hd|m)?(720(p|i)?)|hd|1280\s?x\s?(\d{3,4})'), - '576p': createRegex('(bd|hd|m)?(576(p|i)?)'), + '720p': createRegex('(bd|hd|m)?((720|800)(p|i)?)|hd|1280\s?x\s?(\d{3,4})'), + '576p': createRegex('(bd|hd|m)?((576|534)(p|i)?)'), '480p': createRegex('(bd|hd|m)?(480(p|i)?)|sd'), '360p': createRegex('(bd|hd|m)?(360(p|i)?)'), - '240p': createRegex('(bd|hd|m)?(240(p|i)?)'), + '240p': createRegex('(bd|hd|m)?((240|266)(p|i)?)'), '144p': createRegex('(bd|hd|m)?(144(p|i)?)'), }, qualities: { diff --git a/packages/core/src/parser/streams.ts b/packages/core/src/parser/streams.ts index 599e14f3..d8ed3d66 100644 --- a/packages/core/src/parser/streams.ts +++ b/packages/core/src/parser/streams.ts @@ -93,15 +93,18 @@ class StreamParser { parsedStream.age = this.getAge(stream, parsedStream); parsedStream.message = this.getMessage(stream, parsedStream); - if (parsedStream.filename) { - parsedStream.parsedFile = FileParser.parse(parsedStream.filename); - parsedStream.parsedFile.languages = Array.from( + parsedStream.parsedFile = { + visualTags: [], + audioTags: [], + audioChannels: [], + ...(parsedStream.filename ? FileParser.parse(parsedStream.filename) : {}), + languages: Array.from( new Set([ - ...parsedStream.parsedFile.languages, + ...(parsedStream.parsedFile?.languages ?? []), ...this.getLanguages(stream, parsedStream), ]) - ); - } + ), + }; if (parsedStream.folderName && parsedStream.parsedFile) { const parsedFolder = FileParser.parse(parsedStream.folderName); diff --git a/packages/core/src/presets/nuviostreams.ts b/packages/core/src/presets/nuviostreams.ts index 0cbb5dcd..bb223d20 100644 --- a/packages/core/src/presets/nuviostreams.ts +++ b/packages/core/src/presets/nuviostreams.ts @@ -1,10 +1,69 @@ -import { Addon, Option, UserData, Resource, Stream } from '../db'; +import { Addon, Option, UserData, Resource, Stream, ParsedStream } from '../db'; import { Preset, baseOptions } from './preset'; import { Env, SERVICE_DETAILS } from '../utils'; import { constants, ServiceId } from '../utils'; -import { StreamParser } from '../parser'; +import { FileParser, StreamParser } from '../parser'; + +class NuvioStreamsStreamParser extends StreamParser { + parse(stream: Stream): ParsedStream { + let parsedStream: ParsedStream = { + id: this.getRandomId(), + addon: this.addon, + type: 'http', + url: this.applyUrlModifications(stream.url ?? undefined), + externalUrl: stream.externalUrl ?? undefined, + ytId: stream.ytId ?? undefined, + requestHeaders: stream.behaviorHints?.proxyHeaders?.request, + responseHeaders: stream.behaviorHints?.proxyHeaders?.response, + notWebReady: stream.behaviorHints?.notWebReady ?? undefined, + videoHash: stream.behaviorHints?.videoHash ?? undefined, + originalName: stream.name ?? undefined, + originalDescription: (stream.description || stream.title) ?? undefined, + }; + + stream.description = stream.description || stream.title; + + parsedStream.type = this.getStreamType( + stream, + parsedStream.service, + parsedStream + ); + + parsedStream.parsedFile = FileParser.parse( + `${stream.name}\n${stream.description}` + ); + parsedStream.filename = stream.description?.split('\n')[0]; + parsedStream.folderName = undefined; + + parsedStream.message = stream.name + ?.replace(/\d+p?/gi, '') + ?.trim() + ?.replace(/-$/, '') + ?.trim(); + + if (stream.description?.split('\n')?.[-1]?.includes('⚠️')) { + parsedStream.message += `\n${stream.description?.split('\n')?.[-1]}`; + } + + parsedStream.torrent = { + infoHash: + parsedStream.type === 'p2p' + ? (stream.infoHash ?? undefined) + : this.getInfoHash(stream, parsedStream), + seeders: this.getSeeders(stream, parsedStream), + sources: stream.sources ?? undefined, + fileIdx: stream.fileIdx ?? undefined, + }; + + return parsedStream; + } +} export class NuvioStreamsPreset extends Preset { + static override getParser(): typeof StreamParser { + return NuvioStreamsStreamParser; + } + static override get METADATA() { const supportedResources = [constants.STREAM_RESOURCE]; const regions = [ @@ -130,15 +189,6 @@ export class NuvioStreamsPreset extends Preset { options: providers, default: providers.map((provider) => provider.value), }, - { - id: 'streamPassthrough', - name: 'Stream Passthrough', - description: - 'Whether to use the original stream name and description. Recommended to be left on in order to get all the information.', - type: 'boolean', - required: false, - default: true, - }, { id: 'socials', name: '', @@ -182,7 +232,6 @@ export class NuvioStreamsPreset extends Preset { identifyingName: options.name || this.METADATA.NAME, manifestUrl: this.generateManifestUrl(userData, options), enabled: true, - streamPassthrough: options.streamPassthrough ?? true, resources: options.resources || this.METADATA.SUPPORTED_RESOURCES, timeout: options.timeout || this.METADATA.TIMEOUT, presetType: this.METADATA.ID, diff --git a/packages/core/src/presets/presetManager.ts b/packages/core/src/presets/presetManager.ts index fef9e426..b849da8d 100644 --- a/packages/core/src/presets/presetManager.ts +++ b/packages/core/src/presets/presetManager.ts @@ -32,6 +32,7 @@ import { TorrentCatalogsPreset } from './torrentCatalogs'; import { StreamingCatalogsPreset } from './streamingCatalogs'; import { AnimeCatalogsPreset } from './animeCatalogs'; import { DoctorWhoUniversePreset } from './doctorWhoUniverse'; +import { WebStreamrPreset } from './webstreamr'; const PRESET_LIST: string[] = [ 'custom', @@ -47,6 +48,7 @@ const PRESET_LIST: string[] = [ 'easynewsPlus', 'easynewsPlusPlus', 'nuvio-streams', + 'webstreamr', 'debridio', 'debridio-tv', 'debridio-watchtower', @@ -138,6 +140,8 @@ export class PresetManager { return AnimeKitsuPreset; case 'nuvio-streams': return NuvioStreamsPreset; + case 'webstreamr': + return WebStreamrPreset; case 'streaming-catalogs': return StreamingCatalogsPreset; case 'anime-catalogs': diff --git a/packages/core/src/presets/webstreamr.ts b/packages/core/src/presets/webstreamr.ts new file mode 100644 index 00000000..6e58987c --- /dev/null +++ b/packages/core/src/presets/webstreamr.ts @@ -0,0 +1,191 @@ +import { + Addon, + Option, + UserData, + Resource, + Stream, + ParsedStream, + PresetMinimalMetadata, + PresetMetadata, +} from '../db'; +import { Preset, baseOptions } from './preset'; +import { Env, SERVICE_DETAILS } from '../utils'; +import { constants, ServiceId } from '../utils'; +import { FileParser, StreamParser } from '../parser'; + +class WebStreamrStreamParser extends StreamParser { + protected get indexerEmojis(): string[] { + return ['🔗']; + } + + protected override getMessage( + stream: Stream, + currentParsedStream: ParsedStream + ): string | undefined { + const messageRegex = this.getRegexForTextAfterEmojis(['🐢']); + const message = stream.description?.match(messageRegex)?.[1]; + return message; + } + + protected override getFilename( + stream: Stream, + currentParsedStream: ParsedStream + ): string | undefined { + let filename = undefined; + const resolution = stream.name?.match(/\d+p?/i)?.[0]; + if (stream.description?.split('\n')?.[0]?.includes('📂')) { + filename = stream.description + ?.split('\n')?.[0] + ?.replace('📂', '') + ?.trim(); + } + + const str = `${filename ? filename + ' ' : ''}${resolution ? resolution : ''}`; + return str ? str : undefined; + } +} + +export class WebStreamrPreset extends Preset { + static override getParser(): typeof StreamParser { + return WebStreamrStreamParser; + } + + static override get METADATA(): PresetMetadata { + const supportedResources = [constants.STREAM_RESOURCE]; + /** + * German 🇩🇪 (KinoGer, MeineCloud, StreamKiste) +English 🇺🇸 (Soaper, VidSrc) +Castilian Spanish 🇪🇸 (CineHDPlus, Cuevana, VerHdLink) +French 🇫🇷 (Frembed, FrenchCloud) +Italian 🇮🇹 (Eurostreaming, MostraGuarda) +Latin American Spanish 🇲🇽 (CineHDPlus, Cuevana, VerHdLink) +Exclude external URLs from results + + +{"de":"on","en":"on","es":"on","fr":"on","it":"on","mx":"on","excludeExternalUrls":"on"} + */ + const providers = [ + { + label: '🇺🇸 English (Soaper, VidSrc)', + value: 'en', + }, + { + label: '🇩🇪 German (KinoGer, MeineCloud, StreamKiste)', + value: 'de', + }, + { + label: '🇪🇸 Castilian Spanish (CineHDPlus, Cuevana, VerHdLink)', + value: 'es', + }, + { + label: '🇫🇷 French (Frembed, FrenchCloud)', + value: 'fr', + }, + { + label: '🇮🇹 Italian (Eurostreaming, MostraGuarda)', + value: 'it', + }, + { + label: '🇲🇽 Latin American Spanish (CineHDPlus, Cuevana, VerHdLink)', + value: 'mx', + }, + ]; + const options: Option[] = [ + ...baseOptions( + 'WebStreamr', + supportedResources, + Env.DEFAULT_WEBSTREAMR_TIMEOUT + ), + { + id: 'providers', + name: 'Providers', + description: 'Select the providers to use', + type: 'multi-select', + options: providers, + default: ['en'], + }, + { + id: 'excludeExternalUrls', + name: 'Exclude External URLs', + description: 'Exclude external URLs from results', + type: 'boolean', + default: false, + }, + { + id: 'socials', + name: '', + description: '', + type: 'socials', + socials: [ + { id: 'github', url: 'https://github.com/webstreamr/webstreamr' }, + ], + }, + ]; + + return { + ID: 'webstreamr', + NAME: 'WebStreamr', + URL: Env.WEBSTREAMR_URL, + TIMEOUT: Env.DEFAULT_WEBSTREAMR_TIMEOUT || Env.DEFAULT_TIMEOUT, + USER_AGENT: Env.DEFAULT_WEBSTREAMR_USER_AGENT || Env.DEFAULT_USER_AGENT, + SUPPORTED_SERVICES: [], + DESCRIPTION: 'Provides HTTP URLs from streaming websites.', + OPTIONS: options, + SUPPORTED_STREAM_TYPES: [constants.HTTP_STREAM_TYPE], + SUPPORTED_RESOURCES: supportedResources, + }; + } + + static async generateAddons( + userData: UserData, + options: Record + ): Promise { + return [this.generateAddon(userData, options)]; + } + + private static generateAddon( + userData: UserData, + options: Record + ): Addon { + return { + name: options.name || this.METADATA.NAME, + identifyingName: options.name || this.METADATA.NAME, + manifestUrl: this.generateManifestUrl(userData, options), + enabled: true, + streamPassthrough: false, + resources: options.resources || this.METADATA.SUPPORTED_RESOURCES, + timeout: options.timeout || this.METADATA.TIMEOUT, + presetType: this.METADATA.ID, + presetInstanceId: '', + headers: { + 'User-Agent': this.METADATA.USER_AGENT, + }, + }; + } + + private static generateManifestUrl( + userData: UserData, + options: Record + ) { + let url = options.url || this.METADATA.URL; + if (url.endsWith('/manifest.json')) { + return url; + } + + url = url.replace(/\/$/, ''); + + const checkedOptions = [ + ...(options.providers || []), + options.excludeExternalUrls ?? undefined, + ].filter(Boolean); + + const config = this.urlEncodeJSON({ + ...checkedOptions.reduce((acc, option) => { + acc[option] = 'on'; + return acc; + }, {}), + }); + + return `${url}${config ? '/' + config : ''}/manifest.json`; + } +} diff --git a/packages/core/src/utils/config.ts b/packages/core/src/utils/config.ts index 38d6f1d9..b1fa74df 100644 --- a/packages/core/src/utils/config.ts +++ b/packages/core/src/utils/config.ts @@ -10,7 +10,7 @@ import { import { AIOStreams } from '../main'; import { Preset, PresetManager } from '../presets'; import { createProxy } from '../proxy'; -import { constants } from '.'; +import { constants, TMDBMetadata } from '.'; import { isEncrypted, decryptString, encryptString } from './crypto'; import { Env } from './env'; import { createLogger, maskSensitiveInfo } from './logger'; @@ -344,6 +344,11 @@ export async function validateConfig( } } + if (config.titleMatching?.enabled === true) { + const tmdb = new TMDBMetadata(config.tmdbAccessToken); + await tmdb.validateAccessToken(); + } + if (FeatureControl.disabledServices.size > 0) { for (const service of config.services ?? []) { if (FeatureControl.disabledServices.has(service.id)) { diff --git a/packages/core/src/utils/env.ts b/packages/core/src/utils/env.ts index 1c354d13..9226bb43 100644 --- a/packages/core/src/utils/env.ts +++ b/packages/core/src/utils/env.ts @@ -1036,6 +1036,19 @@ export const Env = cleanEnv(process.env, { desc: 'Default Doctor Who Universe user agent', }), + WEBSTREAMR_URL: url({ + default: 'https://webstreamr.hayd.uk', + desc: 'WebStreamr URL', + }), + DEFAULT_WEBSTREAMR_TIMEOUT: num({ + default: undefined, + desc: 'Default WebStreamr timeout', + }), + DEFAULT_WEBSTREAMR_USER_AGENT: userAgent({ + default: undefined, + desc: 'Default WebStreamr user agent', + }), + // Rate limiting settings DISABLE_RATE_LIMITS: bool({ default: false, diff --git a/packages/core/src/utils/languages.ts b/packages/core/src/utils/languages.ts index cad84a9f..4c3f1ec9 100644 --- a/packages/core/src/utils/languages.ts +++ b/packages/core/src/utils/languages.ts @@ -1692,7 +1692,7 @@ export const FULL_LANGUAGE_MAPPING = [ iso_639_2: 'spa', iso_3166_1: 'MX', flag: '🇲🇽', - english_name: 'Spanish (Mexico)', + english_name: 'Latino', name: 'Español', }, { diff --git a/packages/core/src/utils/metadata.ts b/packages/core/src/utils/metadata.ts index 5ae157d1..254f7d58 100644 --- a/packages/core/src/utils/metadata.ts +++ b/packages/core/src/utils/metadata.ts @@ -18,6 +18,7 @@ const ALTERNATIVE_TITLES_PATH = '/alternative_titles'; // Cache TTLs in seconds const ID_CACHE_TTL = 24 * 60 * 60; // 24 hours const TITLE_CACHE_TTL = 7 * 24 * 60 * 60; // 7 days +const ACCESS_TOKEN_CACHE_TTL = 2 * 24 * 60 * 60; // 2 day export interface Metadata { titles: string[]; @@ -31,7 +32,7 @@ export class TMDBMetadata { private readonly idCache: Cache; private readonly metadataCache: Cache; private readonly accessToken: string; - + private readonly validationCache: Cache; public constructor(accessToken?: string) { if (!accessToken && !Env.TMDB_ACCESS_TOKEN) { throw new Error('TMDB Access Token is not set'); @@ -39,6 +40,9 @@ export class TMDBMetadata { this.accessToken = (accessToken || Env.TMDB_ACCESS_TOKEN)!; this.idCache = Cache.getInstance('tmdb_id_conversion'); this.metadataCache = Cache.getInstance('tmdb_metadata'); + this.validationCache = Cache.getInstance( + 'tmdb_validation' + ); } private getHeaders(): Record { @@ -194,4 +198,26 @@ export class TMDBMetadata { this.metadataCache.set(cacheKey, metadata, TITLE_CACHE_TTL); return metadata; } + + public async validateAccessToken() { + const cacheKey = this.accessToken; + const cachedResult = this.validationCache.get(cacheKey); + if (cachedResult) { + return cachedResult; + } + const url = new URL(API_BASE_URL + '/authentication'); + const validationResponse = await fetch(url, { + headers: this.getHeaders(), + signal: AbortSignal.timeout(10000), + }); + if (!validationResponse.ok) { + throw new Error( + `Failed to validate TMDB access token: ${validationResponse.statusText}` + ); + } + const validationData = await validationResponse.json(); + const isValid = validationData.success; + this.validationCache.set(cacheKey, isValid, ACCESS_TOKEN_CACHE_TTL); + return isValid; + } } diff --git a/packages/frontend/src/components/menu/addons.tsx b/packages/frontend/src/components/menu/addons.tsx index 8c8690fc..443f3064 100644 --- a/packages/frontend/src/components/menu/addons.tsx +++ b/packages/frontend/src/components/menu/addons.tsx @@ -62,6 +62,7 @@ import { } from '../ui/accordion'; import { FaArrowRightLong, FaRankingStar, FaShuffle } from 'react-icons/fa6'; import { PiStarFill, PiStarBold } from 'react-icons/pi'; +import { IoExtensionPuzzle } from 'react-icons/io5'; interface CatalogModification { id: string; @@ -610,12 +611,16 @@ function AddonCard({
- ) : ( + ) : preset.LOGO ? ( {preset.NAME} + ) : ( +
+ +
)}
{preset.NAME}