Viren070/AIOStreams
1
0
Fork 0
mirror of https://github.com/Viren070/AIOStreams.git synced 2025-12-01 23:14:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: use urlsafe encoded configs, add requiredFields, adjust playback URL handling, and sql cache store.

Browse Source
This commit is contained in:
Viren070
2025-09-29 19:38:57 +01:00
parent 5e3f6fb188
commit 9a9d3d2775
37 changed files with 600 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/src/builtins/base/debrid.ts
+10 -10
View File
@@ -20,6 +20,7 @@ import {
UnprocessedTorrent,
ServiceAuth,
DebridError,
generatePlaybackUrl,
} from '../../debrid/index.js';
import { processTorrents, processNZBs } from '../utils/debrid.js';
import { calculateAbsoluteEpisode } from '../utils/general.js';
@@ -211,11 +212,10 @@ export abstract class BaseDebridAddon<T extends BaseDebridConfig> {
),
]);
const resultStreams = [
...processedTorrents.results,
...processedNzbs.results,
].map((result) =>
this._createStream(result, this.userData, searchMetadata)
const resultStreams = await Promise.all(
[...processedTorrents.results, ...processedNzbs.results].map((result) =>
this._createStream(result, this.userData, searchMetadata)
)
);
const processingErrors = [
@@ -470,11 +470,11 @@ export abstract class BaseDebridAddon<T extends BaseDebridConfig> {
return {
url: torrentOrNzb.service
? `${Env.BASE_URL}/api/v1/debrid/playback/${encodeURIComponent(
Buffer.from(JSON.stringify(storeAuth)).toString('base64')
)}/${encodeURIComponent(
Buffer.from(JSON.stringify(playbackInfo)).toString('base64')
)}/${encodeURIComponent(torrentOrNzb.file.name || torrentOrNzb.title || 'unknown')}`
? generatePlaybackUrl(
storeAuth!,
playbackInfo!,
torrentOrNzb.file.name || torrentOrNzb.title || 'unknown'
)
: undefined,
name,
description,
packages/core/src/builtins/torbox-search/source-handlers.ts
+12 -4
View File
@@ -23,6 +23,7 @@ import { processNZBs, processTorrents } from '../utils/debrid.js';
import {
NZBWithSelectedFile,
TorrentWithSelectedFile,
generatePlaybackUrl,
} from '../../debrid/utils.js';
import { DebridFile, PlaybackInfo } from '../../debrid/index.js';
import { getTraktAliases } from '../../metadata/trakt.js';
@@ -83,9 +84,10 @@ abstract class SourceHandler {
}
const storeAuth = {
id: torrentOrNZB.service.id,
credential: userData.services.find(
(service) => service.id === torrentOrNZB.service!.id
)?.credential,
credential:
userData.services.find(
(service) => service.id === torrentOrNZB.service!.id
)?.credential ?? '',
};
// const playbackInfo: PlaybackInfo = {
@@ -122,7 +124,13 @@ abstract class SourceHandler {
const description = `${torrentOrNZB.title}\n${torrentOrNZB.file.name}\n${torrentOrNZB.indexer ? `🔍 ${torrentOrNZB.indexer}` : ''} ${torrentOrNZB.seeders ? `👤 ${torrentOrNZB.seeders}` : ''} ${torrentOrNZB.age && torrentOrNZB.age !== '0d' ? `🕒 ${torrentOrNZB.age}` : ''}`;
return {
url: `${Env.BASE_URL}/api/v1/debrid/playback/${encodeURIComponent(Buffer.from(JSON.stringify(storeAuth)).toString('base64'))}/${encodeURIComponent(Buffer.from(JSON.stringify(playbackInfo)).toString('base64'))}/${encodeURIComponent(torrentOrNZB.file.name || torrentOrNZB.title || 'unknown')}`,
url: torrentOrNZB.service
? generatePlaybackUrl(
storeAuth!,
playbackInfo!,
torrentOrNZB.file.name || torrentOrNZB.title || 'unknown'
)
: undefined,
name,
description,
type: torrentOrNZB.type,
packages/core/src/db/schemas.ts
+7
View File
@@ -488,6 +488,13 @@ export const TABLES = {
expires_at BIGINT NOT NULL,
result TEXT
`,
cache: `
key TEXT PRIMARY KEY,
value TEXT NOT NULL,
expires_at BIGINT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
last_accessed TIMESTAMP DEFAULT CURRENT_TIMESTAMP
`,
};
const strictManifestResourceSchema = z.object({
packages/core/src/debrid/utils.ts
+41 -2
View File
@@ -1,6 +1,19 @@
import { z } from 'zod';
import { constants, createLogger, BuiltinServiceId } from '../utils/index.js';
import { DebridFile, DebridDownload } from './base.js';
import {
constants,
createLogger,
BuiltinServiceId,
Env,
Cache,
getSimpleTextHash,
encryptString,
} from '../utils/index.js';
import {
DebridFile,
DebridDownload,
PlaybackInfo,
ServiceAuth,
} from './base.js';
import { normaliseTitle, titleMatch } from '../parser/utils.js';
const logger = createLogger('debrid');
@@ -296,3 +309,29 @@ export function isVideoFile(file: DebridFile): boolean {
videoExtensions.some((ext) => file.name?.endsWith(ext) ?? false)
);
}
export const pbiCache = () => {
const prefix = 'pbi';
if (Env.REDIS_URI) {
return Cache.getInstance<string, PlaybackInfo>(
prefix,
1_000_000_000,
'redis'
);
}
return Cache.getInstance<string, PlaybackInfo>(prefix, 1_000_000_000, 'sql');
};
export function generatePlaybackUrl(
storeAuth: ServiceAuth,
playbackInfo: PlaybackInfo,
filename: string
) {
const encryptedStoreAuth = encryptString(JSON.stringify(storeAuth));
if (!encryptedStoreAuth.success) {
throw new Error('Failed to encrypt store auth');
}
const playbackId = getSimpleTextHash(JSON.stringify(playbackInfo));
pbiCache().set(playbackId, playbackInfo, 2 * 24 * 60 * 60);
return `${Env.BASE_URL}/api/v1/debrid/playback/${encryptedStoreAuth.data}/${playbackId}/${encodeURIComponent(filename)}`;
}
packages/core/src/main.ts
+1 -1
View File
@@ -52,7 +52,7 @@ const shuffleCache = Cache.getInstance<string, MetaPreview[]>('shuffle');
const precacheCache = Cache.getInstance<string, boolean>(
'precache',
undefined,
true
'memory'
);
export interface AIOStreamsError {
packages/core/src/parser/go-ptt.ts
+1 -1
View File
@@ -13,7 +13,7 @@ const logger = createLogger('parser');
const parseCache = Cache.getInstance<string, ParseResult | null>(
'ptt',
10000,
true
'memory'
);
class PTT {
packages/core/src/presets/animetosho.ts
+1 -1
View File
@@ -84,7 +84,7 @@ export class AnimeToshoPreset extends TorznabPreset {
apiPath: '/api',
};
const configString = this.base64EncodeJSON(config);
const configString = this.base64EncodeJSON(config, 'urlSafe');
return `${Env.INTERNAL_URL}/builtins/torznab/${configString}/manifest.json`;
}
}
packages/core/src/presets/bitmagnet.ts
+1 -1
View File
@@ -73,7 +73,7 @@ export class BitmagnetPreset extends TorznabPreset {
forceQuerySearch: true,
};
const configString = this.base64EncodeJSON(config);
const configString = this.base64EncodeJSON(config, 'urlSafe');
return `${Env.INTERNAL_URL}/builtins/torznab/${configString}/manifest.json`;
}
}
packages/core/src/presets/gdrive.ts
+12 -9
View File
@@ -188,15 +188,18 @@ export class GDrivePreset extends Preset {
);
}
}
const config = this.base64EncodeJSON({
refreshToken: options.refreshToken,
metadataSource: options.metadataSource || 'imdb',
includeAudioFiles: options.includeAudioFiles ?? false,
tmdbReadAccessToken:
options.metadataSource === 'tmdb'
? userData.tmdbAccessToken || Env.TMDB_ACCESS_TOKEN
: undefined,
});
const config = this.base64EncodeJSON(
{
refreshToken: options.refreshToken,
metadataSource: options.metadataSource || 'imdb',
includeAudioFiles: options.includeAudioFiles ?? false,
tmdbReadAccessToken:
options.metadataSource === 'tmdb'
? userData.tmdbAccessToken || Env.TMDB_ACCESS_TOKEN
: undefined,
},
'urlSafe'
);
return `${this.METADATA.URL}/${config}/manifest.json`;
}
}
packages/core/src/presets/jackett.ts
+1 -1
View File
@@ -134,7 +134,7 @@ export class JackettPreset extends TorznabPreset {
forceQuerySearch: true,
};
const configString = this.base64EncodeJSON(config);
const configString = this.base64EncodeJSON(config, 'urlSafe');
return `${Env.INTERNAL_URL}/builtins/torznab/${configString}/manifest.json`;
}
}
packages/core/src/presets/knaben.ts
+2 -1
View File
@@ -63,7 +63,8 @@ export class KnabenPreset extends TorznabPreset {
options: Record<string, any>
): string {
return `${Env.INTERNAL_URL}/builtins/knaben/${this.base64EncodeJSON(
this.getBaseConfig(userData, services)
this.getBaseConfig(userData, services),
'urlSafe'
)}/manifest.json`;
}
}
packages/core/src/presets/mediafusion.ts
+1 -2
View File
@@ -521,8 +521,7 @@ export class MediaFusionPreset extends Preset {
contribution_streams: options.contributorStreams ?? false,
mdblist_config: null,
},
false,
true
'urlSafe'
);
return encodedUserData;
packages/core/src/presets/newznab.ts
+1 -1
View File
@@ -137,7 +137,7 @@ export class NewznabPreset extends BuiltinAddonPreset {
forceQuerySearch: options.forceQuerySearch ?? false,
};
const configString = this.base64EncodeJSON(config);
const configString = this.base64EncodeJSON(config, 'urlSafe');
return `${this.METADATA.URL}/${configString}/manifest.json`;
}
}
packages/core/src/presets/nzbhydra.ts
+1 -1
View File
@@ -109,7 +109,7 @@ export class NZBHydraPreset extends NewznabPreset {
forceQuerySearch: options.forceQuerySearch ?? true,
};
const configString = this.base64EncodeJSON(config);
const configString = this.base64EncodeJSON(config, 'urlSafe');
return `${this.METADATA.URL}/${configString}/manifest.json`;
}
}
packages/core/src/presets/preset.ts
+10 -12
View File
@@ -8,7 +8,7 @@ import {
Addon,
} from '../db/index.js';
import { StreamParser } from '../parser/index.js';
import { Env, ServiceId, constants } from '../utils/index.js';
import { Env, ServiceId, constants, toUrlSafeBase64 } from '../utils/index.js';
/**
*
* What modifications are needed for each preset:
@@ -143,19 +143,17 @@ export abstract class Preset {
*/
protected static base64EncodeJSON(
json: any,
urlEncode: boolean = false, // url encode the string
makeUrlSafe: boolean = false // replace + with -, / with _ and = with nothing
mode: 'urlEncode' | 'urlSafe' | 'default' = 'default'
) {
let encoded = Buffer.from(JSON.stringify(json)).toString('base64');
if (makeUrlSafe) {
encoded = encoded
.replace(/\+/g, '-')
.replace(/\//g, '_')
.replace(/=+$/, '');
} else if (urlEncode) {
encoded = encodeURIComponent(encoded);
let jsonStr = JSON.stringify(json);
switch (mode) {
case 'urlEncode':
return encodeURIComponent(Buffer.from(jsonStr).toString('base64'));
case 'urlSafe':
return toUrlSafeBase64(jsonStr);
case 'default':
return Buffer.from(jsonStr).toString('base64');
}
return encoded;
}
protected static urlEncodeJSON(json: any) {
packages/core/src/presets/prowlarr.ts
+1 -1
View File
@@ -198,7 +198,7 @@ export class ProwlarrPreset extends BuiltinAddonPreset {
tags: typeof options.tags === 'string' ? options.tags.split(',') : [],
};
const configString = this.base64EncodeJSON(config);
const configString = this.base64EncodeJSON(config, 'urlSafe');
return `${this.METADATA.URL}/${configString}/manifest.json`;
}
}
packages/core/src/presets/streamasia.ts
+1 -1
View File
@@ -446,7 +446,7 @@ there is no need to provide these details here.
hideUnsupportedHosters: options.hideUnsupportedHosters,
version: '1.3.1',
},
true
'urlEncode'
);
return encodedUserData;
packages/core/src/presets/torboxSearch.ts
+1 -1
View File
@@ -249,7 +249,7 @@ export class TorBoxSearchPreset extends StremThruPreset {
})),
};
const configString = this.base64EncodeJSON(config);
const configString = this.base64EncodeJSON(config, 'urlSafe');
return `${this.METADATA.URL}/${configString}/manifest.json`;
}
}
packages/core/src/presets/torrentGalaxy.ts
+2 -1
View File
@@ -64,7 +64,8 @@ export class TorrentGalaxyPreset extends TorznabPreset {
options: Record<string, any>
): string {
return `${Env.INTERNAL_URL}/builtins/torrent-galaxy/${this.base64EncodeJSON(
this.getBaseConfig(userData, services)
this.getBaseConfig(userData, services),
'urlSafe'
)}/manifest.json`;
}
}
packages/core/src/presets/torznab.ts
+1 -1
View File
@@ -154,7 +154,7 @@ export class TorznabPreset extends BuiltinAddonPreset {
forceQuerySearch: options.forceQuerySearch ?? false,
};
const configString = this.base64EncodeJSON(config);
const configString = this.base64EncodeJSON(config, 'urlSafe');
return `${this.METADATA.URL}/${configString}/manifest.json`;
}
}
packages/core/src/presets/zilean.ts
+1 -1
View File
@@ -67,7 +67,7 @@ export class ZileanPreset extends TorznabPreset {
apiPath: '/api',
};
const configString = this.base64EncodeJSON(config);
const configString = this.base64EncodeJSON(config, 'urlSafe');
return `${Env.INTERNAL_URL}/builtins/torznab/${configString}/manifest.json`;
}
}
packages/core/src/transformers/api.ts
+84 -57
View File
@@ -1,40 +1,53 @@
import { ParsedStream, Resource, Subtitle, UserData } from '../db/index.js';
import {
ParsedStream,
Resource,
SubtitleSchema,
UserData,
} from '../db/index.js';
import { AIOStreamsResponse } from '../main.js';
export interface ApiSearchResponseData {
results: ApiSearchResult[];
export interface SearchApiResponseData {
results: SearchApiResult[];
filtered: number;
errors: {
title: string;
description: string;
}[];
}
interface ApiSearchResult {
infoHash: string | null;
seeders: number | null;
age: string | null;
sources: string[] | null;
ytId: string | null;
externalUrl: string | null;
fileIdx: number | null;
url: string | null;
proxied: boolean;
filename: string | null;
folderName: string | null;
size: number | null;
folderSize: number | null;
message: string | null;
library: boolean;
type: string;
indexer: string | null;
addon: string | null;
duration: number | null;
videoHash: string | null;
subtitles: Subtitle[];
countryWhitelist: string[];
requestHeaders: Record<string, string>;
responseHeaders: Record<string, string>;
}
import { z } from 'zod';
const SearchApiResultSchema = z.object({
infoHash: z.string().nullable(),
seeders: z.number().nullable(),
age: z.string().nullable(),
sources: z.array(z.string()).nullable(),
ytId: z.string().nullable(),
externalUrl: z.string().nullable(),
fileIdx: z.number().nullable(),
url: z.string().nullable(),
proxied: z.boolean(),
filename: z.string().nullable(),
folderName: z.string().nullable(),
size: z.number().nullable(),
folderSize: z.number().nullable(),
message: z.string().nullable(),
library: z.boolean(),
type: z.string(),
indexer: z.string().nullable(),
addon: z.string().nullable(),
duration: z.number().nullable(),
videoHash: z.string().nullable(),
subtitles: z.array(SubtitleSchema),
countryWhitelist: z.array(z.string()),
requestHeaders: z.partialRecord(z.string(), z.string()),
responseHeaders: z.partialRecord(z.string(), z.string()),
});
export type SearchApiResult = z.infer<typeof SearchApiResultSchema>;
export type SearchApiResultField = keyof SearchApiResult;
export const SearchApiResultField = z.keyof(SearchApiResultSchema);
export class ApiTransformer {
constructor(private readonly userData: UserData) {}
@@ -43,36 +56,50 @@ export class ApiTransformer {
response: AIOStreamsResponse<{
streams: ParsedStream[];
statistics: { title: string; description: string }[];
}>
): Promise<ApiSearchResponseData> {
}>,
requiredFields: SearchApiResultField[]
): Promise<SearchApiResponseData> {
const { data, errors } = response;
const results: ApiSearchResult[] = data.streams.map((stream) => ({
infoHash: stream.torrent?.infoHash ?? null,
url: stream.url ?? null,
seeders: stream.torrent?.seeders ?? null,
age: stream.age ?? null,
sources: stream.torrent?.sources ?? null,
ytId: stream.ytId ?? null,
externalUrl: stream.externalUrl ?? null,
fileIdx: stream.torrent?.fileIdx ?? null,
proxied: stream.proxied ?? false,
filename: stream.filename ?? null,
folderName: stream.folderName ?? null,
size: stream.size ?? null,
folderSize: stream.folderSize ?? null,
message: stream.message ?? null,
library: stream.library ?? false,
addon: stream.addon.name ?? null,
type: stream.type ?? '',
indexer: stream.indexer ?? null,
duration: stream.duration ?? null,
videoHash: stream.videoHash ?? null,
subtitles: stream.subtitles ?? [],
countryWhitelist: stream.countryWhitelist ?? [],
requestHeaders: stream.requestHeaders ?? {},
responseHeaders: stream.responseHeaders ?? {},
}));
let filteredCount = 0;
const results: SearchApiResult[] = data.streams
.map((stream) => ({
infoHash: stream.torrent?.infoHash ?? null,
url: stream.url ?? null,
seeders: stream.torrent?.seeders ?? null,
age: stream.age ?? null,
sources: stream.torrent?.sources ?? null,
ytId: stream.ytId ?? null,
externalUrl: stream.externalUrl ?? null,
fileIdx: stream.torrent?.fileIdx ?? null,
proxied: stream.proxied ?? false,
filename: stream.filename ?? null,
folderName: stream.folderName ?? null,
size: stream.size ?? null,
folderSize: stream.folderSize ?? null,
message: stream.message ?? null,
library: stream.library ?? false,
addon: stream.addon.name ?? null,
type: stream.type ?? '',
indexer: stream.indexer ?? null,
duration: stream.duration ?? null,
videoHash: stream.videoHash ?? null,
subtitles: stream.subtitles ?? [],
countryWhitelist: stream.countryWhitelist ?? [],
requestHeaders: stream.requestHeaders ?? {},
responseHeaders: stream.responseHeaders ?? {},
}))
?.filter((result) => {
const hasRequiredFields = requiredFields.every(
(field) => result[field] !== null
);
if (!hasRequiredFields) {
filteredCount++;
}
return hasRequiredFields;
});
return {
filtered: filteredCount,
results,
errors: errors.map((error) => ({
title: error.title ?? '',
packages/core/src/utils/cache-adapter.ts
+210 -1
View File
@@ -1,6 +1,7 @@
import { RedisClientType } from 'redis';
import { REDIS_PREFIX, Env } from './index.js';
import { createLogger } from './logger.js';
import { DB } from '../db/db.js';
const logger = createLogger('cache');
@@ -271,7 +272,215 @@ export class RedisCacheBackend<K, V> implements CacheBackend<K, V> {
}
}
// Item stored in memory cache
// SQL cache implementation
export class SQLCacheBackend<K, V> implements CacheBackend<K, V> {
private db: DB;
private prefix: string;
private maxSize: number;
static maintenanceStarted: boolean = false;
constructor(
prefix: string = '',
maxSize: number = Env.DEFAULT_MAX_CACHE_SIZE
) {
this.db = DB.getInstance();
this.prefix = prefix;
this.maxSize = maxSize;
this.startMaintenance();
}
private startMaintenance() {
if (SQLCacheBackend.maintenanceStarted) return;
logger.debug('Starting SQL cache maintenance');
SQLCacheBackend.maintenanceStarted = true;
setInterval(
() => {
this.db
.execute('DELETE FROM cache WHERE expires_at < ?', [Date.now()])
.then((result) => {
logger.debug(
`${result.changed || result.rowCount || 0} stale entries removed from SQL cache`
);
})
.catch((err) => {
logger.error(`Error during SQL cache maintenance: ${err}`);
});
},
1 * 60 * 60 * 1000 // hourly
);
}
private getKey(key: K): string {
return `${this.prefix}${String(key)}`;
}
async get(key: K, updateTTL: boolean = false): Promise<V | undefined> {
const sqlKey = this.getKey(key);
const now = Date.now();
try {
// Get the value and check expiration
const result = await this.db.query(
'SELECT value, expires_at FROM cache WHERE key = ?',
[sqlKey]
);
if (!result.length) {
return undefined;
}
const row = result[0];
if (now > row.expires_at) {
// Remove expired entry
await this.db.execute('DELETE FROM cache WHERE key = ?', [sqlKey]);
return undefined;
}
if (updateTTL) {
const ttl = Math.max(0, row.expires_at - now);
const timestampFunc = this.db.isSQLite()
? 'CURRENT_TIMESTAMP'
: 'NOW()';
await this.db.execute(
`UPDATE cache SET expires_at = ?, last_accessed = ${timestampFunc} WHERE key = ?`,
[now + ttl, sqlKey]
);
} else {
const timestampFunc = this.db.isSQLite()
? 'CURRENT_TIMESTAMP'
: 'NOW()';
await this.db.execute(
`UPDATE cache SET last_accessed = ${timestampFunc} WHERE key = ?`,
[sqlKey]
);
}
return JSON.parse(row.value) as V;
} catch (err) {
logger.error(`Error getting key ${String(key)} from SQL cache: ${err}`);
return undefined;
}
}
async set(key: K, value: V, ttl: number): Promise<void> {
if (ttl === 0) return;
const sqlKey = this.getKey(key);
const expiresAt = Date.now() + ttl * 1000;
const jsonValue = JSON.stringify(value);
try {
// Check current cache size
const countResult = await this.db.query(
'SELECT COUNT(*) as count FROM cache'
);
const currentSize = countResult[0].count;
if (currentSize >= this.maxSize) {
// Remove oldest accessed entry
if (this.db.isSQLite()) {
await this.db.execute(
'DELETE FROM cache WHERE key IN (SELECT key FROM cache ORDER BY last_accessed ASC LIMIT 1)'
);
} else {
// PostgreSQL compatible version
await this.db.execute(
'DELETE FROM cache WHERE key = (SELECT key FROM cache ORDER BY last_accessed ASC LIMIT 1)'
);
}
}
// Upsert the new value
if (this.db.isSQLite()) {
await this.db.execute(
'INSERT OR REPLACE INTO cache (key, value, expires_at) VALUES (?, ?, ?)',
[sqlKey, jsonValue, expiresAt]
);
} else {
const timestampFunc = this.db.isSQLite()
? 'CURRENT_TIMESTAMP'
: 'NOW()';
await this.db.execute(
`INSERT INTO cache (key, value, expires_at) VALUES (?, ?, ?) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, expires_at = EXCLUDED.expires_at, last_accessed = ${timestampFunc}`,
[sqlKey, jsonValue, expiresAt]
);
}
} catch (err) {
logger.error(`Error setting key ${String(key)} in SQL cache: ${err}`);
}
}
async update(key: K, value: V): Promise<void> {
const sqlKey = this.getKey(key);
try {
const result = await this.db.query(
'SELECT expires_at FROM cache WHERE key = ?',
[sqlKey]
);
if (!result.length) return;
const row = result[0];
if (Date.now() > row.expires_at) {
await this.db.execute('DELETE FROM cache WHERE key = ?', [sqlKey]);
return;
}
const timestampFunc = this.db.isSQLite() ? 'CURRENT_TIMESTAMP' : 'NOW()';
await this.db.execute(
`UPDATE cache SET value = ?, last_accessed = ${timestampFunc} WHERE key = ?`,
[JSON.stringify(value), sqlKey]
);
} catch (err) {
logger.error(`Error updating key ${String(key)} in SQL cache: ${err}`);
}
}
async clear(): Promise<void> {
try {
if (this.prefix) {
await this.db.execute('DELETE FROM cache WHERE key LIKE ?', [
`${this.prefix}%`,
]);
} else {
await this.db.execute('DELETE FROM cache');
}
} catch (err) {
logger.error(`Error clearing SQL cache: ${err}`);
}
}
async getTTL(key: K): Promise<number> {
const sqlKey = this.getKey(key);
const now = Date.now();
try {
const result = await this.db.query(
'SELECT expires_at FROM cache WHERE key = ?',
[sqlKey]
);
if (!result.length) return 0;
const ttl = Math.max(0, Math.floor((result[0].expires_at - now) / 1000));
return ttl;
} catch (err) {
logger.error(
`Error getting TTL for key ${String(key)} from SQL cache: ${err}`
);
return 0;
}
}
async waitUntilReady(): Promise<void> {
if (!this.db.isInitialised()) {
throw new Error('Database is not initialized');
}
return Promise.resolve();
}
}
class CacheItem<T> {
constructor(
public value: T,
packages/core/src/utils/cache.ts
+18 -7
View File
@@ -2,6 +2,7 @@ import {
CacheBackend,
MemoryCacheBackend,
RedisCacheBackend,
SQLCacheBackend,
} from './cache-adapter.js';
import { createLogger, Env } from './index.js';
import { createClient, RedisClientType } from 'redis';
@@ -30,12 +31,20 @@ export class Cache<K, V> {
// Redis client singleton
private static redisClient: RedisClientType | null = null;
private constructor(name: string, maxSize: number, forceMemory: boolean) {
private constructor(
name: string,
maxSize: number,
store?: 'redis' | 'sql' | 'memory'
) {
this.name = name;
this.maxSize = maxSize;
// Initialize the appropriate backend based on environment configuration
if (Env.REDIS_URI && !forceMemory) {
// Initialize the appropriate backend based on environment configuration and store preference
if (store === 'sql') {
this.backend = new SQLCacheBackend<K, V>(`${name}:`, maxSize);
logger.debug(`Created SQL cache backend for ${name}`);
} else if (Env.REDIS_URI && (!store || store === 'redis')) {
// use redis if provided and no store preference or redis is specified
this.backend = new RedisCacheBackend<K, V>(
Cache.getRedisClient(),
`${name}:`,
@@ -151,11 +160,11 @@ export class Cache<K, V> {
public static getInstance<K, V>(
name: string,
maxSize: number = Env.DEFAULT_MAX_CACHE_SIZE,
forceMemory: boolean = false
store?: 'redis' | 'sql' | 'memory'
): Cache<K, V> {
if (!this.instances.has(name)) {
logger.debug(`Creating new cache instance: ${name}`);
this.instances.set(name, new Cache<K, V>(name, maxSize, forceMemory));
this.instances.set(name, new Cache<K, V>(name, maxSize, store));
}
return this.instances.get(name) as Cache<K, V>;
}
@@ -280,7 +289,9 @@ export class Cache<K, V> {
return this.backend.waitUntilReady();
}
getType(): 'memory' | 'redis' {
return this.backend instanceof MemoryCacheBackend ? 'memory' : 'redis';
getType(): 'memory' | 'redis' | 'sql' {
if (this.backend instanceof MemoryCacheBackend) return 'memory';
if (this.backend instanceof RedisCacheBackend) return 'redis';
return 'sql';
}
}
packages/core/src/utils/crypto.ts
+4 -23
View File
@@ -10,37 +10,18 @@ import { genSalt, hash, compare } from 'bcrypt';
import { deflateSync, inflateSync } from 'zlib';
import { Env } from './index.js';
import { createLogger } from './logger.js';
import { fromUrlSafeBase64, toUrlSafeBase64 } from './general.js';
const logger = createLogger('crypto');
const saltRounds = 10;
function base64UrlSafe(data: string): string {
return Buffer.from(data)
.toString('base64')
.replace(/\+/g, '-')
.replace(/\//g, '_')
.replace(/=+$/, '');
}
function fromUrlSafeBase64(data: string): string {
// Add padding if needed
const padding = data.length % 4;
const paddedData = padding ? data + '='.repeat(4 - padding) : data;
return Buffer.from(
paddedData.replace(/-/g, '+').replace(/_/g, '/'),
'base64'
).toString('utf-8');
}
const compressData = (data: string): Buffer => {
export const compressData = (data: string): Buffer => {
return deflateSync(Buffer.from(data, 'utf-8'), {
level: 9,
});
};
const decompressData = (data: Buffer): string => {
export const decompressData = (data: Buffer): string => {
return inflateSync(data).toString('utf-8');
};
@@ -116,7 +97,7 @@ export function encryptString(data: string, secretKey?: Buffer): Response {
const { iv, data: encrypted } = encryptData(secretKey, compressed);
return {
success: true,
data: base64UrlSafe(
data: toUrlSafeBase64(
JSON.stringify({ iv, encrypted, type: 'aioEncrypt' })
),
error: null,
packages/core/src/utils/general.ts
+29
View File
@@ -83,3 +83,32 @@ export async function withRetry<T>(
// This line should never be reached due to the throw in the catch block
throw new Error('Unexpected state in retry logic');
}
/**
* Base64 URL safe encoding
* @param data - The data to encode
* @returns The base64 URL safe encoded data
*/
export function toUrlSafeBase64(string: string): string {
return Buffer.from(string)
.toString('base64')
.replace(/\+/g, '-')
.replace(/\//g, '_')
.replace(/=+$/, '');
}
/**
* Base64 URL safe decoding
* @param data - The data to decode
* @returns The base64 URL safe decoded data
*/
export function fromUrlSafeBase64(data: string): string {
// Add padding if needed
const padding = data.length % 4;
const paddedData = padding ? data + '='.repeat(4 - padding) : data;
return Buffer.from(
paddedData.replace(/-/g, '+').replace(/_/g, '/'),
'base64'
).toString('utf-8');
}
packages/core/src/utils/http.ts
+1 -1
View File
@@ -21,7 +21,7 @@ const logger = createLogger('http');
const urlCount = Cache.getInstance<string, number>(
'url-count',
undefined,
true
'memory'
);
export class PossibleRecursiveRequestError extends Error {
packages/core/src/utils/regex.ts
+5 -1
View File
@@ -4,7 +4,11 @@ import { getSimpleTextHash } from './crypto.js';
import { createLogger } from './logger.js';
const DEFAULT_TIMEOUT = 1000; // 1 second timeout
const regexCache = Cache.getInstance<string, RegExp>('regexCache', 1_000, true);
const regexCache = Cache.getInstance<string, RegExp>(
'regexCache',
1_000,
'memory'
);
const resultCache = Cache.getInstance<string, boolean>(
'regexResultCache',
1_000_000
packages/server/src/routes/api/debrid.ts
+28 -8
View File
@@ -8,6 +8,12 @@ import {
PlaybackInfoSchema,
getDebridService,
ServiceAuthSchema,
fromUrlSafeBase64,
Cache,
PlaybackInfo,
ServiceAuth,
decryptString,
pbiCache,
} from '@aiostreams/core';
import { ZodError } from 'zod';
import { StaticFiles } from '../../app.js';
@@ -24,24 +30,38 @@ router.use((req: Request, res: Response, next: NextFunction) => {
});
router.get(
'/playback/:encodedStoreAuth/:encodedPlaybackInfo/:filename',
'/playback/:encryptedStoreAuth/:playbackId/:filename',
async (req: Request, res: Response, next: NextFunction) => {
try {
const { encodedStoreAuth, encodedPlaybackInfo, filename } = req.params;
if (!encodedStoreAuth || !encodedPlaybackInfo || !filename) {
const { encryptedStoreAuth, playbackId, filename } = req.params;
if (!playbackId || !filename) {
throw new APIError(
constants.ErrorCode.BAD_REQUEST,
undefined,
'Store auth, playback info and filename are required'
'Encrypted store auth, playback info and filename are required'
);
}
const decryptedStoreAuth = decryptString(encryptedStoreAuth);
if (!decryptedStoreAuth.success) {
throw new APIError(
constants.ErrorCode.BAD_REQUEST,
undefined,
'Failed to decrypt store auth'
);
}
const playbackInfo = PlaybackInfoSchema.parse(
JSON.parse(Buffer.from(encodedPlaybackInfo, 'base64').toString('utf-8'))
);
const storeAuth = ServiceAuthSchema.parse(
JSON.parse(Buffer.from(encodedStoreAuth, 'base64').toString('utf-8'))
JSON.parse(decryptedStoreAuth.data)
);
const playbackInfo = await pbiCache().get(playbackId);
if (!playbackInfo) {
throw new APIError(
constants.ErrorCode.BAD_REQUEST,
undefined,
'Playback info not found'
);
}
const debridInterface = getDebridService(
storeAuth.id,
packages/server/src/routes/api/search.ts
+24 -10
View File
@@ -13,7 +13,8 @@ import {
decryptString,
createLogger,
ApiTransformer,
ApiSearchResponseData,
SearchApiResponseData,
SearchApiResultField,
} from '@aiostreams/core';
import { streamApiRateLimiter } from '../../middlewares/ratelimit.js';
import { ApiResponse, createResponse } from '../../utils/responses.js';
@@ -24,20 +25,32 @@ const logger = createLogger('server');
router.use(streamApiRateLimiter);
const SearchApiRequestSchema = z.object({
type: z.string(),
id: z.string(),
requiredFields: z
.union([z.array(SearchApiResultField), SearchApiResultField])
.optional()
.default([])
.transform((val) => {
if (Array.isArray(val)) {
return val;
}
return [val];
}),
});
router.get(
'/',
async (
req: Request,
res: Response<ApiResponse<ApiSearchResponseData>>,
res: Response<ApiResponse<SearchApiResponseData>>,
next
) => {
try {
const { type, id } = z
.object({
type: z.string(),
id: z.string(),
})
.parse(req.query);
const { type, id, requiredFields } = SearchApiRequestSchema.parse(
req.query
);
let encodedUserData: string | undefined = z
.string()
.optional()
@@ -161,12 +174,13 @@ router.get(
const transformer = new ApiTransformer(userData);
res.status(200).json(
createResponse<ApiSearchResponseData>({
createResponse<SearchApiResponseData>({
success: true,
data: await transformer.transformStreams(
await (
await new AIOStreams(userData).initialise()
).getStreams(id, type)
).getStreams(id, type),
requiredFields
),
})
);
packages/server/src/routes/builtins/gdrive.ts
+5 -11
View File
@@ -1,5 +1,5 @@
import { Router, Request, Response, NextFunction } from 'express';
import { createLogger, GDriveAddon } from '@aiostreams/core';
import { createLogger, fromUrlSafeBase64, GDriveAddon } from '@aiostreams/core';
const router: Router = Router();
const logger = createLogger('server');
@@ -9,7 +9,7 @@ router.get(
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig } = req.params;
const config = encodedConfig
? JSON.parse(Buffer.from(encodedConfig, 'base64').toString('utf-8'))
? JSON.parse(fromUrlSafeBase64(encodedConfig))
: undefined;
try {
@@ -27,9 +27,7 @@ router.get(
'/:encodedConfig/meta/:type/:id.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig, type, id } = req.params;
const config = JSON.parse(
Buffer.from(encodedConfig, 'base64').toString('utf-8')
);
const config = JSON.parse(fromUrlSafeBase64(encodedConfig));
try {
const addon = new GDriveAddon(config);
@@ -47,9 +45,7 @@ router.get(
'/:encodedConfig/catalog/:type/:id{/:extras}.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig, type, id, extras } = req.params;
const config = JSON.parse(
Buffer.from(encodedConfig, 'base64').toString('utf-8')
);
const config = JSON.parse(fromUrlSafeBase64(encodedConfig));
try {
const addon = new GDriveAddon(config);
@@ -67,9 +63,7 @@ router.get(
'/:encodedConfig/stream/:type/:id.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig, type, id } = req.params;
const config = JSON.parse(
Buffer.from(encodedConfig, 'base64').toString('utf-8')
);
const config = JSON.parse(fromUrlSafeBase64(encodedConfig));
try {
const addon = new GDriveAddon(config);
packages/server/src/routes/builtins/knaben.ts
+13 -10
View File
@@ -1,5 +1,5 @@
import { Router, Request, Response, NextFunction } from 'express';
import { AIOStreams, AIOStreamResponse, KnabenAddon } from '@aiostreams/core';
import { KnabenAddon, fromUrlSafeBase64 } from '@aiostreams/core';
import { createLogger } from '@aiostreams/core';
const router: Router = Router();
@@ -9,12 +9,13 @@ router.get(
'/:encodedConfig/manifest.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig } = req.params;
const config = encodedConfig
? JSON.parse(Buffer.from(encodedConfig, 'base64').toString('utf-8'))
: undefined;
try {
const manifest = new KnabenAddon(config, req.userIp).getManifest();
const manifest = new KnabenAddon(
encodedConfig
? JSON.parse(fromUrlSafeBase64(encodedConfig))
: undefined,
req.userIp
).getManifest();
res.json(manifest);
} catch (error) {
next(error);
@@ -26,12 +27,14 @@ router.get(
'/:encodedConfig/stream/:type/:id.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig, type, id } = req.params;
const config = JSON.parse(
Buffer.from(encodedConfig, 'base64').toString('utf-8')
);
try {
const addon = new KnabenAddon(config, req.userIp);
const addon = new KnabenAddon(
encodedConfig
? JSON.parse(fromUrlSafeBase64(encodedConfig))
: undefined,
req.userIp
);
const streams = await addon.getStreams(type, id);
res.json({
streams: streams,
packages/server/src/routes/builtins/newznab.ts
+17 -9
View File
@@ -1,5 +1,9 @@
import { Router, Request, Response, NextFunction } from 'express';
import { NewznabAddon, createLogger } from '@aiostreams/core';
import {
NewznabAddon,
createLogger,
fromUrlSafeBase64,
} from '@aiostreams/core';
const router: Router = Router();
const logger = createLogger('server');
@@ -8,12 +12,14 @@ router.get(
'/:encodedConfig/manifest.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig } = req.params;
const config = encodedConfig
? JSON.parse(Buffer.from(encodedConfig, 'base64').toString('utf-8'))
: undefined;
try {
const manifest = new NewznabAddon(config, req.userIp).getManifest();
const manifest = new NewznabAddon(
encodedConfig
? JSON.parse(fromUrlSafeBase64(encodedConfig))
: undefined,
req.userIp
).getManifest();
res.json(manifest);
} catch (error) {
next(error);
@@ -25,12 +31,14 @@ router.get(
'/:encodedConfig/stream/:type/:id.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig, type, id } = req.params;
const config = JSON.parse(
Buffer.from(encodedConfig, 'base64').toString('utf-8')
);
try {
const addon = new NewznabAddon(config, req.userIp);
const addon = new NewznabAddon(
encodedConfig
? JSON.parse(fromUrlSafeBase64(encodedConfig))
: undefined,
req.userIp
);
const streams = await addon.getStreams(type, id);
res.json({
streams: streams,
packages/server/src/routes/builtins/prowlarr.ts
+13 -9
View File
@@ -1,5 +1,5 @@
import { Router, Request, Response, NextFunction } from 'express';
import { ProwlarrAddon } from '@aiostreams/core';
import { ProwlarrAddon, fromUrlSafeBase64 } from '@aiostreams/core';
import { createLogger } from '@aiostreams/core';
const router: Router = Router();
@@ -9,12 +9,14 @@ router.get(
'/:encodedConfig/manifest.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig } = req.params;
const config = encodedConfig
? JSON.parse(Buffer.from(encodedConfig, 'base64').toString('utf-8'))
: undefined;
try {
const manifest = new ProwlarrAddon(config, req.userIp).getManifest();
const manifest = new ProwlarrAddon(
encodedConfig
? JSON.parse(fromUrlSafeBase64(encodedConfig))
: undefined,
req.userIp
).getManifest();
res.json(manifest);
} catch (error) {
next(error);
@@ -26,12 +28,14 @@ router.get(
'/:encodedConfig/stream/:type/:id.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig, type, id } = req.params;
const config = JSON.parse(
Buffer.from(encodedConfig, 'base64').toString('utf-8')
);
try {
const addon = new ProwlarrAddon(config, req.userIp);
const addon = new ProwlarrAddon(
encodedConfig
? JSON.parse(fromUrlSafeBase64(encodedConfig))
: undefined,
req.userIp
);
const streams = await addon.getStreams(type, id);
res.json({
streams: streams,
packages/server/src/routes/builtins/torbox-search.ts
+14 -11
View File
@@ -3,6 +3,7 @@ import {
createLogger,
TorBoxSearchAddon,
TorBoxSearchAddonError,
fromUrlSafeBase64,
} from '@aiostreams/core';
import { createResponse } from '../../utils/responses.js';
const router: Router = Router();
@@ -13,14 +14,14 @@ router.get(
'{/:encodedConfig}/manifest.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig } = req.params;
const config = encodedConfig
? JSON.parse(Buffer.from(encodedConfig, 'base64').toString('utf-8'))
: undefined;
try {
const manifest = config
? new TorBoxSearchAddon(config, req.userIp).getManifest()
const manifest = encodedConfig
? new TorBoxSearchAddon(
encodedConfig
? JSON.parse(fromUrlSafeBase64(encodedConfig))
: undefined,
req.userIp
).getManifest()
: TorBoxSearchAddon.getManifest();
res.json(manifest);
} catch (error) {
@@ -45,12 +46,14 @@ router.get(
'/:encodedConfig/stream/:type/:id.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig, type, id } = req.params;
const config = JSON.parse(
Buffer.from(encodedConfig, 'base64').toString('utf-8')
);
try {
const addon = new TorBoxSearchAddon(config, req.userIp);
const addon = new TorBoxSearchAddon(
encodedConfig
? JSON.parse(fromUrlSafeBase64(encodedConfig))
: undefined,
req.userIp
);
const streams = await addon.getStreams(type as any, id);
res.json({
streams: streams,
packages/server/src/routes/builtins/torrent-galaxy.ts
+13 -8
View File
@@ -3,6 +3,7 @@ import {
AIOStreams,
AIOStreamResponse,
TorrentGalaxyAddon,
fromUrlSafeBase64,
} from '@aiostreams/core';
import { createLogger } from '@aiostreams/core';
const router: Router = Router();
@@ -13,12 +14,14 @@ router.get(
'/:encodedConfig/manifest.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig } = req.params;
const config = encodedConfig
? JSON.parse(Buffer.from(encodedConfig, 'base64').toString('utf-8'))
: undefined;
try {
const manifest = new TorrentGalaxyAddon(config, req.userIp).getManifest();
const manifest = new TorrentGalaxyAddon(
encodedConfig
? JSON.parse(fromUrlSafeBase64(encodedConfig))
: undefined,
req.userIp
).getManifest();
res.json(manifest);
} catch (error) {
next(error);
@@ -30,12 +33,14 @@ router.get(
'/:encodedConfig/stream/:type/:id.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig, type, id } = req.params;
const config = JSON.parse(
Buffer.from(encodedConfig, 'base64').toString('utf-8')
);
try {
const addon = new TorrentGalaxyAddon(config, req.userIp);
const addon = new TorrentGalaxyAddon(
encodedConfig
? JSON.parse(fromUrlSafeBase64(encodedConfig))
: undefined,
req.userIp
);
const streams = await addon.getStreams(type, id);
res.json({
streams: streams,
packages/server/src/routes/builtins/torznab.ts
+13 -9
View File
@@ -1,5 +1,5 @@
import { Router, Request, Response, NextFunction } from 'express';
import { AIOStreams, AIOStreamResponse, TorznabAddon } from '@aiostreams/core';
import { TorznabAddon, fromUrlSafeBase64 } from '@aiostreams/core';
import { createLogger } from '@aiostreams/core';
const router: Router = Router();
@@ -9,12 +9,14 @@ router.get(
'/:encodedConfig/manifest.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig } = req.params;
const config = encodedConfig
? JSON.parse(Buffer.from(encodedConfig, 'base64').toString('utf-8'))
: undefined;
try {
const manifest = new TorznabAddon(config, req.userIp).getManifest();
const manifest = new TorznabAddon(
encodedConfig
? JSON.parse(fromUrlSafeBase64(encodedConfig))
: undefined,
req.userIp
).getManifest();
res.json(manifest);
} catch (error) {
next(error);
@@ -26,12 +28,14 @@ router.get(
'/:encodedConfig/stream/:type/:id.json',
async (req: Request, res: Response, next: NextFunction) => {
const { encodedConfig, type, id } = req.params;
const config = JSON.parse(
Buffer.from(encodedConfig, 'base64').toString('utf-8')
);
try {
const addon = new TorznabAddon(config, req.userIp);
const addon = new TorznabAddon(
encodedConfig
? JSON.parse(fromUrlSafeBase64(encodedConfig))
: undefined,
req.userIp
);
const streams = await addon.getStreams(type, id);
res.json({
streams: streams,