From e435f1c7bdd6189ae8ebb4c57d53a0325a241674 Mon Sep 17 00:00:00 2001 From: Viren070 Date: Sun, 26 Oct 2025 14:59:26 +0000 Subject: [PATCH] fix(builtins): processing adjustments --- .../builtins/torbox-search/source-handlers.ts | 1 - packages/core/src/builtins/utils/debrid.ts | 205 +++++++++++------- packages/core/src/debrid/utils.ts | 96 +++++++- 3 files changed, 218 insertions(+), 84 deletions(-) diff --git a/packages/core/src/builtins/torbox-search/source-handlers.ts b/packages/core/src/builtins/torbox-search/source-handlers.ts index eab5effb..b27918c4 100644 --- a/packages/core/src/builtins/torbox-search/source-handlers.ts +++ b/packages/core/src/builtins/torbox-search/source-handlers.ts @@ -330,7 +330,6 @@ export class TorrentSourceHandler extends SourceHandler { const { results, errors } = await processTorrents( fetchResult.torrents.map((torrent) => ({ ...torrent, - confirmed: true, type: 'torrent', })), this.services, diff --git a/packages/core/src/builtins/utils/debrid.ts b/packages/core/src/builtins/utils/debrid.ts index e3407119..ed8c70cd 100644 --- a/packages/core/src/builtins/utils/debrid.ts +++ b/packages/core/src/builtins/utils/debrid.ts @@ -15,6 +15,8 @@ import { isSeasonWrong, isEpisodeWrong, isTitleWrong, + DebridDownload, + isNotVideoFile, } from '../../debrid/index.js'; import { PTT } from '../../parser/index.js'; import { ParseResult } from 'go-ptt'; @@ -129,41 +131,28 @@ async function processTorrentsForDebridService( time: getTimeTakenSincePoint(startTime), }); - const allStrings: string[] = []; - - // First add all torrent titles - for (const torrent of torrents) { - allStrings.push(torrent.title ?? ''); - } - - // Then add all filenames from all torrents - for (const torrent of magnetCheckResults) { - if (torrent.files && Array.isArray(torrent.files)) { - for (const file of torrent.files) { - allStrings.push(file.name ?? ''); - } - } - } - - // Parse all strings in one call - const allParsedResults = await PTT.parse(allStrings); - - // Split the results into parsed titles and files - const parsedFiles = new Map(); - for (const [index, result] of allParsedResults.entries()) { + // Parse only torrent titles and perform validation checks + const torrentTitles = torrents.map((torrent) => torrent.title ?? ''); + const parsedTitles = await PTT.parse(torrentTitles); + const parsedTitlesMap = new Map(); + for (const [index, result] of parsedTitles.entries()) { if (result) { - parsedFiles.set(allStrings[index], result); + parsedTitlesMap.set(torrentTitles[index], result); } } - const processingStart = Date.now(); - for (const [index, torrent] of torrents.entries()) { - let file: DebridFile | undefined; + // Filter torrents that pass validation checks + const validTorrents: { + torrent: Torrent; + magnetCheckResult: DebridDownload | undefined; + parsedTitle: ParseResult; + }[] = []; + for (const torrent of torrents) { const magnetCheckResult = magnetCheckResults.find( (result) => result.hash === torrent.hash ); + const parsedTorrent = parsedTitlesMap.get(torrent.title ?? ''); - const parsedTorrent = parsedFiles.get(torrent.title ?? ''); if (metadata && parsedTorrent) { const preprocessedTitle = preprocessTitle( parsedTorrent.title, @@ -183,12 +172,47 @@ async function processTorrentsForDebridService( continue; } } + + validTorrents.push({ + torrent, + magnetCheckResult, + parsedTitle: parsedTorrent!, + }); + } + + // Parse files only for valid torrents + const allFileStrings: string[] = []; + for (const { magnetCheckResult } of validTorrents) { + if (magnetCheckResult?.files && Array.isArray(magnetCheckResult.files)) { + for (const file of magnetCheckResult.files) { + if (isNotVideoFile(file)) continue; + allFileStrings.push(file.name ?? ''); + } + } + } + + // Parse all file strings in one call + const allParsedFiles = await PTT.parse(allFileStrings); + const parsedFiles = new Map(); + for (const [index, result] of allParsedFiles.entries()) { + if (result) { + parsedFiles.set(allFileStrings[index], result); + } + } + + const processingStart = Date.now(); + for (const { torrent, magnetCheckResult, parsedTitle } of validTorrents) { + let file: DebridFile | undefined; + file = magnetCheckResult ? await selectFileInTorrentOrNZB( torrent, magnetCheckResult, parsedFiles, - metadata + metadata, + { + useLevenshteinMatching: false, + } ) : { name: torrent.title, size: torrent.size, index: -1 }; @@ -218,28 +242,20 @@ export async function processTorrentsForP2P( ): Promise { const results: TorrentWithSelectedFile[] = []; - const allStrings: string[] = []; - for (const torrent of torrents) { - allStrings.push(torrent.title ?? ''); - if (torrent.files && Array.isArray(torrent.files)) { - for (const file of torrent.files) { - allStrings.push(file.name ?? ''); - } - } - } - - const allParsedResults = await PTT.parse(allStrings); - const parsedFiles = new Map(); - for (const [index, result] of allParsedResults.entries()) { + // Parse only torrent titles and perform validation checks + const torrentTitles = torrents.map((torrent) => torrent.title ?? ''); + const parsedTitles = await PTT.parse(torrentTitles); + const parsedTitlesMap = new Map(); + for (const [index, result] of parsedTitles.entries()) { if (result) { - parsedFiles.set(allStrings[index], result); + parsedTitlesMap.set(torrentTitles[index], result); } } - for (const [index, torrent] of torrents.entries()) { - let file: DebridFile | undefined; - - const parsedTorrent = parsedFiles.get(torrent.title ?? ''); + // Filter torrents that pass validation checks + const validTorrents: { torrent: Torrent; parsedTitle: ParseResult }[] = []; + for (const torrent of torrents) { + const parsedTorrent = parsedTitlesMap.get(torrent.title ?? ''); if (metadata && parsedTorrent) { if (isSeasonWrong(parsedTorrent, metadata)) { continue; @@ -248,6 +264,31 @@ export async function processTorrentsForP2P( continue; } } + validTorrents.push({ torrent, parsedTitle: parsedTorrent! }); + } + + // Parse files only for valid torrents + const allFileStrings: string[] = []; + for (const { torrent } of validTorrents) { + if (torrent.files && Array.isArray(torrent.files)) { + for (const file of torrent.files) { + if (isNotVideoFile(file)) continue; + allFileStrings.push(file.name ?? ''); + } + } + } + + const allParsedFiles = await PTT.parse(allFileStrings); + const parsedFiles = new Map(); + for (const [index, result] of allParsedFiles.entries()) { + if (result) { + parsedFiles.set(allFileStrings[index], result); + } + } + + for (const { torrent } of validTorrents) { + let file: DebridFile | undefined; + file = torrent.files ? await selectFileInTorrentOrNZB( torrent, @@ -259,7 +300,10 @@ export async function processTorrentsForP2P( files: torrent.files, }, parsedFiles, - metadata + metadata, + { + useLevenshteinMatching: false, + } ) : undefined; @@ -346,42 +390,31 @@ async function processNZBsForDebridService( logger.debug(`Retrieved NZB status from debrid`, { service: debridService.serviceName, - magnetCount: nzbs.length, + nzbCount: nzbs.length, time: getTimeTakenSincePoint(startTime), }); - // parse all files from all nzbs in one call - - const allStrings: string[] = []; - - for (const nzb of nzbs) { - allStrings.push(nzb.title ?? ''); - } - - for (const nzb of nzbCheckResults) { - if (nzb.files && Array.isArray(nzb.files)) { - for (const file of nzb.files) { - allStrings.push(file.name ?? ''); - } - } - } - - const allParsedResults = await PTT.parse(allStrings); - const parsedFiles = new Map(); - for (const [index, result] of allParsedResults.entries()) { + // Parse only NZB titles and perform validation checks + const nzbTitles = nzbs.map((nzb) => nzb.title ?? ''); + const parsedTitles = await PTT.parse(nzbTitles); + const parsedTitlesMap = new Map(); + for (const [index, result] of parsedTitles.entries()) { if (result) { - parsedFiles.set(allStrings[index], result); + parsedTitlesMap.set(nzbTitles[index], result); } } - const processingStart = Date.now(); - for (const [index, nzb] of nzbs.entries()) { - let file: DebridFile | undefined; - + // Filter NZBs that pass validation checks + const validNZBs: { + nzb: NZB; + nzbCheckResult: any; + parsedTitle: ParseResult; + }[] = []; + for (const nzb of nzbs) { const nzbCheckResult = nzbCheckResults.find( (result) => result.hash === nzb.hash ); - const parsedNzb = parsedFiles.get(nzb.title ?? ''); + const parsedNzb = parsedTitlesMap.get(nzb.title ?? ''); if (metadata && parsedNzb) { if (isSeasonWrong(parsedNzb, metadata)) { continue; @@ -390,6 +423,32 @@ async function processNZBsForDebridService( continue; } } + validNZBs.push({ nzb, nzbCheckResult, parsedTitle: parsedNzb! }); + } + + // Parse files only for valid NZBs + const allFileStrings: string[] = []; + for (const { nzbCheckResult } of validNZBs) { + if (nzbCheckResult?.files && Array.isArray(nzbCheckResult.files)) { + for (const file of nzbCheckResult.files) { + if (isNotVideoFile(file)) continue; + allFileStrings.push(file.name ?? ''); + } + } + } + + const allParsedFiles = await PTT.parse(allFileStrings); + const parsedFiles = new Map(); + for (const [index, result] of allParsedFiles.entries()) { + if (result) { + parsedFiles.set(allFileStrings[index], result); + } + } + + const processingStart = Date.now(); + for (const { nzb, nzbCheckResult } of validNZBs) { + let file: DebridFile | undefined; + file = nzbCheckResult ? await selectFileInTorrentOrNZB( nzb, diff --git a/packages/core/src/debrid/utils.ts b/packages/core/src/debrid/utils.ts index 08366f5c..e8b877d6 100644 --- a/packages/core/src/debrid/utils.ts +++ b/packages/core/src/debrid/utils.ts @@ -144,6 +144,18 @@ export const isTitleWrong = ( return false; }; +export const isTitleWrongN = ( + parsed: { title?: string }, + metadata?: { titles?: string[] } +) => { + if (parsed.title && metadata?.titles) { + const normalisedParsedTitle = normaliseTitle(parsed.title); + return !metadata.titles + .map(normaliseTitle) + .some((title) => title == normalisedParsedTitle); + } + return false; +}; export async function selectFileInTorrentOrNZB( torrentOrNZB: Torrent | NZB, debridDownload: DebridDownload, @@ -167,6 +179,7 @@ export async function selectFileInTorrentOrNZB( options?: { chosenFilename?: string; chosenIndex?: number; + useLevenshteinMatching?: boolean; } ): Promise { if (!debridDownload.files?.length) { @@ -178,15 +191,8 @@ export async function selectFileInTorrentOrNZB( } const isVideo = debridDownload.files.map((file) => isVideoFile(file)); - if (metadata?.titles && metadata.titles.length) { - const uniqueTitles = [ - ...new Set(metadata?.titles.map(normaliseTitle)), - ].slice(0, 100); - metadata = { - ...metadata, - titles: uniqueTitles, - }; - } + const isNotVideo = debridDownload.files.map((file) => isNotVideoFile(file)); + const videoExists = isVideo.map((f) => f == true); // Create a scoring system for each file const fileScores = []; @@ -195,8 +201,13 @@ export async function selectFileInTorrentOrNZB( let score = 0; const parsed = parsedFiles.get(file.name ?? ''); + if (isNotVideo[index]) { + continue; + } + if (!parsed) { logger.warn(`Parsed file not found for ${file.name}`); + continue; } // Base score from video file status (highest priority) @@ -233,9 +244,12 @@ export async function selectFileInTorrentOrNZB( } // Title matching (third priority) + const titleMatchFunc = + options?.useLevenshteinMatching == false ? isTitleWrongN : isTitleWrong; if ( parsed?.title && - !isTitleWrong( + (videoExists ? isVideo[index] : true) && + !titleMatchFunc( { title: preprocessTitle( parsed.title, @@ -277,6 +291,10 @@ export async function selectFileInTorrentOrNZB( } } + if (fileScores.length === 0) { + logger.warn(`Torrent ${torrentOrNZB.title} had no files selected`); + return undefined; + } // Sort by score descending fileScores.sort((a, b) => b.score - a.score); @@ -367,6 +385,64 @@ export function isVideoFile(file: DebridFile): boolean { ); } +export function isNotVideoFile(file: DebridFile): boolean { + const nonVideoExtensions = [ + '.txt', + '.jpg', + '.jpeg', + '.png', + '.gif', + '.bmp', + '.svg', + '.webp', + '.nfo', + '.sfv', + '.srt', + '.ass', + '.sub', + '.idx', + '.cue', + '.log', + '.doc', + '.docx', + '.xls', + '.xlsx', + '.ppt', + '.pptx', + '.pdf', + '.rtf', + '.odt', + '.ods', + '.odp', + '.csv', + '.tsv', + '.exe', + '.bat', + '.apk', + '.dll', + '.iso', + '.zip', + '.rar', + '.7z', + '.tar', + '.gz', + '.bz2', + '.xz', + '.md', + '.json', + '.xml', + '.ini', + '.dat', + '.db', + '.dbf', + '.bak', + ]; + return ( + (file.mimeType && !file.mimeType.includes('video')) || + nonVideoExtensions.some((ext) => file.name?.endsWith(ext) ?? false) + ); +} + export const metadataStore = () => { const prefix = 'mds'; const store: 'redis' | 'sql' | 'memory' =