fix(builtins): processing adjustments

packages/core/src/builtins/torbox-search/source-handlers.ts

@@ -330,7 +330,6 @@ export class TorrentSourceHandler extends SourceHandler {
     const { results, errors } = await processTorrents(
       fetchResult.torrents.map((torrent) => ({
         ...torrent,
-        confirmed: true,
         type: 'torrent',
       })),
       this.services,

packages/core/src/builtins/utils/debrid.ts

@@ -15,6 +15,8 @@ import {
   isSeasonWrong,
   isEpisodeWrong,
   isTitleWrong,
+  DebridDownload,
+  isNotVideoFile,
 } from '../../debrid/index.js';
 import { PTT } from '../../parser/index.js';
 import { ParseResult } from 'go-ptt';
@@ -129,41 +131,28 @@ async function processTorrentsForDebridService(
     time: getTimeTakenSincePoint(startTime),
   });
 
-  const allStrings: string[] = [];
-
-  // First add all torrent titles
-  for (const torrent of torrents) {
-    allStrings.push(torrent.title ?? '');
-  }
-
-  // Then add all filenames from all torrents
-  for (const torrent of magnetCheckResults) {
-    if (torrent.files && Array.isArray(torrent.files)) {
-      for (const file of torrent.files) {
-        allStrings.push(file.name ?? '');
-      }
-    }
-  }
-
-  // Parse all strings in one call
-  const allParsedResults = await PTT.parse(allStrings);
-
-  // Split the results into parsed titles and files
-  const parsedFiles = new Map<string, ParseResult>();
-  for (const [index, result] of allParsedResults.entries()) {
+  // Parse only torrent titles and perform validation checks
+  const torrentTitles = torrents.map((torrent) => torrent.title ?? '');
+  const parsedTitles = await PTT.parse(torrentTitles);
+  const parsedTitlesMap = new Map<string, ParseResult>();
+  for (const [index, result] of parsedTitles.entries()) {
     if (result) {
-      parsedFiles.set(allStrings[index], result);
+      parsedTitlesMap.set(torrentTitles[index], result);
     }
   }
-  const processingStart = Date.now();
-  for (const [index, torrent] of torrents.entries()) {
-    let file: DebridFile | undefined;
 
+  // Filter torrents that pass validation checks
+  const validTorrents: {
+    torrent: Torrent;
+    magnetCheckResult: DebridDownload | undefined;
+    parsedTitle: ParseResult;
+  }[] = [];
+  for (const torrent of torrents) {
     const magnetCheckResult = magnetCheckResults.find(
       (result) => result.hash === torrent.hash
     );
+    const parsedTorrent = parsedTitlesMap.get(torrent.title ?? '');
 
-    const parsedTorrent = parsedFiles.get(torrent.title ?? '');
     if (metadata && parsedTorrent) {
       const preprocessedTitle = preprocessTitle(
         parsedTorrent.title,
@@ -183,12 +172,47 @@ async function processTorrentsForDebridService(
         continue;
       }
     }
+
+    validTorrents.push({
+      torrent,
+      magnetCheckResult,
+      parsedTitle: parsedTorrent!,
+    });
+  }
+
+  // Parse files only for valid torrents
+  const allFileStrings: string[] = [];
+  for (const { magnetCheckResult } of validTorrents) {
+    if (magnetCheckResult?.files && Array.isArray(magnetCheckResult.files)) {
+      for (const file of magnetCheckResult.files) {
+        if (isNotVideoFile(file)) continue;
+        allFileStrings.push(file.name ?? '');
+      }
+    }
+  }
+
+  // Parse all file strings in one call
+  const allParsedFiles = await PTT.parse(allFileStrings);
+  const parsedFiles = new Map<string, ParseResult>();
+  for (const [index, result] of allParsedFiles.entries()) {
+    if (result) {
+      parsedFiles.set(allFileStrings[index], result);
+    }
+  }
+
+  const processingStart = Date.now();
+  for (const { torrent, magnetCheckResult, parsedTitle } of validTorrents) {
+    let file: DebridFile | undefined;
+
     file = magnetCheckResult
       ? await selectFileInTorrentOrNZB(
           torrent,
           magnetCheckResult,
           parsedFiles,
-          metadata
+          metadata,
+          {
+            useLevenshteinMatching: false,
+          }
         )
       : { name: torrent.title, size: torrent.size, index: -1 };
 
@@ -218,28 +242,20 @@ export async function processTorrentsForP2P(
 ): Promise<TorrentWithSelectedFile[]> {
   const results: TorrentWithSelectedFile[] = [];
 
-  const allStrings: string[] = [];
-  for (const torrent of torrents) {
-    allStrings.push(torrent.title ?? '');
-    if (torrent.files && Array.isArray(torrent.files)) {
-      for (const file of torrent.files) {
-        allStrings.push(file.name ?? '');
-      }
-    }
-  }
-
-  const allParsedResults = await PTT.parse(allStrings);
-  const parsedFiles = new Map<string, ParseResult>();
-  for (const [index, result] of allParsedResults.entries()) {
+  // Parse only torrent titles and perform validation checks
+  const torrentTitles = torrents.map((torrent) => torrent.title ?? '');
+  const parsedTitles = await PTT.parse(torrentTitles);
+  const parsedTitlesMap = new Map<string, ParseResult>();
+  for (const [index, result] of parsedTitles.entries()) {
     if (result) {
-      parsedFiles.set(allStrings[index], result);
+      parsedTitlesMap.set(torrentTitles[index], result);
     }
   }
 
-  for (const [index, torrent] of torrents.entries()) {
-    let file: DebridFile | undefined;
-
-    const parsedTorrent = parsedFiles.get(torrent.title ?? '');
+  // Filter torrents that pass validation checks
+  const validTorrents: { torrent: Torrent; parsedTitle: ParseResult }[] = [];
+  for (const torrent of torrents) {
+    const parsedTorrent = parsedTitlesMap.get(torrent.title ?? '');
     if (metadata && parsedTorrent) {
       if (isSeasonWrong(parsedTorrent, metadata)) {
         continue;
@@ -248,6 +264,31 @@ export async function processTorrentsForP2P(
         continue;
       }
     }
+    validTorrents.push({ torrent, parsedTitle: parsedTorrent! });
+  }
+
+  // Parse files only for valid torrents
+  const allFileStrings: string[] = [];
+  for (const { torrent } of validTorrents) {
+    if (torrent.files && Array.isArray(torrent.files)) {
+      for (const file of torrent.files) {
+        if (isNotVideoFile(file)) continue;
+        allFileStrings.push(file.name ?? '');
+      }
+    }
+  }
+
+  const allParsedFiles = await PTT.parse(allFileStrings);
+  const parsedFiles = new Map<string, ParseResult>();
+  for (const [index, result] of allParsedFiles.entries()) {
+    if (result) {
+      parsedFiles.set(allFileStrings[index], result);
+    }
+  }
+
+  for (const { torrent } of validTorrents) {
+    let file: DebridFile | undefined;
+
     file = torrent.files
       ? await selectFileInTorrentOrNZB(
           torrent,
@@ -259,7 +300,10 @@ export async function processTorrentsForP2P(
             files: torrent.files,
           },
           parsedFiles,
-          metadata
+          metadata,
+          {
+            useLevenshteinMatching: false,
+          }
         )
       : undefined;
 
@@ -346,42 +390,31 @@ async function processNZBsForDebridService(
 
   logger.debug(`Retrieved NZB status from debrid`, {
     service: debridService.serviceName,
-    magnetCount: nzbs.length,
+    nzbCount: nzbs.length,
     time: getTimeTakenSincePoint(startTime),
   });
 
-  // parse all files from all nzbs in one call
-
-  const allStrings: string[] = [];
-
-  for (const nzb of nzbs) {
-    allStrings.push(nzb.title ?? '');
-  }
-
-  for (const nzb of nzbCheckResults) {
-    if (nzb.files && Array.isArray(nzb.files)) {
-      for (const file of nzb.files) {
-        allStrings.push(file.name ?? '');
-      }
-    }
-  }
-
-  const allParsedResults = await PTT.parse(allStrings);
-  const parsedFiles = new Map<string, ParseResult>();
-  for (const [index, result] of allParsedResults.entries()) {
+  // Parse only NZB titles and perform validation checks
+  const nzbTitles = nzbs.map((nzb) => nzb.title ?? '');
+  const parsedTitles = await PTT.parse(nzbTitles);
+  const parsedTitlesMap = new Map<string, ParseResult>();
+  for (const [index, result] of parsedTitles.entries()) {
     if (result) {
-      parsedFiles.set(allStrings[index], result);
+      parsedTitlesMap.set(nzbTitles[index], result);
     }
   }
 
-  const processingStart = Date.now();
-  for (const [index, nzb] of nzbs.entries()) {
-    let file: DebridFile | undefined;
-
+  // Filter NZBs that pass validation checks
+  const validNZBs: {
+    nzb: NZB;
+    nzbCheckResult: any;
+    parsedTitle: ParseResult;
+  }[] = [];
+  for (const nzb of nzbs) {
     const nzbCheckResult = nzbCheckResults.find(
       (result) => result.hash === nzb.hash
     );
-    const parsedNzb = parsedFiles.get(nzb.title ?? '');
+    const parsedNzb = parsedTitlesMap.get(nzb.title ?? '');
     if (metadata && parsedNzb) {
       if (isSeasonWrong(parsedNzb, metadata)) {
         continue;
@@ -390,6 +423,32 @@ async function processNZBsForDebridService(
         continue;
       }
     }
+    validNZBs.push({ nzb, nzbCheckResult, parsedTitle: parsedNzb! });
+  }
+
+  // Parse files only for valid NZBs
+  const allFileStrings: string[] = [];
+  for (const { nzbCheckResult } of validNZBs) {
+    if (nzbCheckResult?.files && Array.isArray(nzbCheckResult.files)) {
+      for (const file of nzbCheckResult.files) {
+        if (isNotVideoFile(file)) continue;
+        allFileStrings.push(file.name ?? '');
+      }
+    }
+  }
+
+  const allParsedFiles = await PTT.parse(allFileStrings);
+  const parsedFiles = new Map<string, ParseResult>();
+  for (const [index, result] of allParsedFiles.entries()) {
+    if (result) {
+      parsedFiles.set(allFileStrings[index], result);
+    }
+  }
+
+  const processingStart = Date.now();
+  for (const { nzb, nzbCheckResult } of validNZBs) {
+    let file: DebridFile | undefined;
+
     file = nzbCheckResult
       ? await selectFileInTorrentOrNZB(
           nzb,

packages/core/src/debrid/utils.ts

@@ -144,6 +144,18 @@ export const isTitleWrong = (
   return false;
 };
 
+export const isTitleWrongN = (
+  parsed: { title?: string },
+  metadata?: { titles?: string[] }
+) => {
+  if (parsed.title && metadata?.titles) {
+    const normalisedParsedTitle = normaliseTitle(parsed.title);
+    return !metadata.titles
+      .map(normaliseTitle)
+      .some((title) => title == normalisedParsedTitle);
+  }
+  return false;
+};
 export async function selectFileInTorrentOrNZB(
   torrentOrNZB: Torrent | NZB,
   debridDownload: DebridDownload,
@@ -167,6 +179,7 @@ export async function selectFileInTorrentOrNZB(
   options?: {
     chosenFilename?: string;
     chosenIndex?: number;
+    useLevenshteinMatching?: boolean;
   }
 ): Promise<DebridFile | undefined> {
   if (!debridDownload.files?.length) {
@@ -178,15 +191,8 @@ export async function selectFileInTorrentOrNZB(
   }
 
   const isVideo = debridDownload.files.map((file) => isVideoFile(file));
-  if (metadata?.titles && metadata.titles.length) {
-    const uniqueTitles = [
-      ...new Set(metadata?.titles.map(normaliseTitle)),
-    ].slice(0, 100);
-    metadata = {
-      ...metadata,
-      titles: uniqueTitles,
-    };
-  }
+  const isNotVideo = debridDownload.files.map((file) => isNotVideoFile(file));
+  const videoExists = isVideo.map((f) => f == true);
 
   // Create a scoring system for each file
   const fileScores = [];
@@ -195,8 +201,13 @@ export async function selectFileInTorrentOrNZB(
     let score = 0;
     const parsed = parsedFiles.get(file.name ?? '');
 
+    if (isNotVideo[index]) {
+      continue;
+    }
+
     if (!parsed) {
       logger.warn(`Parsed file not found for ${file.name}`);
+      continue;
     }
 
     // Base score from video file status (highest priority)
@@ -233,9 +244,12 @@ export async function selectFileInTorrentOrNZB(
     }
 
     // Title matching (third priority)
+    const titleMatchFunc =
+      options?.useLevenshteinMatching == false ? isTitleWrongN : isTitleWrong;
     if (
       parsed?.title &&
-      !isTitleWrong(
+      (videoExists ? isVideo[index] : true) &&
+      !titleMatchFunc(
         {
           title: preprocessTitle(
             parsed.title,
@@ -277,6 +291,10 @@ export async function selectFileInTorrentOrNZB(
     }
   }
 
+  if (fileScores.length === 0) {
+    logger.warn(`Torrent ${torrentOrNZB.title} had no files selected`);
+    return undefined;
+  }
   // Sort by score descending
   fileScores.sort((a, b) => b.score - a.score);
 
@@ -367,6 +385,64 @@ export function isVideoFile(file: DebridFile): boolean {
   );
 }
 
+export function isNotVideoFile(file: DebridFile): boolean {
+  const nonVideoExtensions = [
+    '.txt',
+    '.jpg',
+    '.jpeg',
+    '.png',
+    '.gif',
+    '.bmp',
+    '.svg',
+    '.webp',
+    '.nfo',
+    '.sfv',
+    '.srt',
+    '.ass',
+    '.sub',
+    '.idx',
+    '.cue',
+    '.log',
+    '.doc',
+    '.docx',
+    '.xls',
+    '.xlsx',
+    '.ppt',
+    '.pptx',
+    '.pdf',
+    '.rtf',
+    '.odt',
+    '.ods',
+    '.odp',
+    '.csv',
+    '.tsv',
+    '.exe',
+    '.bat',
+    '.apk',
+    '.dll',
+    '.iso',
+    '.zip',
+    '.rar',
+    '.7z',
+    '.tar',
+    '.gz',
+    '.bz2',
+    '.xz',
+    '.md',
+    '.json',
+    '.xml',
+    '.ini',
+    '.dat',
+    '.db',
+    '.dbf',
+    '.bak',
+  ];
+  return (
+    (file.mimeType && !file.mimeType.includes('video')) ||
+    nonVideoExtensions.some((ext) => file.name?.endsWith(ext) ?? false)
+  );
+}
+
 export const metadataStore = () => {
   const prefix = 'mds';
   const store: 'redis' | 'sql' | 'memory' =