From c26140d6ce5ef0474ca2d1f616af138560f158ed Mon Sep 17 00:00:00 2001 From: g0ldyy <153996346+g0ldyy@users.noreply.github.com> Date: Sun, 4 Jan 2026 20:38:55 +0100 Subject: [PATCH] feat: add TorrentsDB scraper and remove redundant `pass` statements in other scrapers --- .env-sample | 2 ++ comet/core/logger.py | 5 ++++ comet/core/models.py | 1 + comet/scrapers/comet.py | 1 - comet/scrapers/mediafusion.py | 1 - comet/scrapers/torrentsdb.py | 54 +++++++++++++++++++++++++++++++++++ 6 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 comet/scrapers/torrentsdb.py diff --git a/.env-sample b/.env-sample index 5d7a055..177b4ee 100644 --- a/.env-sample +++ b/.env-sample @@ -216,6 +216,8 @@ DEBRIDIO_PROVIDER_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX SCRAPE_TORBOX=False TORBOX_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +SCRAPE_TORRENTSDB=False + # ============================== # # Debrid Stream Proxy Settings # # ============================== # diff --git a/comet/core/logger.py b/comet/core/logger.py index fc3e343..0bbca1e 100644 --- a/comet/core/logger.py +++ b/comet/core/logger.py @@ -370,6 +370,11 @@ def log_startup_info(settings): f"TorBox Scraper: {settings.format_scraper_mode(settings.SCRAPE_TORBOX)}{torbox_api_key}", ) + logger.log( + "COMET", + f"TorrentsDB Scraper: {settings.format_scraper_mode(settings.SCRAPE_TORRENTSDB)}", + ) + debrid_stream_proxy_display = ( f" - Password: {settings.PROXY_DEBRID_STREAM_PASSWORD} - Max Connections: {settings.PROXY_DEBRID_STREAM_MAX_CONNECTIONS} - Inactivity Threshold: {settings.PROXY_DEBRID_STREAM_INACTIVITY_THRESHOLD}s - Default Debrid Service: {settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE} - Default Debrid API Key: {settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY}" if settings.PROXY_DEBRID_STREAM diff --git a/comet/core/models.py b/comet/core/models.py index 70492a3..cdd5cf6 100644 --- a/comet/core/models.py +++ b/comet/core/models.py @@ -100,6 +100,7 @@ class AppSettings(BaseSettings): DEBRIDIO_PROVIDER_KEY: Optional[str] = None SCRAPE_TORBOX: Union[bool, str] = False TORBOX_API_KEY: Optional[str] = None + SCRAPE_TORRENTSDB: Union[bool, str] = False CUSTOM_HEADER_HTML: Optional[str] = None PROXY_DEBRID_STREAM: Optional[bool] = False PROXY_DEBRID_STREAM_PASSWORD: Optional[str] = "".join( diff --git a/comet/scrapers/comet.py b/comet/scrapers/comet.py index f4401fe..48d8dfd 100644 --- a/comet/scrapers/comet.py +++ b/comet/scrapers/comet.py @@ -39,6 +39,5 @@ class CometScraper(BaseScraper): ) except Exception as e: log_scraper_error("Comet", self.url, request.media_id, e) - pass return torrents diff --git a/comet/scrapers/mediafusion.py b/comet/scrapers/mediafusion.py index 2cf9a57..aa29586 100644 --- a/comet/scrapers/mediafusion.py +++ b/comet/scrapers/mediafusion.py @@ -53,6 +53,5 @@ class MediaFusionScraper(BaseScraper): ) except Exception as e: log_scraper_error("MediaFusion", self.url, request.media_id, e) - pass return torrents diff --git a/comet/scrapers/torrentsdb.py b/comet/scrapers/torrentsdb.py new file mode 100644 index 0000000..57338b9 --- /dev/null +++ b/comet/scrapers/torrentsdb.py @@ -0,0 +1,54 @@ +import re + +from comet.core.logger import log_scraper_error +from comet.scrapers.base import BaseScraper +from comet.scrapers.models import ScrapeRequest +from comet.utils.formatting import size_to_bytes + +METADATA_PATTERN = re.compile( + r"(?:📅 S\d+E\d+ )?(?:👤 (\d+) )?💾 ([\d.]+ [KMGT]?B)(?: ⚙️ (.+))?", re.IGNORECASE +) + + +class TorrentsDBScraper(BaseScraper): + def __init__(self, manager, session): + super().__init__(manager, session) + + async def scrape(self, request: ScrapeRequest): + torrents = [] + try: + async with self.session.get( + f"https://torrentsdb.com/stream/{request.media_type}/{request.media_id}.json", + ) as response: + results = await response.json() + + for torrent in results["streams"]: + description = torrent["title"] + + lines = description.split("\n") + title = lines[0] + metadata_line = lines[-1] + + match = METADATA_PATTERN.search(metadata_line) + + seeders = int(match.group(1)) if match and match.group(1) else None + size = size_to_bytes(match.group(2)) if match and match.group(2) else 0 + tracker = match.group(3) if match and match.group(3) else None + + torrents.append( + { + "title": title, + "infoHash": torrent["infoHash"].lower(), + "fileIndex": torrent.get("fileIdx", None), + "seeders": seeders, + "size": size, + "tracker": f"TorrentsDB|{tracker}" if tracker else "TorrentsDB", + "sources": torrent.get("sources", []), + } + ) + except Exception as e: + log_scraper_error( + "TorrentsDB", "https://torrentsdb.com", request.media_id, e + ) + + return torrents