From 460fad22b48ddcf465fd4f126a244bb1295bec1d Mon Sep 17 00:00:00 2001 From: itzme Date: Wed, 10 Dec 2025 11:15:30 +0100 Subject: [PATCH 1/5] feat: decouple Jackett and Prowlarr scraper configurations from a generic indexer manager --- .env-sample | 23 ++++++++++++------ comet/core/logger.py | 19 +++++++++------ comet/core/models.py | 37 +++++++++++++++++++++++++++- comet/scrapers/jackett.py | 6 ++--- comet/scrapers/manager.py | 49 ++++++++++++-------------------------- comet/scrapers/prowlarr.py | 12 ++++------ 6 files changed, 87 insertions(+), 59 deletions(-) diff --git a/.env-sample b/.env-sample index 5aff92d..62b936c 100644 --- a/.env-sample +++ b/.env-sample @@ -64,14 +64,23 @@ BACKGROUND_SCRAPER_MAX_SERIES_PER_RUN=100 # Maximum number of series to scrape p BYPASS_PROXY_URL=http://warp:1080 # To bypass scraper IP blacklists # ============================== # -# Indexer Manager Settings # +# Jackett & Prowlarr Settings # # ============================== # -INDEXER_MANAGER_TYPE=none # Options: jackett, prowlarr, none -INDEXER_MANAGER_URL=http://127.0.0.1:9117 -INDEXER_MANAGER_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -INDEXER_MANAGER_MODE=both # Context mode: live, background, both, false - Controls when indexer is used -INDEXER_MANAGER_TIMEOUT=60 # Max time to get search results (seconds) -INDEXER_MANAGER_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' # Jackett/Prowlarr indexers - get names from https://github.com/Jackett/Jackett/tree/master/src/Jackett.Common/Definitions for Jackett + +# Jackett Configuration +SCRAPE_JACKETT=False # Context mode: live, background, both, false +JACKETT_URL=http://127.0.0.1:9117 +JACKETT_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +JACKETT_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' # Get names from https://github.com/Jackett/Jackett/tree/master/src/Jackett.Common/Definitions + +# Prowlarr Configuration +SCRAPE_PROWLARR=False # Context mode: live, background, both, false +PROWLARR_URL=http://127.0.0.1:9696 +PROWLARR_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +PROWLARR_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' + +# Shared Settings +INDEXER_MANAGER_TIMEOUT=60 # Max time to get search results (seconds) - Shared by both # ============================== # # Torrent Settings # diff --git a/comet/core/logger.py b/comet/core/logger.py index 58e7b5f..810e487 100644 --- a/comet/core/logger.py +++ b/comet/core/logger.py @@ -184,18 +184,23 @@ def log_startup_info(settings): ) logger.log("COMET", f"Bypass Proxy: {settings.BYPASS_PROXY_URL}") - if settings.is_any_context_enabled(settings.INDEXER_MANAGER_MODE): + if settings.is_any_context_enabled(settings.SCRAPE_JACKETT): logger.log( "COMET", - f"Indexer Manager: {settings.INDEXER_MANAGER_TYPE}|{settings.INDEXER_MANAGER_URL} - Mode: {settings.INDEXER_MANAGER_MODE} - Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s", + f"Jackett Scraper: {settings.format_scraper_mode(settings.SCRAPE_JACKETT)} - {settings.JACKETT_URL} - Indexers: {', '.join(settings.JACKETT_INDEXERS)}", ) - logger.log("COMET", f"Indexers: {', '.join(settings.INDEXER_MANAGER_INDEXERS)}") - logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s") + + if settings.is_any_context_enabled(settings.SCRAPE_PROWLARR): logger.log( - "COMET", f"Download Torrent Files: {bool(settings.DOWNLOAD_TORRENT_FILES)}" + "COMET", + f"Prowlarr Scraper: {settings.format_scraper_mode(settings.SCRAPE_PROWLARR)} - {settings.PROWLARR_URL} - Indexers: {', '.join(settings.PROWLARR_INDEXERS)}", ) - else: - logger.log("COMET", "Indexer Manager: False") + + logger.log("COMET", f"Indexer Manager Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s") + logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s") + logger.log( + "COMET", f"Download Torrent Files: {bool(settings.DOWNLOAD_TORRENT_FILES)}" + ) comet_url = ( f" - {settings.COMET_URL}" diff --git a/comet/core/models.py b/comet/core/models.py index 9036b4b..760a56c 100644 --- a/comet/core/models.py +++ b/comet/core/models.py @@ -48,6 +48,14 @@ class AppSettings(BaseSettings): INDEXER_MANAGER_MODE: Union[bool, str] = "both" INDEXER_MANAGER_TIMEOUT: Optional[int] = 30 INDEXER_MANAGER_INDEXERS: List[str] = [] + SCRAPE_JACKETT: Union[bool, str] = False + JACKETT_URL: Optional[str] = "http://127.0.0.1:9117" + JACKETT_API_KEY: Optional[str] = None + JACKETT_INDEXERS: List[str] = [] + SCRAPE_PROWLARR: Union[bool, str] = False + PROWLARR_URL: Optional[str] = "http://127.0.0.1:9696" + PROWLARR_API_KEY: Optional[str] = None + PROWLARR_INDEXERS: List[str] = [] GET_TORRENT_TIMEOUT: Optional[int] = 5 DOWNLOAD_TORRENT_FILES: Optional[bool] = False SCRAPE_COMET: Union[bool, str] = False @@ -108,7 +116,9 @@ class AppSettings(BaseSettings): return None return v - @field_validator("INDEXER_MANAGER_INDEXERS") + @field_validator( + "INDEXER_MANAGER_INDEXERS", "JACKETT_INDEXERS", "PROWLARR_INDEXERS" + ) def indexer_manager_indexers_normalization(cls, v, values): v = [indexer.replace(" ", "").lower() for indexer in v] return v @@ -124,6 +134,8 @@ class AppSettings(BaseSettings): "MEDIAFUSION_URL", "AIOSTREAMS_URL", "JACKETTIO_URL", + "JACKETT_URL", + "PROWLARR_URL", ) def normalize_urls(cls, v): if isinstance(v, str): @@ -166,6 +178,29 @@ class AppSettings(BaseSettings): scraper_setting = scraper_setting.lower() return scraper_setting in ["true", "both", "live", "background"] + def model_post_init(self, __context): + if self.INDEXER_MANAGER_TYPE == "jackett": + if not self.SCRAPE_JACKETT: + self.SCRAPE_JACKETT = self.INDEXER_MANAGER_MODE + if self.JACKETT_URL == "http://127.0.0.1:9117" and self.INDEXER_MANAGER_URL: + self.JACKETT_URL = self.INDEXER_MANAGER_URL + if not self.JACKETT_API_KEY and self.INDEXER_MANAGER_API_KEY: + self.JACKETT_API_KEY = self.INDEXER_MANAGER_API_KEY + if not self.JACKETT_INDEXERS and self.INDEXER_MANAGER_INDEXERS: + self.JACKETT_INDEXERS = self.INDEXER_MANAGER_INDEXERS + elif self.INDEXER_MANAGER_TYPE == "prowlarr": + if not self.SCRAPE_PROWLARR: + self.SCRAPE_PROWLARR = self.INDEXER_MANAGER_MODE + if ( + self.PROWLARR_URL == "http://127.0.0.1:9696" + and self.INDEXER_MANAGER_URL + ): + self.PROWLARR_URL = self.INDEXER_MANAGER_URL + if not self.PROWLARR_API_KEY and self.INDEXER_MANAGER_API_KEY: + self.PROWLARR_API_KEY = self.INDEXER_MANAGER_API_KEY + if not self.PROWLARR_INDEXERS and self.INDEXER_MANAGER_INDEXERS: + self.PROWLARR_INDEXERS = self.INDEXER_MANAGER_INDEXERS + settings = AppSettings() diff --git a/comet/scrapers/jackett.py b/comet/scrapers/jackett.py index 1c8d1cb..aae632a 100644 --- a/comet/scrapers/jackett.py +++ b/comet/scrapers/jackett.py @@ -16,7 +16,7 @@ from comet.services.torrent_manager import (add_torrent_queue, class JackettScraper(BaseScraper): def __init__(self, manager, session: aiohttp.ClientSession): super().__init__(manager, session) - self.url = settings.INDEXER_MANAGER_URL + self.url = settings.JACKETT_URL async def process_torrent(self, result: dict, media_id: str, season: int): base_torrent = { @@ -88,7 +88,7 @@ class JackettScraper(BaseScraper): async def fetch_jackett_results(self, indexer: str, query: str): try: response = await self.session.get( - f"{self.url}/api/v2.0/indexers/all/results?apikey={settings.INDEXER_MANAGER_API_KEY}&Query={query}&Tracker[]={indexer}", + f"{self.url}/api/v2.0/indexers/all/results?apikey={settings.JACKETT_API_KEY}&Query={query}&Tracker[]={indexer}", timeout=aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT), ) response = await response.json() @@ -116,7 +116,7 @@ class JackettScraper(BaseScraper): tasks.extend( [ self.fetch_jackett_results(indexer, query) - for indexer in settings.INDEXER_MANAGER_INDEXERS + for indexer in settings.JACKETT_INDEXERS ] ) diff --git a/comet/scrapers/manager.py b/comet/scrapers/manager.py index c5f3f90..b4a968c 100644 --- a/comet/scrapers/manager.py +++ b/comet/scrapers/manager.py @@ -60,44 +60,25 @@ class ScraperManager: setting_name = scraper_name_clean.upper() setting_key = f"SCRAPE_{setting_name}" - if scraper_name == "JackettScraper": - if not ( - settings.INDEXER_MANAGER_API_KEY - and settings.is_scraper_enabled( - settings.INDEXER_MANAGER_MODE, request.context - ) - and settings.INDEXER_MANAGER_TYPE == "jackett" - ): - continue - elif scraper_name == "ProwlarrScraper": - if not ( - settings.INDEXER_MANAGER_API_KEY - and settings.is_scraper_enabled( - settings.INDEXER_MANAGER_MODE, request.context - ) - and settings.INDEXER_MANAGER_TYPE == "prowlarr" + if hasattr(settings, setting_key): + if not settings.is_scraper_enabled( + getattr(settings, setting_key), request.context ): continue else: - if hasattr(settings, setting_key): - if not settings.is_scraper_enabled( - getattr(settings, setting_key), request.context - ): - continue - else: - logger.debug( - f"No {setting_key} found for {scraper_name_clean}, disabling" - ) - continue + logger.debug( + f"No {setting_key} found for {scraper_name_clean}, disabling" + ) + continue - if ( - scraper_name == "NyaaScraper" - and settings.NYAA_ANIME_ONLY - and not anime_mapper.is_anime_content( - request.media_id, request.media_only_id - ) - ): - continue + if ( + scraper_name == "NyaaScraper" + and settings.NYAA_ANIME_ONLY + and not anime_mapper.is_anime_content( + request.media_id, request.media_only_id + ) + ): + continue if scraper_name == "MediaFusionScraper": url_credentials_pairs = associate_urls_credentials( diff --git a/comet/scrapers/prowlarr.py b/comet/scrapers/prowlarr.py index 29683a2..f0ff6f7 100644 --- a/comet/scrapers/prowlarr.py +++ b/comet/scrapers/prowlarr.py @@ -94,13 +94,11 @@ class ProwlarrScraper(BaseScraper): ) try: - indexers = [ - indexer.lower() for indexer in settings.INDEXER_MANAGER_INDEXERS - ] + indexers = [indexer.lower() for indexer in settings.PROWLARR_INDEXERS] get_indexers = await self.session.get( - f"{settings.INDEXER_MANAGER_URL}/api/v1/indexer", - headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY}, + f"{settings.PROWLARR_URL}/api/v1/indexer", + headers={"X-Api-Key": settings.PROWLARR_API_KEY}, ) get_indexers = await get_indexers.json() @@ -116,8 +114,8 @@ class ProwlarrScraper(BaseScraper): for query in queries: tasks.append( self.session.get( - f"{settings.INDEXER_MANAGER_URL}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search", - headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY}, + f"{settings.PROWLARR_URL}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search", + headers={"X-Api-Key": settings.PROWLARR_API_KEY}, ) ) From c0d70607b28164f6e29ca7481bf17ea0fa126b64 Mon Sep 17 00:00:00 2001 From: itzme Date: Wed, 10 Dec 2025 11:25:04 +0100 Subject: [PATCH 2/5] feat: add timeouts to Prowlarr API requests --- .env-sample | 5 +++-- comet/scrapers/prowlarr.py | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.env-sample b/.env-sample index 62b936c..949a180 100644 --- a/.env-sample +++ b/.env-sample @@ -98,7 +98,7 @@ DOWNLOAD_TORRENT_FILES=False # Enable torrent file retrieval (instead of magnet # - Example multi: COMET_URL='["https://comet1.example.com", "https://comet2.example.com"]' # # Scraper Context Modes: -# Each SCRAPE_* setting and INDEXER_MANAGER_MODE can control when scrapers are used: +# Each SCRAPE_* setting can control when scrapers are used: # - true/both: Used for live scraping AND background scraping (default) # - live: Only used for live scraping (when users request content) # - background: Only used for background scraping (automatic content pre-caching) @@ -109,7 +109,8 @@ DOWNLOAD_TORRENT_FILES=False # Enable torrent file retrieval (instead of magnet # SCRAPE_TORRENTIO=live # Fast live scraping only # SCRAPE_ZILEAN=background # Background cache building only # SCRAPE_NYAA=false # Completely disabled -# INDEXER_MANAGER_MODE=live # Jackett/Prowlarr for live scraping only +# SCRAPE_JACKETT=live # Jackett for live scraping only +# SCRAPE_PROWLARR=background # Prowlarr for background scraping only SCRAPE_COMET=False COMET_URL=https://comet.elfhosted.com diff --git a/comet/scrapers/prowlarr.py b/comet/scrapers/prowlarr.py index f0ff6f7..4501a64 100644 --- a/comet/scrapers/prowlarr.py +++ b/comet/scrapers/prowlarr.py @@ -99,6 +99,7 @@ class ProwlarrScraper(BaseScraper): get_indexers = await self.session.get( f"{settings.PROWLARR_URL}/api/v1/indexer", headers={"X-Api-Key": settings.PROWLARR_API_KEY}, + timeout=aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT), ) get_indexers = await get_indexers.json() @@ -116,6 +117,9 @@ class ProwlarrScraper(BaseScraper): self.session.get( f"{settings.PROWLARR_URL}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search", headers={"X-Api-Key": settings.PROWLARR_API_KEY}, + timeout=aiohttp.ClientTimeout( + total=settings.INDEXER_MANAGER_TIMEOUT + ), ) ) From 72825f1145b93ea6f0f40be1d5bb857845fbe674 Mon Sep 17 00:00:00 2001 From: g0ldyy <153996346+g0ldyy@users.noreply.github.com> Date: Wed, 10 Dec 2025 16:56:30 +0100 Subject: [PATCH 3/5] refactor: Jackett and Prowlarr scrapers now accept URL via constructor --- comet/core/logger.py | 20 ++++++++++++-------- comet/scrapers/jackett.py | 5 ++--- comet/scrapers/prowlarr.py | 8 ++++---- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/comet/core/logger.py b/comet/core/logger.py index 810e487..b4362ce 100644 --- a/comet/core/logger.py +++ b/comet/core/logger.py @@ -184,17 +184,21 @@ def log_startup_info(settings): ) logger.log("COMET", f"Bypass Proxy: {settings.BYPASS_PROXY_URL}") + jackett_info = "" if settings.is_any_context_enabled(settings.SCRAPE_JACKETT): - logger.log( - "COMET", - f"Jackett Scraper: {settings.format_scraper_mode(settings.SCRAPE_JACKETT)} - {settings.JACKETT_URL} - Indexers: {', '.join(settings.JACKETT_INDEXERS)}", - ) + jackett_info = f" - {settings.JACKETT_URL} - Indexers: {', '.join(settings.JACKETT_INDEXERS)}" + logger.log( + "COMET", + f"Jackett Scraper: {settings.format_scraper_mode(settings.SCRAPE_JACKETT)}{jackett_info}", + ) + prowlarr_info = "" if settings.is_any_context_enabled(settings.SCRAPE_PROWLARR): - logger.log( - "COMET", - f"Prowlarr Scraper: {settings.format_scraper_mode(settings.SCRAPE_PROWLARR)} - {settings.PROWLARR_URL} - Indexers: {', '.join(settings.PROWLARR_INDEXERS)}", - ) + prowlarr_info = f" - {settings.PROWLARR_URL} - Indexers: {', '.join(settings.PROWLARR_INDEXERS)}" + logger.log( + "COMET", + f"Prowlarr Scraper: {settings.format_scraper_mode(settings.SCRAPE_PROWLARR)}{prowlarr_info}", + ) logger.log("COMET", f"Indexer Manager Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s") logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s") diff --git a/comet/scrapers/jackett.py b/comet/scrapers/jackett.py index aae632a..97f4b0a 100644 --- a/comet/scrapers/jackett.py +++ b/comet/scrapers/jackett.py @@ -14,9 +14,8 @@ from comet.services.torrent_manager import (add_torrent_queue, class JackettScraper(BaseScraper): - def __init__(self, manager, session: aiohttp.ClientSession): - super().__init__(manager, session) - self.url = settings.JACKETT_URL + def __init__(self, manager, session: aiohttp.ClientSession, url: str): + super().__init__(manager, session, url) async def process_torrent(self, result: dict, media_id: str, season: int): base_torrent = { diff --git a/comet/scrapers/prowlarr.py b/comet/scrapers/prowlarr.py index 4501a64..3f92e7d 100644 --- a/comet/scrapers/prowlarr.py +++ b/comet/scrapers/prowlarr.py @@ -14,8 +14,8 @@ from comet.services.torrent_manager import (add_torrent_queue, class ProwlarrScraper(BaseScraper): - def __init__(self, manager, session: aiohttp.ClientSession): - super().__init__(manager, session) + def __init__(self, manager, session: aiohttp.ClientSession, url: str): + super().__init__(manager, session, url) async def process_torrent(self, result: dict, media_id: str, season: int): base_torrent = { @@ -97,7 +97,7 @@ class ProwlarrScraper(BaseScraper): indexers = [indexer.lower() for indexer in settings.PROWLARR_INDEXERS] get_indexers = await self.session.get( - f"{settings.PROWLARR_URL}/api/v1/indexer", + f"{self.url}/api/v1/indexer", headers={"X-Api-Key": settings.PROWLARR_API_KEY}, timeout=aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT), ) @@ -115,7 +115,7 @@ class ProwlarrScraper(BaseScraper): for query in queries: tasks.append( self.session.get( - f"{settings.PROWLARR_URL}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search", + f"{self.url}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search", headers={"X-Api-Key": settings.PROWLARR_API_KEY}, timeout=aiohttp.ClientTimeout( total=settings.INDEXER_MANAGER_TIMEOUT From 1c8903d57e278e64735eec589c555e248088f346 Mon Sep 17 00:00:00 2001 From: g0ldyy <153996346+g0ldyy@users.noreply.github.com> Date: Wed, 10 Dec 2025 17:06:53 +0100 Subject: [PATCH 4/5] refactor: extract aiohttp client timeouts into named constants --- comet/background_scraper/cinemata_client.py | 6 +++--- comet/scrapers/jackett.py | 4 +++- comet/scrapers/prowlarr.py | 8 ++++---- comet/services/torrent_manager.py | 3 ++- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/comet/background_scraper/cinemata_client.py b/comet/background_scraper/cinemata_client.py index 316ffc9..c82116e 100644 --- a/comet/background_scraper/cinemata_client.py +++ b/comet/background_scraper/cinemata_client.py @@ -2,6 +2,8 @@ import aiohttp from comet.core.logger import logger +CATALOG_TIMEOUT = aiohttp.ClientTimeout(total=30) + class CinemataClient: BASE_URL = "https://cinemeta-catalogs.strem.io" @@ -28,9 +30,7 @@ class CinemataClient: url = "/".join(url_parts) + f"/skip={skip}.json" try: - async with self.session.get( - url, timeout=aiohttp.ClientTimeout(total=30) - ) as response: + async with self.session.get(url, timeout=CATALOG_TIMEOUT) as response: response.raise_for_status() data = await response.json() return data diff --git a/comet/scrapers/jackett.py b/comet/scrapers/jackett.py index 97f4b0a..de855ed 100644 --- a/comet/scrapers/jackett.py +++ b/comet/scrapers/jackett.py @@ -12,6 +12,8 @@ from comet.services.torrent_manager import (add_torrent_queue, extract_torrent_metadata, extract_trackers_from_magnet) +INDEXER_TIMEOUT = aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT) + class JackettScraper(BaseScraper): def __init__(self, manager, session: aiohttp.ClientSession, url: str): @@ -88,7 +90,7 @@ class JackettScraper(BaseScraper): try: response = await self.session.get( f"{self.url}/api/v2.0/indexers/all/results?apikey={settings.JACKETT_API_KEY}&Query={query}&Tracker[]={indexer}", - timeout=aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT), + timeout=INDEXER_TIMEOUT, ) response = await response.json() return response.get("Results", []) diff --git a/comet/scrapers/prowlarr.py b/comet/scrapers/prowlarr.py index 3f92e7d..9bedc19 100644 --- a/comet/scrapers/prowlarr.py +++ b/comet/scrapers/prowlarr.py @@ -12,6 +12,8 @@ from comet.services.torrent_manager import (add_torrent_queue, extract_torrent_metadata, extract_trackers_from_magnet) +INDEXER_TIMEOUT = aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT) + class ProwlarrScraper(BaseScraper): def __init__(self, manager, session: aiohttp.ClientSession, url: str): @@ -99,7 +101,7 @@ class ProwlarrScraper(BaseScraper): get_indexers = await self.session.get( f"{self.url}/api/v1/indexer", headers={"X-Api-Key": settings.PROWLARR_API_KEY}, - timeout=aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT), + timeout=INDEXER_TIMEOUT, ) get_indexers = await get_indexers.json() @@ -117,9 +119,7 @@ class ProwlarrScraper(BaseScraper): self.session.get( f"{self.url}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search", headers={"X-Api-Key": settings.PROWLARR_API_KEY}, - timeout=aiohttp.ClientTimeout( - total=settings.INDEXER_MANAGER_TIMEOUT - ), + timeout=INDEXER_TIMEOUT, ) ) diff --git a/comet/services/torrent_manager.py b/comet/services/torrent_manager.py index 341a3e5..88beb13 100644 --- a/comet/services/torrent_manager.py +++ b/comet/services/torrent_manager.py @@ -19,6 +19,7 @@ from comet.core.models import database, settings from comet.utils.parsing import default_dump, is_video INFO_HASH_PATTERN = re.compile(r"btih:([a-fA-F0-9]{40}|[a-zA-Z0-9]{32})") +TORRENT_TIMEOUT = aiohttp.ClientTimeout(total=settings.GET_TORRENT_TIMEOUT) def extract_trackers_from_magnet(magnet_uri: str): @@ -34,7 +35,7 @@ def extract_trackers_from_magnet(magnet_uri: str): async def download_torrent(session: aiohttp.ClientSession, url: str): try: - timeout = aiohttp.ClientTimeout(total=settings.GET_TORRENT_TIMEOUT) + timeout = TORRENT_TIMEOUT async with session.get(url, allow_redirects=False, timeout=timeout) as response: if response.status == 200: return (await response.read(), None, None) From eb3b5e87c76729ba15c9110dd068256f3f57bbf6 Mon Sep 17 00:00:00 2001 From: g0ldyy <153996346+g0ldyy@users.noreply.github.com> Date: Wed, 10 Dec 2025 17:23:17 +0100 Subject: [PATCH 5/5] feat: add catalog and magnet resolve timeouts to configuration --- .env-sample | 3 +++ comet/background_scraper/cinemata_client.py | 3 +-- comet/core/constants.py | 7 +++++++ comet/core/logger.py | 1 + comet/core/models.py | 2 ++ comet/scrapers/jackett.py | 3 +-- comet/scrapers/prowlarr.py | 3 +-- comet/services/torrent_manager.py | 9 +++++---- 8 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 comet/core/constants.py diff --git a/.env-sample b/.env-sample index 949a180..84e2f93 100644 --- a/.env-sample +++ b/.env-sample @@ -57,6 +57,8 @@ BACKGROUND_SCRAPER_CONCURRENT_WORKERS=1 # Number of concurrent workers for scrap BACKGROUND_SCRAPER_INTERVAL=3600 # Interval between scraping cycles in seconds BACKGROUND_SCRAPER_MAX_MOVIES_PER_RUN=100 # Maximum number of movies to scrape per run BACKGROUND_SCRAPER_MAX_SERIES_PER_RUN=100 # Maximum number of series to scrape per run +CATALOG_TIMEOUT=30 # Max time to fetch catalog pages (seconds) + # ============================== # # Proxy Configuration # @@ -87,6 +89,7 @@ INDEXER_MANAGER_TIMEOUT=60 # Max time to get search results (seconds) - Shared # ============================== # GET_TORRENT_TIMEOUT=5 # Max time to obtain torrent info hash (seconds) DOWNLOAD_TORRENT_FILES=False # Enable torrent file retrieval (instead of magnet link only) +MAGNET_RESOLVE_TIMEOUT=60 # Max time to resolve a magnet link (seconds) # ============================== # # Scraping Configuration # diff --git a/comet/background_scraper/cinemata_client.py b/comet/background_scraper/cinemata_client.py index c82116e..a7d57b9 100644 --- a/comet/background_scraper/cinemata_client.py +++ b/comet/background_scraper/cinemata_client.py @@ -1,9 +1,8 @@ import aiohttp +from comet.core.constants import CATALOG_TIMEOUT from comet.core.logger import logger -CATALOG_TIMEOUT = aiohttp.ClientTimeout(total=30) - class CinemataClient: BASE_URL = "https://cinemeta-catalogs.strem.io" diff --git a/comet/core/constants.py b/comet/core/constants.py new file mode 100644 index 0000000..f2b25f0 --- /dev/null +++ b/comet/core/constants.py @@ -0,0 +1,7 @@ +import aiohttp + +from comet.core.models import settings + +INDEXER_TIMEOUT = aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT) +TORRENT_TIMEOUT = aiohttp.ClientTimeout(total=settings.GET_TORRENT_TIMEOUT) +CATALOG_TIMEOUT = aiohttp.ClientTimeout(total=settings.CATALOG_TIMEOUT) diff --git a/comet/core/logger.py b/comet/core/logger.py index b4362ce..5cb4b3e 100644 --- a/comet/core/logger.py +++ b/comet/core/logger.py @@ -202,6 +202,7 @@ def log_startup_info(settings): logger.log("COMET", f"Indexer Manager Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s") logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s") + logger.log("COMET", f"Magnet Resolve Timeout: {settings.MAGNET_RESOLVE_TIMEOUT}s") logger.log( "COMET", f"Download Torrent Files: {bool(settings.DOWNLOAD_TORRENT_FILES)}" ) diff --git a/comet/core/models.py b/comet/core/models.py index 760a56c..653201b 100644 --- a/comet/core/models.py +++ b/comet/core/models.py @@ -57,6 +57,8 @@ class AppSettings(BaseSettings): PROWLARR_API_KEY: Optional[str] = None PROWLARR_INDEXERS: List[str] = [] GET_TORRENT_TIMEOUT: Optional[int] = 5 + MAGNET_RESOLVE_TIMEOUT: Optional[int] = 60 + CATALOG_TIMEOUT: Optional[int] = 30 DOWNLOAD_TORRENT_FILES: Optional[bool] = False SCRAPE_COMET: Union[bool, str] = False COMET_URL: Union[str, List[str]] = "https://comet.elfhosted.com" diff --git a/comet/scrapers/jackett.py b/comet/scrapers/jackett.py index de855ed..e95f370 100644 --- a/comet/scrapers/jackett.py +++ b/comet/scrapers/jackett.py @@ -3,6 +3,7 @@ from typing import List, Set import aiohttp +from comet.core.constants import INDEXER_TIMEOUT from comet.core.logger import logger from comet.core.models import settings from comet.scrapers.base import BaseScraper @@ -12,8 +13,6 @@ from comet.services.torrent_manager import (add_torrent_queue, extract_torrent_metadata, extract_trackers_from_magnet) -INDEXER_TIMEOUT = aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT) - class JackettScraper(BaseScraper): def __init__(self, manager, session: aiohttp.ClientSession, url: str): diff --git a/comet/scrapers/prowlarr.py b/comet/scrapers/prowlarr.py index 9bedc19..fc5dfb3 100644 --- a/comet/scrapers/prowlarr.py +++ b/comet/scrapers/prowlarr.py @@ -3,6 +3,7 @@ from typing import List, Set import aiohttp +from comet.core.constants import INDEXER_TIMEOUT from comet.core.logger import logger from comet.core.models import settings from comet.scrapers.base import BaseScraper @@ -12,8 +13,6 @@ from comet.services.torrent_manager import (add_torrent_queue, extract_torrent_metadata, extract_trackers_from_magnet) -INDEXER_TIMEOUT = aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT) - class ProwlarrScraper(BaseScraper): def __init__(self, manager, session: aiohttp.ClientSession, url: str): diff --git a/comet/services/torrent_manager.py b/comet/services/torrent_manager.py index 88beb13..2368e12 100644 --- a/comet/services/torrent_manager.py +++ b/comet/services/torrent_manager.py @@ -14,12 +14,12 @@ from demagnetize.core import Demagnetizer from RTN import ParsedData, parse from torf import Magnet +from comet.core.constants import TORRENT_TIMEOUT from comet.core.logger import logger from comet.core.models import database, settings from comet.utils.parsing import default_dump, is_video INFO_HASH_PATTERN = re.compile(r"btih:([a-fA-F0-9]{40}|[a-zA-Z0-9]{32})") -TORRENT_TIMEOUT = aiohttp.ClientTimeout(total=settings.GET_TORRENT_TIMEOUT) def extract_trackers_from_magnet(magnet_uri: str): @@ -35,8 +35,9 @@ def extract_trackers_from_magnet(magnet_uri: str): async def download_torrent(session: aiohttp.ClientSession, url: str): try: - timeout = TORRENT_TIMEOUT - async with session.get(url, allow_redirects=False, timeout=timeout) as response: + async with session.get( + url, allow_redirects=False, timeout=TORRENT_TIMEOUT + ) as response: if response.status == 200: return (await response.read(), None, None) @@ -64,7 +65,7 @@ demagnetizer = Demagnetizer() async def get_torrent_from_magnet(magnet_uri: str): try: magnet = Magnet.from_string(magnet_uri) - with anyio.fail_after(60): + with anyio.fail_after(settings.MAGNET_RESOLVE_TIMEOUT): torrent_data = await demagnetizer.demagnetize(magnet) if torrent_data: return torrent_data.dump()