From 460fad22b48ddcf465fd4f126a244bb1295bec1d Mon Sep 17 00:00:00 2001 From: itzme Date: Wed, 10 Dec 2025 11:15:30 +0100 Subject: [PATCH] feat: decouple Jackett and Prowlarr scraper configurations from a generic indexer manager --- .env-sample | 23 ++++++++++++------ comet/core/logger.py | 19 +++++++++------ comet/core/models.py | 37 +++++++++++++++++++++++++++- comet/scrapers/jackett.py | 6 ++--- comet/scrapers/manager.py | 49 ++++++++++++-------------------------- comet/scrapers/prowlarr.py | 12 ++++------ 6 files changed, 87 insertions(+), 59 deletions(-) diff --git a/.env-sample b/.env-sample index 5aff92d..62b936c 100644 --- a/.env-sample +++ b/.env-sample @@ -64,14 +64,23 @@ BACKGROUND_SCRAPER_MAX_SERIES_PER_RUN=100 # Maximum number of series to scrape p BYPASS_PROXY_URL=http://warp:1080 # To bypass scraper IP blacklists # ============================== # -# Indexer Manager Settings # +# Jackett & Prowlarr Settings # # ============================== # -INDEXER_MANAGER_TYPE=none # Options: jackett, prowlarr, none -INDEXER_MANAGER_URL=http://127.0.0.1:9117 -INDEXER_MANAGER_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -INDEXER_MANAGER_MODE=both # Context mode: live, background, both, false - Controls when indexer is used -INDEXER_MANAGER_TIMEOUT=60 # Max time to get search results (seconds) -INDEXER_MANAGER_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' # Jackett/Prowlarr indexers - get names from https://github.com/Jackett/Jackett/tree/master/src/Jackett.Common/Definitions for Jackett + +# Jackett Configuration +SCRAPE_JACKETT=False # Context mode: live, background, both, false +JACKETT_URL=http://127.0.0.1:9117 +JACKETT_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +JACKETT_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' # Get names from https://github.com/Jackett/Jackett/tree/master/src/Jackett.Common/Definitions + +# Prowlarr Configuration +SCRAPE_PROWLARR=False # Context mode: live, background, both, false +PROWLARR_URL=http://127.0.0.1:9696 +PROWLARR_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +PROWLARR_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' + +# Shared Settings +INDEXER_MANAGER_TIMEOUT=60 # Max time to get search results (seconds) - Shared by both # ============================== # # Torrent Settings # diff --git a/comet/core/logger.py b/comet/core/logger.py index 58e7b5f..810e487 100644 --- a/comet/core/logger.py +++ b/comet/core/logger.py @@ -184,18 +184,23 @@ def log_startup_info(settings): ) logger.log("COMET", f"Bypass Proxy: {settings.BYPASS_PROXY_URL}") - if settings.is_any_context_enabled(settings.INDEXER_MANAGER_MODE): + if settings.is_any_context_enabled(settings.SCRAPE_JACKETT): logger.log( "COMET", - f"Indexer Manager: {settings.INDEXER_MANAGER_TYPE}|{settings.INDEXER_MANAGER_URL} - Mode: {settings.INDEXER_MANAGER_MODE} - Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s", + f"Jackett Scraper: {settings.format_scraper_mode(settings.SCRAPE_JACKETT)} - {settings.JACKETT_URL} - Indexers: {', '.join(settings.JACKETT_INDEXERS)}", ) - logger.log("COMET", f"Indexers: {', '.join(settings.INDEXER_MANAGER_INDEXERS)}") - logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s") + + if settings.is_any_context_enabled(settings.SCRAPE_PROWLARR): logger.log( - "COMET", f"Download Torrent Files: {bool(settings.DOWNLOAD_TORRENT_FILES)}" + "COMET", + f"Prowlarr Scraper: {settings.format_scraper_mode(settings.SCRAPE_PROWLARR)} - {settings.PROWLARR_URL} - Indexers: {', '.join(settings.PROWLARR_INDEXERS)}", ) - else: - logger.log("COMET", "Indexer Manager: False") + + logger.log("COMET", f"Indexer Manager Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s") + logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s") + logger.log( + "COMET", f"Download Torrent Files: {bool(settings.DOWNLOAD_TORRENT_FILES)}" + ) comet_url = ( f" - {settings.COMET_URL}" diff --git a/comet/core/models.py b/comet/core/models.py index 9036b4b..760a56c 100644 --- a/comet/core/models.py +++ b/comet/core/models.py @@ -48,6 +48,14 @@ class AppSettings(BaseSettings): INDEXER_MANAGER_MODE: Union[bool, str] = "both" INDEXER_MANAGER_TIMEOUT: Optional[int] = 30 INDEXER_MANAGER_INDEXERS: List[str] = [] + SCRAPE_JACKETT: Union[bool, str] = False + JACKETT_URL: Optional[str] = "http://127.0.0.1:9117" + JACKETT_API_KEY: Optional[str] = None + JACKETT_INDEXERS: List[str] = [] + SCRAPE_PROWLARR: Union[bool, str] = False + PROWLARR_URL: Optional[str] = "http://127.0.0.1:9696" + PROWLARR_API_KEY: Optional[str] = None + PROWLARR_INDEXERS: List[str] = [] GET_TORRENT_TIMEOUT: Optional[int] = 5 DOWNLOAD_TORRENT_FILES: Optional[bool] = False SCRAPE_COMET: Union[bool, str] = False @@ -108,7 +116,9 @@ class AppSettings(BaseSettings): return None return v - @field_validator("INDEXER_MANAGER_INDEXERS") + @field_validator( + "INDEXER_MANAGER_INDEXERS", "JACKETT_INDEXERS", "PROWLARR_INDEXERS" + ) def indexer_manager_indexers_normalization(cls, v, values): v = [indexer.replace(" ", "").lower() for indexer in v] return v @@ -124,6 +134,8 @@ class AppSettings(BaseSettings): "MEDIAFUSION_URL", "AIOSTREAMS_URL", "JACKETTIO_URL", + "JACKETT_URL", + "PROWLARR_URL", ) def normalize_urls(cls, v): if isinstance(v, str): @@ -166,6 +178,29 @@ class AppSettings(BaseSettings): scraper_setting = scraper_setting.lower() return scraper_setting in ["true", "both", "live", "background"] + def model_post_init(self, __context): + if self.INDEXER_MANAGER_TYPE == "jackett": + if not self.SCRAPE_JACKETT: + self.SCRAPE_JACKETT = self.INDEXER_MANAGER_MODE + if self.JACKETT_URL == "http://127.0.0.1:9117" and self.INDEXER_MANAGER_URL: + self.JACKETT_URL = self.INDEXER_MANAGER_URL + if not self.JACKETT_API_KEY and self.INDEXER_MANAGER_API_KEY: + self.JACKETT_API_KEY = self.INDEXER_MANAGER_API_KEY + if not self.JACKETT_INDEXERS and self.INDEXER_MANAGER_INDEXERS: + self.JACKETT_INDEXERS = self.INDEXER_MANAGER_INDEXERS + elif self.INDEXER_MANAGER_TYPE == "prowlarr": + if not self.SCRAPE_PROWLARR: + self.SCRAPE_PROWLARR = self.INDEXER_MANAGER_MODE + if ( + self.PROWLARR_URL == "http://127.0.0.1:9696" + and self.INDEXER_MANAGER_URL + ): + self.PROWLARR_URL = self.INDEXER_MANAGER_URL + if not self.PROWLARR_API_KEY and self.INDEXER_MANAGER_API_KEY: + self.PROWLARR_API_KEY = self.INDEXER_MANAGER_API_KEY + if not self.PROWLARR_INDEXERS and self.INDEXER_MANAGER_INDEXERS: + self.PROWLARR_INDEXERS = self.INDEXER_MANAGER_INDEXERS + settings = AppSettings() diff --git a/comet/scrapers/jackett.py b/comet/scrapers/jackett.py index 1c8d1cb..aae632a 100644 --- a/comet/scrapers/jackett.py +++ b/comet/scrapers/jackett.py @@ -16,7 +16,7 @@ from comet.services.torrent_manager import (add_torrent_queue, class JackettScraper(BaseScraper): def __init__(self, manager, session: aiohttp.ClientSession): super().__init__(manager, session) - self.url = settings.INDEXER_MANAGER_URL + self.url = settings.JACKETT_URL async def process_torrent(self, result: dict, media_id: str, season: int): base_torrent = { @@ -88,7 +88,7 @@ class JackettScraper(BaseScraper): async def fetch_jackett_results(self, indexer: str, query: str): try: response = await self.session.get( - f"{self.url}/api/v2.0/indexers/all/results?apikey={settings.INDEXER_MANAGER_API_KEY}&Query={query}&Tracker[]={indexer}", + f"{self.url}/api/v2.0/indexers/all/results?apikey={settings.JACKETT_API_KEY}&Query={query}&Tracker[]={indexer}", timeout=aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT), ) response = await response.json() @@ -116,7 +116,7 @@ class JackettScraper(BaseScraper): tasks.extend( [ self.fetch_jackett_results(indexer, query) - for indexer in settings.INDEXER_MANAGER_INDEXERS + for indexer in settings.JACKETT_INDEXERS ] ) diff --git a/comet/scrapers/manager.py b/comet/scrapers/manager.py index c5f3f90..b4a968c 100644 --- a/comet/scrapers/manager.py +++ b/comet/scrapers/manager.py @@ -60,44 +60,25 @@ class ScraperManager: setting_name = scraper_name_clean.upper() setting_key = f"SCRAPE_{setting_name}" - if scraper_name == "JackettScraper": - if not ( - settings.INDEXER_MANAGER_API_KEY - and settings.is_scraper_enabled( - settings.INDEXER_MANAGER_MODE, request.context - ) - and settings.INDEXER_MANAGER_TYPE == "jackett" - ): - continue - elif scraper_name == "ProwlarrScraper": - if not ( - settings.INDEXER_MANAGER_API_KEY - and settings.is_scraper_enabled( - settings.INDEXER_MANAGER_MODE, request.context - ) - and settings.INDEXER_MANAGER_TYPE == "prowlarr" + if hasattr(settings, setting_key): + if not settings.is_scraper_enabled( + getattr(settings, setting_key), request.context ): continue else: - if hasattr(settings, setting_key): - if not settings.is_scraper_enabled( - getattr(settings, setting_key), request.context - ): - continue - else: - logger.debug( - f"No {setting_key} found for {scraper_name_clean}, disabling" - ) - continue + logger.debug( + f"No {setting_key} found for {scraper_name_clean}, disabling" + ) + continue - if ( - scraper_name == "NyaaScraper" - and settings.NYAA_ANIME_ONLY - and not anime_mapper.is_anime_content( - request.media_id, request.media_only_id - ) - ): - continue + if ( + scraper_name == "NyaaScraper" + and settings.NYAA_ANIME_ONLY + and not anime_mapper.is_anime_content( + request.media_id, request.media_only_id + ) + ): + continue if scraper_name == "MediaFusionScraper": url_credentials_pairs = associate_urls_credentials( diff --git a/comet/scrapers/prowlarr.py b/comet/scrapers/prowlarr.py index 29683a2..f0ff6f7 100644 --- a/comet/scrapers/prowlarr.py +++ b/comet/scrapers/prowlarr.py @@ -94,13 +94,11 @@ class ProwlarrScraper(BaseScraper): ) try: - indexers = [ - indexer.lower() for indexer in settings.INDEXER_MANAGER_INDEXERS - ] + indexers = [indexer.lower() for indexer in settings.PROWLARR_INDEXERS] get_indexers = await self.session.get( - f"{settings.INDEXER_MANAGER_URL}/api/v1/indexer", - headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY}, + f"{settings.PROWLARR_URL}/api/v1/indexer", + headers={"X-Api-Key": settings.PROWLARR_API_KEY}, ) get_indexers = await get_indexers.json() @@ -116,8 +114,8 @@ class ProwlarrScraper(BaseScraper): for query in queries: tasks.append( self.session.get( - f"{settings.INDEXER_MANAGER_URL}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search", - headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY}, + f"{settings.PROWLARR_URL}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search", + headers={"X-Api-Key": settings.PROWLARR_API_KEY}, ) )