diff --git a/.env-sample b/.env-sample index 6c193a4..fe20566 100644 --- a/.env-sample +++ b/.env-sample @@ -60,6 +60,11 @@ BACKGROUND_SCRAPER_CONCURRENT_WORKERS=1 # Number of concurrent workers for scrap BACKGROUND_SCRAPER_INTERVAL=3600 # Interval between scraping cycles in seconds BACKGROUND_SCRAPER_MAX_MOVIES_PER_RUN=100 # Maximum number of movies to scrape per run BACKGROUND_SCRAPER_MAX_SERIES_PER_RUN=100 # Maximum number of series to scrape per run +CATALOG_TIMEOUT=30 # Max time to fetch catalog pages (seconds) + +# ============================== # +# Anime Mapping Configuration # +# ============================== # ANIME_MAPPING_SOURCE=remote # Options: remote, database - remote downloads on startup; database reads cached table ANIME_MAPPING_REFRESH_INTERVAL=86400 # Seconds between background anime mapping refreshes when using database cache (<=0 disables) @@ -69,20 +74,30 @@ ANIME_MAPPING_REFRESH_INTERVAL=86400 # Seconds between background anime mapping BYPASS_PROXY_URL=http://warp:1080 # To bypass scraper IP blacklists # ============================== # -# Indexer Manager Settings # +# Jackett & Prowlarr Settings # # ============================== # -INDEXER_MANAGER_TYPE=none # Options: jackett, prowlarr, none -INDEXER_MANAGER_URL=http://127.0.0.1:9117 -INDEXER_MANAGER_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -INDEXER_MANAGER_MODE=both # Context mode: live, background, both, false - Controls when indexer is used -INDEXER_MANAGER_TIMEOUT=60 # Max time to get search results (seconds) -INDEXER_MANAGER_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' # Jackett/Prowlarr indexers - get names from https://github.com/Jackett/Jackett/tree/master/src/Jackett.Common/Definitions for Jackett + +# Jackett Configuration +SCRAPE_JACKETT=False # Context mode: live, background, both, false +JACKETT_URL=http://127.0.0.1:9117 +JACKETT_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +JACKETT_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' # Get names from https://github.com/Jackett/Jackett/tree/master/src/Jackett.Common/Definitions + +# Prowlarr Configuration +SCRAPE_PROWLARR=False # Context mode: live, background, both, false +PROWLARR_URL=http://127.0.0.1:9696 +PROWLARR_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +PROWLARR_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' + +# Shared Settings +INDEXER_MANAGER_TIMEOUT=60 # Max time to get search results (seconds) - Shared by both # ============================== # # Torrent Settings # # ============================== # GET_TORRENT_TIMEOUT=5 # Max time to obtain torrent info hash (seconds) DOWNLOAD_TORRENT_FILES=False # Enable torrent file retrieval (instead of magnet link only) +MAGNET_RESOLVE_TIMEOUT=60 # Max time to resolve a magnet link (seconds) # ============================== # # Scraping Configuration # @@ -94,7 +109,7 @@ DOWNLOAD_TORRENT_FILES=False # Enable torrent file retrieval (instead of magnet # - Example multi: COMET_URL='["https://comet1.example.com", "https://comet2.example.com"]' # # Scraper Context Modes: -# Each SCRAPE_* setting and INDEXER_MANAGER_MODE can control when scrapers are used: +# Each SCRAPE_* setting can control when scrapers are used: # - true/both: Used for live scraping AND background scraping (default) # - live: Only used for live scraping (when users request content) # - background: Only used for background scraping (automatic content pre-caching) @@ -105,7 +120,8 @@ DOWNLOAD_TORRENT_FILES=False # Enable torrent file retrieval (instead of magnet # SCRAPE_TORRENTIO=live # Fast live scraping only # SCRAPE_ZILEAN=background # Background cache building only # SCRAPE_NYAA=false # Completely disabled -# INDEXER_MANAGER_MODE=live # Jackett/Prowlarr for live scraping only +# SCRAPE_JACKETT=live # Jackett for live scraping only +# SCRAPE_PROWLARR=background # Prowlarr for background scraping only SCRAPE_COMET=False COMET_URL=https://comet.elfhosted.com diff --git a/comet/background_scraper/cinemata_client.py b/comet/background_scraper/cinemata_client.py index 316ffc9..a7d57b9 100644 --- a/comet/background_scraper/cinemata_client.py +++ b/comet/background_scraper/cinemata_client.py @@ -1,5 +1,6 @@ import aiohttp +from comet.core.constants import CATALOG_TIMEOUT from comet.core.logger import logger @@ -28,9 +29,7 @@ class CinemataClient: url = "/".join(url_parts) + f"/skip={skip}.json" try: - async with self.session.get( - url, timeout=aiohttp.ClientTimeout(total=30) - ) as response: + async with self.session.get(url, timeout=CATALOG_TIMEOUT) as response: response.raise_for_status() data = await response.json() return data diff --git a/comet/core/constants.py b/comet/core/constants.py new file mode 100644 index 0000000..f2b25f0 --- /dev/null +++ b/comet/core/constants.py @@ -0,0 +1,7 @@ +import aiohttp + +from comet.core.models import settings + +INDEXER_TIMEOUT = aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT) +TORRENT_TIMEOUT = aiohttp.ClientTimeout(total=settings.GET_TORRENT_TIMEOUT) +CATALOG_TIMEOUT = aiohttp.ClientTimeout(total=settings.CATALOG_TIMEOUT) diff --git a/comet/core/logger.py b/comet/core/logger.py index dff6a69..4b50d7f 100644 --- a/comet/core/logger.py +++ b/comet/core/logger.py @@ -184,18 +184,28 @@ def log_startup_info(settings): ) logger.log("COMET", f"Bypass Proxy: {settings.BYPASS_PROXY_URL}") - if settings.is_any_context_enabled(settings.INDEXER_MANAGER_MODE): - logger.log( - "COMET", - f"Indexer Manager: {settings.INDEXER_MANAGER_TYPE}|{settings.INDEXER_MANAGER_URL} - Mode: {settings.INDEXER_MANAGER_MODE} - Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s", - ) - logger.log("COMET", f"Indexers: {', '.join(settings.INDEXER_MANAGER_INDEXERS)}") - logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s") - logger.log( - "COMET", f"Download Torrent Files: {bool(settings.DOWNLOAD_TORRENT_FILES)}" - ) - else: - logger.log("COMET", "Indexer Manager: False") + jackett_info = "" + if settings.is_any_context_enabled(settings.SCRAPE_JACKETT): + jackett_info = f" - {settings.JACKETT_URL} - Indexers: {', '.join(settings.JACKETT_INDEXERS)}" + logger.log( + "COMET", + f"Jackett Scraper: {settings.format_scraper_mode(settings.SCRAPE_JACKETT)}{jackett_info}", + ) + + prowlarr_info = "" + if settings.is_any_context_enabled(settings.SCRAPE_PROWLARR): + prowlarr_info = f" - {settings.PROWLARR_URL} - Indexers: {', '.join(settings.PROWLARR_INDEXERS)}" + logger.log( + "COMET", + f"Prowlarr Scraper: {settings.format_scraper_mode(settings.SCRAPE_PROWLARR)}{prowlarr_info}", + ) + + logger.log("COMET", f"Indexer Manager Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s") + logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s") + logger.log("COMET", f"Magnet Resolve Timeout: {settings.MAGNET_RESOLVE_TIMEOUT}s") + logger.log( + "COMET", f"Download Torrent Files: {bool(settings.DOWNLOAD_TORRENT_FILES)}" + ) comet_url = ( f" - {settings.COMET_URL}" diff --git a/comet/core/models.py b/comet/core/models.py index e766c71..972c878 100644 --- a/comet/core/models.py +++ b/comet/core/models.py @@ -54,7 +54,17 @@ class AppSettings(BaseSettings): INDEXER_MANAGER_MODE: Union[bool, str] = "both" INDEXER_MANAGER_TIMEOUT: Optional[int] = 30 INDEXER_MANAGER_INDEXERS: List[str] = [] + SCRAPE_JACKETT: Union[bool, str] = False + JACKETT_URL: Optional[str] = "http://127.0.0.1:9117" + JACKETT_API_KEY: Optional[str] = None + JACKETT_INDEXERS: List[str] = [] + SCRAPE_PROWLARR: Union[bool, str] = False + PROWLARR_URL: Optional[str] = "http://127.0.0.1:9696" + PROWLARR_API_KEY: Optional[str] = None + PROWLARR_INDEXERS: List[str] = [] GET_TORRENT_TIMEOUT: Optional[int] = 5 + MAGNET_RESOLVE_TIMEOUT: Optional[int] = 60 + CATALOG_TIMEOUT: Optional[int] = 30 DOWNLOAD_TORRENT_FILES: Optional[bool] = False SCRAPE_COMET: Union[bool, str] = False COMET_URL: Union[str, List[str]] = "https://comet.elfhosted.com" @@ -143,7 +153,9 @@ class AppSettings(BaseSettings): return "sqlite" return value - @field_validator("INDEXER_MANAGER_INDEXERS") + @field_validator( + "INDEXER_MANAGER_INDEXERS", "JACKETT_INDEXERS", "PROWLARR_INDEXERS" + ) def indexer_manager_indexers_normalization(cls, v, values): v = [indexer.replace(" ", "").lower() for indexer in v] return v @@ -159,6 +171,8 @@ class AppSettings(BaseSettings): "MEDIAFUSION_URL", "AIOSTREAMS_URL", "JACKETTIO_URL", + "JACKETT_URL", + "PROWLARR_URL", ) def normalize_urls(cls, v): if isinstance(v, str): @@ -201,6 +215,29 @@ class AppSettings(BaseSettings): scraper_setting = scraper_setting.lower() return scraper_setting in ["true", "both", "live", "background"] + def model_post_init(self, __context): + if self.INDEXER_MANAGER_TYPE == "jackett": + if not self.SCRAPE_JACKETT: + self.SCRAPE_JACKETT = self.INDEXER_MANAGER_MODE + if self.JACKETT_URL == "http://127.0.0.1:9117" and self.INDEXER_MANAGER_URL: + self.JACKETT_URL = self.INDEXER_MANAGER_URL + if not self.JACKETT_API_KEY and self.INDEXER_MANAGER_API_KEY: + self.JACKETT_API_KEY = self.INDEXER_MANAGER_API_KEY + if not self.JACKETT_INDEXERS and self.INDEXER_MANAGER_INDEXERS: + self.JACKETT_INDEXERS = self.INDEXER_MANAGER_INDEXERS + elif self.INDEXER_MANAGER_TYPE == "prowlarr": + if not self.SCRAPE_PROWLARR: + self.SCRAPE_PROWLARR = self.INDEXER_MANAGER_MODE + if ( + self.PROWLARR_URL == "http://127.0.0.1:9696" + and self.INDEXER_MANAGER_URL + ): + self.PROWLARR_URL = self.INDEXER_MANAGER_URL + if not self.PROWLARR_API_KEY and self.INDEXER_MANAGER_API_KEY: + self.PROWLARR_API_KEY = self.INDEXER_MANAGER_API_KEY + if not self.PROWLARR_INDEXERS and self.INDEXER_MANAGER_INDEXERS: + self.PROWLARR_INDEXERS = self.INDEXER_MANAGER_INDEXERS + settings = AppSettings() diff --git a/comet/scrapers/jackett.py b/comet/scrapers/jackett.py index 1c8d1cb..e95f370 100644 --- a/comet/scrapers/jackett.py +++ b/comet/scrapers/jackett.py @@ -3,6 +3,7 @@ from typing import List, Set import aiohttp +from comet.core.constants import INDEXER_TIMEOUT from comet.core.logger import logger from comet.core.models import settings from comet.scrapers.base import BaseScraper @@ -14,9 +15,8 @@ from comet.services.torrent_manager import (add_torrent_queue, class JackettScraper(BaseScraper): - def __init__(self, manager, session: aiohttp.ClientSession): - super().__init__(manager, session) - self.url = settings.INDEXER_MANAGER_URL + def __init__(self, manager, session: aiohttp.ClientSession, url: str): + super().__init__(manager, session, url) async def process_torrent(self, result: dict, media_id: str, season: int): base_torrent = { @@ -88,8 +88,8 @@ class JackettScraper(BaseScraper): async def fetch_jackett_results(self, indexer: str, query: str): try: response = await self.session.get( - f"{self.url}/api/v2.0/indexers/all/results?apikey={settings.INDEXER_MANAGER_API_KEY}&Query={query}&Tracker[]={indexer}", - timeout=aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT), + f"{self.url}/api/v2.0/indexers/all/results?apikey={settings.JACKETT_API_KEY}&Query={query}&Tracker[]={indexer}", + timeout=INDEXER_TIMEOUT, ) response = await response.json() return response.get("Results", []) @@ -116,7 +116,7 @@ class JackettScraper(BaseScraper): tasks.extend( [ self.fetch_jackett_results(indexer, query) - for indexer in settings.INDEXER_MANAGER_INDEXERS + for indexer in settings.JACKETT_INDEXERS ] ) diff --git a/comet/scrapers/manager.py b/comet/scrapers/manager.py index c5f3f90..b4a968c 100644 --- a/comet/scrapers/manager.py +++ b/comet/scrapers/manager.py @@ -60,44 +60,25 @@ class ScraperManager: setting_name = scraper_name_clean.upper() setting_key = f"SCRAPE_{setting_name}" - if scraper_name == "JackettScraper": - if not ( - settings.INDEXER_MANAGER_API_KEY - and settings.is_scraper_enabled( - settings.INDEXER_MANAGER_MODE, request.context - ) - and settings.INDEXER_MANAGER_TYPE == "jackett" - ): - continue - elif scraper_name == "ProwlarrScraper": - if not ( - settings.INDEXER_MANAGER_API_KEY - and settings.is_scraper_enabled( - settings.INDEXER_MANAGER_MODE, request.context - ) - and settings.INDEXER_MANAGER_TYPE == "prowlarr" + if hasattr(settings, setting_key): + if not settings.is_scraper_enabled( + getattr(settings, setting_key), request.context ): continue else: - if hasattr(settings, setting_key): - if not settings.is_scraper_enabled( - getattr(settings, setting_key), request.context - ): - continue - else: - logger.debug( - f"No {setting_key} found for {scraper_name_clean}, disabling" - ) - continue + logger.debug( + f"No {setting_key} found for {scraper_name_clean}, disabling" + ) + continue - if ( - scraper_name == "NyaaScraper" - and settings.NYAA_ANIME_ONLY - and not anime_mapper.is_anime_content( - request.media_id, request.media_only_id - ) - ): - continue + if ( + scraper_name == "NyaaScraper" + and settings.NYAA_ANIME_ONLY + and not anime_mapper.is_anime_content( + request.media_id, request.media_only_id + ) + ): + continue if scraper_name == "MediaFusionScraper": url_credentials_pairs = associate_urls_credentials( diff --git a/comet/scrapers/prowlarr.py b/comet/scrapers/prowlarr.py index 29683a2..fc5dfb3 100644 --- a/comet/scrapers/prowlarr.py +++ b/comet/scrapers/prowlarr.py @@ -3,6 +3,7 @@ from typing import List, Set import aiohttp +from comet.core.constants import INDEXER_TIMEOUT from comet.core.logger import logger from comet.core.models import settings from comet.scrapers.base import BaseScraper @@ -14,8 +15,8 @@ from comet.services.torrent_manager import (add_torrent_queue, class ProwlarrScraper(BaseScraper): - def __init__(self, manager, session: aiohttp.ClientSession): - super().__init__(manager, session) + def __init__(self, manager, session: aiohttp.ClientSession, url: str): + super().__init__(manager, session, url) async def process_torrent(self, result: dict, media_id: str, season: int): base_torrent = { @@ -94,13 +95,12 @@ class ProwlarrScraper(BaseScraper): ) try: - indexers = [ - indexer.lower() for indexer in settings.INDEXER_MANAGER_INDEXERS - ] + indexers = [indexer.lower() for indexer in settings.PROWLARR_INDEXERS] get_indexers = await self.session.get( - f"{settings.INDEXER_MANAGER_URL}/api/v1/indexer", - headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY}, + f"{self.url}/api/v1/indexer", + headers={"X-Api-Key": settings.PROWLARR_API_KEY}, + timeout=INDEXER_TIMEOUT, ) get_indexers = await get_indexers.json() @@ -116,8 +116,9 @@ class ProwlarrScraper(BaseScraper): for query in queries: tasks.append( self.session.get( - f"{settings.INDEXER_MANAGER_URL}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search", - headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY}, + f"{self.url}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search", + headers={"X-Api-Key": settings.PROWLARR_API_KEY}, + timeout=INDEXER_TIMEOUT, ) ) diff --git a/comet/services/torrent_manager.py b/comet/services/torrent_manager.py index ed06c42..b6cfd68 100644 --- a/comet/services/torrent_manager.py +++ b/comet/services/torrent_manager.py @@ -15,6 +15,7 @@ from demagnetize.core import Demagnetizer from RTN import ParsedData, parse from torf import Magnet +from comet.core.constants import TORRENT_TIMEOUT from comet.core.logger import logger from comet.core.models import database, settings from comet.utils.parsing import default_dump, is_video @@ -35,8 +36,9 @@ def extract_trackers_from_magnet(magnet_uri: str): async def download_torrent(session: aiohttp.ClientSession, url: str): try: - timeout = aiohttp.ClientTimeout(total=settings.GET_TORRENT_TIMEOUT) - async with session.get(url, allow_redirects=False, timeout=timeout) as response: + async with session.get( + url, allow_redirects=False, timeout=TORRENT_TIMEOUT + ) as response: if response.status == 200: return (await response.read(), None, None) @@ -64,7 +66,7 @@ demagnetizer = Demagnetizer() async def get_torrent_from_magnet(magnet_uri: str): try: magnet = Magnet.from_string(magnet_uri) - with anyio.fail_after(60): + with anyio.fail_after(settings.MAGNET_RESOLVE_TIMEOUT): torrent_data = await demagnetizer.demagnetize(magnet) if torrent_data: return torrent_data.dump()