g0ldyy/comet
1
0
Fork 0
mirror of https://github.com/g0ldyy/comet.git synced 2026-01-12 01:16:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: decouple Jackett and Prowlarr scraper configurations from a generic indexer manager

Browse Source
This commit is contained in:
itzme
2025-12-10 11:15:30 +01:00
parent a964a68a8e
commit 460fad22b4
6 changed files with 87 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env-sample
+16 -7
View File
@@ -64,14 +64,23 @@ BACKGROUND_SCRAPER_MAX_SERIES_PER_RUN=100 # Maximum number of series to scrape p
BYPASS_PROXY_URL=http://warp:1080 # To bypass scraper IP blacklists
# ============================== #
# Indexer Manager Settings #
# Jackett & Prowlarr Settings #
# ============================== #
INDEXER_MANAGER_TYPE=none # Options: jackett, prowlarr, none
INDEXER_MANAGER_URL=http://127.0.0.1:9117
INDEXER_MANAGER_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
INDEXER_MANAGER_MODE=both # Context mode: live, background, both, false - Controls when indexer is used
INDEXER_MANAGER_TIMEOUT=60 # Max time to get search results (seconds)
INDEXER_MANAGER_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' # Jackett/Prowlarr indexers - get names from https://github.com/Jackett/Jackett/tree/master/src/Jackett.Common/Definitions for Jackett
# Jackett Configuration
SCRAPE_JACKETT=False # Context mode: live, background, both, false
JACKETT_URL=http://127.0.0.1:9117
JACKETT_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
JACKETT_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' # Get names from https://github.com/Jackett/Jackett/tree/master/src/Jackett.Common/Definitions
# Prowlarr Configuration
SCRAPE_PROWLARR=False # Context mode: live, background, both, false
PROWLARR_URL=http://127.0.0.1:9696
PROWLARR_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
PROWLARR_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]'
# Shared Settings
INDEXER_MANAGER_TIMEOUT=60 # Max time to get search results (seconds) - Shared by both
# ============================== #
# Torrent Settings #
comet/core/logger.py
+12 -7
View File
@@ -184,18 +184,23 @@ def log_startup_info(settings):
)
logger.log("COMET", f"Bypass Proxy: {settings.BYPASS_PROXY_URL}")
if settings.is_any_context_enabled(settings.INDEXER_MANAGER_MODE):
if settings.is_any_context_enabled(settings.SCRAPE_JACKETT):
logger.log(
"COMET",
f"Indexer Manager: {settings.INDEXER_MANAGER_TYPE}|{settings.INDEXER_MANAGER_URL} - Mode: {settings.INDEXER_MANAGER_MODE} - Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s",
f"Jackett Scraper: {settings.format_scraper_mode(settings.SCRAPE_JACKETT)} - {settings.JACKETT_URL} - Indexers: {', '.join(settings.JACKETT_INDEXERS)}",
)
logger.log("COMET", f"Indexers: {', '.join(settings.INDEXER_MANAGER_INDEXERS)}")
logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s")
if settings.is_any_context_enabled(settings.SCRAPE_PROWLARR):
logger.log(
"COMET", f"Download Torrent Files: {bool(settings.DOWNLOAD_TORRENT_FILES)}"
"COMET",
f"Prowlarr Scraper: {settings.format_scraper_mode(settings.SCRAPE_PROWLARR)} - {settings.PROWLARR_URL} - Indexers: {', '.join(settings.PROWLARR_INDEXERS)}",
)
else:
logger.log("COMET", "Indexer Manager: False")
logger.log("COMET", f"Indexer Manager Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s")
logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s")
logger.log(
"COMET", f"Download Torrent Files: {bool(settings.DOWNLOAD_TORRENT_FILES)}"
)
comet_url = (
f" - {settings.COMET_URL}"
comet/core/models.py
+36 -1
View File
@@ -48,6 +48,14 @@ class AppSettings(BaseSettings):
INDEXER_MANAGER_MODE: Union[bool, str] = "both"
INDEXER_MANAGER_TIMEOUT: Optional[int] = 30
INDEXER_MANAGER_INDEXERS: List[str] = []
SCRAPE_JACKETT: Union[bool, str] = False
JACKETT_URL: Optional[str] = "http://127.0.0.1:9117"
JACKETT_API_KEY: Optional[str] = None
JACKETT_INDEXERS: List[str] = []
SCRAPE_PROWLARR: Union[bool, str] = False
PROWLARR_URL: Optional[str] = "http://127.0.0.1:9696"
PROWLARR_API_KEY: Optional[str] = None
PROWLARR_INDEXERS: List[str] = []
GET_TORRENT_TIMEOUT: Optional[int] = 5
DOWNLOAD_TORRENT_FILES: Optional[bool] = False
SCRAPE_COMET: Union[bool, str] = False
@@ -108,7 +116,9 @@ class AppSettings(BaseSettings):
return None
return v
@field_validator("INDEXER_MANAGER_INDEXERS")
@field_validator(
"INDEXER_MANAGER_INDEXERS", "JACKETT_INDEXERS", "PROWLARR_INDEXERS"
)
def indexer_manager_indexers_normalization(cls, v, values):
v = [indexer.replace(" ", "").lower() for indexer in v]
return v
@@ -124,6 +134,8 @@ class AppSettings(BaseSettings):
"MEDIAFUSION_URL",
"AIOSTREAMS_URL",
"JACKETTIO_URL",
"JACKETT_URL",
"PROWLARR_URL",
)
def normalize_urls(cls, v):
if isinstance(v, str):
@@ -166,6 +178,29 @@ class AppSettings(BaseSettings):
scraper_setting = scraper_setting.lower()
return scraper_setting in ["true", "both", "live", "background"]
def model_post_init(self, __context):
if self.INDEXER_MANAGER_TYPE == "jackett":
if not self.SCRAPE_JACKETT:
self.SCRAPE_JACKETT = self.INDEXER_MANAGER_MODE
if self.JACKETT_URL == "http://127.0.0.1:9117" and self.INDEXER_MANAGER_URL:
self.JACKETT_URL = self.INDEXER_MANAGER_URL
if not self.JACKETT_API_KEY and self.INDEXER_MANAGER_API_KEY:
self.JACKETT_API_KEY = self.INDEXER_MANAGER_API_KEY
if not self.JACKETT_INDEXERS and self.INDEXER_MANAGER_INDEXERS:
self.JACKETT_INDEXERS = self.INDEXER_MANAGER_INDEXERS
elif self.INDEXER_MANAGER_TYPE == "prowlarr":
if not self.SCRAPE_PROWLARR:
self.SCRAPE_PROWLARR = self.INDEXER_MANAGER_MODE
if (
self.PROWLARR_URL == "http://127.0.0.1:9696"
and self.INDEXER_MANAGER_URL
):
self.PROWLARR_URL = self.INDEXER_MANAGER_URL
if not self.PROWLARR_API_KEY and self.INDEXER_MANAGER_API_KEY:
self.PROWLARR_API_KEY = self.INDEXER_MANAGER_API_KEY
if not self.PROWLARR_INDEXERS and self.INDEXER_MANAGER_INDEXERS:
self.PROWLARR_INDEXERS = self.INDEXER_MANAGER_INDEXERS
settings = AppSettings()
comet/scrapers/jackett.py
+3 -3
View File
@@ -16,7 +16,7 @@ from comet.services.torrent_manager import (add_torrent_queue,
class JackettScraper(BaseScraper):
def __init__(self, manager, session: aiohttp.ClientSession):
super().__init__(manager, session)
self.url = settings.INDEXER_MANAGER_URL
self.url = settings.JACKETT_URL
async def process_torrent(self, result: dict, media_id: str, season: int):
base_torrent = {
@@ -88,7 +88,7 @@ class JackettScraper(BaseScraper):
async def fetch_jackett_results(self, indexer: str, query: str):
try:
response = await self.session.get(
f"{self.url}/api/v2.0/indexers/all/results?apikey={settings.INDEXER_MANAGER_API_KEY}&Query={query}&Tracker[]={indexer}",
f"{self.url}/api/v2.0/indexers/all/results?apikey={settings.JACKETT_API_KEY}&Query={query}&Tracker[]={indexer}",
timeout=aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT),
)
response = await response.json()
@@ -116,7 +116,7 @@ class JackettScraper(BaseScraper):
tasks.extend(
[
self.fetch_jackett_results(indexer, query)
for indexer in settings.INDEXER_MANAGER_INDEXERS
for indexer in settings.JACKETT_INDEXERS
]
)
comet/scrapers/manager.py
+15 -34
View File
@@ -60,44 +60,25 @@ class ScraperManager:
setting_name = scraper_name_clean.upper()
setting_key = f"SCRAPE_{setting_name}"
if scraper_name == "JackettScraper":
if not (
settings.INDEXER_MANAGER_API_KEY
and settings.is_scraper_enabled(
settings.INDEXER_MANAGER_MODE, request.context
)
and settings.INDEXER_MANAGER_TYPE == "jackett"
):
continue
elif scraper_name == "ProwlarrScraper":
if not (
settings.INDEXER_MANAGER_API_KEY
and settings.is_scraper_enabled(
settings.INDEXER_MANAGER_MODE, request.context
)
and settings.INDEXER_MANAGER_TYPE == "prowlarr"
if hasattr(settings, setting_key):
if not settings.is_scraper_enabled(
getattr(settings, setting_key), request.context
):
continue
else:
if hasattr(settings, setting_key):
if not settings.is_scraper_enabled(
getattr(settings, setting_key), request.context
):
continue
else:
logger.debug(
f"No {setting_key} found for {scraper_name_clean}, disabling"
)
continue
logger.debug(
f"No {setting_key} found for {scraper_name_clean}, disabling"
)
continue
if (
scraper_name == "NyaaScraper"
and settings.NYAA_ANIME_ONLY
and not anime_mapper.is_anime_content(
request.media_id, request.media_only_id
)
):
continue
if (
scraper_name == "NyaaScraper"
and settings.NYAA_ANIME_ONLY
and not anime_mapper.is_anime_content(
request.media_id, request.media_only_id
)
):
continue
if scraper_name == "MediaFusionScraper":
url_credentials_pairs = associate_urls_credentials(
comet/scrapers/prowlarr.py
+5 -7
View File
@@ -94,13 +94,11 @@ class ProwlarrScraper(BaseScraper):
)
try:
indexers = [
indexer.lower() for indexer in settings.INDEXER_MANAGER_INDEXERS
]
indexers = [indexer.lower() for indexer in settings.PROWLARR_INDEXERS]
get_indexers = await self.session.get(
f"{settings.INDEXER_MANAGER_URL}/api/v1/indexer",
headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY},
f"{settings.PROWLARR_URL}/api/v1/indexer",
headers={"X-Api-Key": settings.PROWLARR_API_KEY},
)
get_indexers = await get_indexers.json()
@@ -116,8 +114,8 @@ class ProwlarrScraper(BaseScraper):
for query in queries:
tasks.append(
self.session.get(
f"{settings.INDEXER_MANAGER_URL}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search",
headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY},
f"{settings.PROWLARR_URL}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search",
headers={"X-Api-Key": settings.PROWLARR_API_KEY},
)
)