g0ldyy/comet
1
0
Fork 0
mirror of https://github.com/g0ldyy/comet.git synced 2026-01-12 01:16:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'development' into fix-pr-391-improved

Browse Source
This commit is contained in:
Goldy
2025-12-10 18:30:48 +01:00
committed by GitHub
parent 0ee147561d d17cb2d7b3
commit 0a53c3c42e
9 changed files with 130 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env-sample
+25 -9
View File
@@ -60,6 +60,11 @@ BACKGROUND_SCRAPER_CONCURRENT_WORKERS=1 # Number of concurrent workers for scrap
BACKGROUND_SCRAPER_INTERVAL=3600 # Interval between scraping cycles in seconds
BACKGROUND_SCRAPER_MAX_MOVIES_PER_RUN=100 # Maximum number of movies to scrape per run
BACKGROUND_SCRAPER_MAX_SERIES_PER_RUN=100 # Maximum number of series to scrape per run
CATALOG_TIMEOUT=30 # Max time to fetch catalog pages (seconds)
# ============================== #
# Anime Mapping Configuration #
# ============================== #
ANIME_MAPPING_SOURCE=remote # Options: remote, database - remote downloads on startup; database reads cached table
ANIME_MAPPING_REFRESH_INTERVAL=86400 # Seconds between background anime mapping refreshes when using database cache (<=0 disables)
@@ -69,20 +74,30 @@ ANIME_MAPPING_REFRESH_INTERVAL=86400 # Seconds between background anime mapping
BYPASS_PROXY_URL=http://warp:1080 # To bypass scraper IP blacklists
# ============================== #
# Indexer Manager Settings #
# Jackett & Prowlarr Settings #
# ============================== #
INDEXER_MANAGER_TYPE=none # Options: jackett, prowlarr, none
INDEXER_MANAGER_URL=http://127.0.0.1:9117
INDEXER_MANAGER_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
INDEXER_MANAGER_MODE=both # Context mode: live, background, both, false - Controls when indexer is used
INDEXER_MANAGER_TIMEOUT=60 # Max time to get search results (seconds)
INDEXER_MANAGER_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' # Jackett/Prowlarr indexers - get names from https://github.com/Jackett/Jackett/tree/master/src/Jackett.Common/Definitions for Jackett
# Jackett Configuration
SCRAPE_JACKETT=False # Context mode: live, background, both, false
JACKETT_URL=http://127.0.0.1:9117
JACKETT_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
JACKETT_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' # Get names from https://github.com/Jackett/Jackett/tree/master/src/Jackett.Common/Definitions
# Prowlarr Configuration
SCRAPE_PROWLARR=False # Context mode: live, background, both, false
PROWLARR_URL=http://127.0.0.1:9696
PROWLARR_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
PROWLARR_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]'
# Shared Settings
INDEXER_MANAGER_TIMEOUT=60 # Max time to get search results (seconds) - Shared by both
# ============================== #
# Torrent Settings #
# ============================== #
GET_TORRENT_TIMEOUT=5 # Max time to obtain torrent info hash (seconds)
DOWNLOAD_TORRENT_FILES=False # Enable torrent file retrieval (instead of magnet link only)
MAGNET_RESOLVE_TIMEOUT=60 # Max time to resolve a magnet link (seconds)
# ============================== #
# Scraping Configuration #
@@ -94,7 +109,7 @@ DOWNLOAD_TORRENT_FILES=False # Enable torrent file retrieval (instead of magnet
# - Example multi: COMET_URL='["https://comet1.example.com", "https://comet2.example.com"]'
#
# Scraper Context Modes:
# Each SCRAPE_* setting and INDEXER_MANAGER_MODE can control when scrapers are used:
# Each SCRAPE_* setting can control when scrapers are used:
# - true/both: Used for live scraping AND background scraping (default)
# - live: Only used for live scraping (when users request content)
# - background: Only used for background scraping (automatic content pre-caching)
@@ -105,7 +120,8 @@ DOWNLOAD_TORRENT_FILES=False # Enable torrent file retrieval (instead of magnet
# SCRAPE_TORRENTIO=live # Fast live scraping only
# SCRAPE_ZILEAN=background # Background cache building only
# SCRAPE_NYAA=false # Completely disabled
# INDEXER_MANAGER_MODE=live # Jackett/Prowlarr for live scraping only
# SCRAPE_JACKETT=live # Jackett for live scraping only
# SCRAPE_PROWLARR=background # Prowlarr for background scraping only
SCRAPE_COMET=False
COMET_URL=https://comet.elfhosted.com
comet/background_scraper/cinemata_client.py
+2 -3
View File
@@ -1,5 +1,6 @@
import aiohttp
from comet.core.constants import CATALOG_TIMEOUT
from comet.core.logger import logger
@@ -28,9 +29,7 @@ class CinemataClient:
url = "/".join(url_parts) + f"/skip={skip}.json"
try:
async with self.session.get(
url, timeout=aiohttp.ClientTimeout(total=30)
) as response:
async with self.session.get(url, timeout=CATALOG_TIMEOUT) as response:
response.raise_for_status()
data = await response.json()
return data
comet/core/constants.py
+7
View File
@@ -0,0 +1,7 @@
import aiohttp
from comet.core.models import settings
INDEXER_TIMEOUT = aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT)
TORRENT_TIMEOUT = aiohttp.ClientTimeout(total=settings.GET_TORRENT_TIMEOUT)
CATALOG_TIMEOUT = aiohttp.ClientTimeout(total=settings.CATALOG_TIMEOUT)
comet/core/logger.py
+22 -12
View File
@@ -184,18 +184,28 @@ def log_startup_info(settings):
)
logger.log("COMET", f"Bypass Proxy: {settings.BYPASS_PROXY_URL}")
if settings.is_any_context_enabled(settings.INDEXER_MANAGER_MODE):
logger.log(
"COMET",
f"Indexer Manager: {settings.INDEXER_MANAGER_TYPE}|{settings.INDEXER_MANAGER_URL} - Mode: {settings.INDEXER_MANAGER_MODE} - Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s",
)
logger.log("COMET", f"Indexers: {', '.join(settings.INDEXER_MANAGER_INDEXERS)}")
logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s")
logger.log(
"COMET", f"Download Torrent Files: {bool(settings.DOWNLOAD_TORRENT_FILES)}"
)
else:
logger.log("COMET", "Indexer Manager: False")
jackett_info = ""
if settings.is_any_context_enabled(settings.SCRAPE_JACKETT):
jackett_info = f" - {settings.JACKETT_URL} - Indexers: {', '.join(settings.JACKETT_INDEXERS)}"
logger.log(
"COMET",
f"Jackett Scraper: {settings.format_scraper_mode(settings.SCRAPE_JACKETT)}{jackett_info}",
)
prowlarr_info = ""
if settings.is_any_context_enabled(settings.SCRAPE_PROWLARR):
prowlarr_info = f" - {settings.PROWLARR_URL} - Indexers: {', '.join(settings.PROWLARR_INDEXERS)}"
logger.log(
"COMET",
f"Prowlarr Scraper: {settings.format_scraper_mode(settings.SCRAPE_PROWLARR)}{prowlarr_info}",
)
logger.log("COMET", f"Indexer Manager Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s")
logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s")
logger.log("COMET", f"Magnet Resolve Timeout: {settings.MAGNET_RESOLVE_TIMEOUT}s")
logger.log(
"COMET", f"Download Torrent Files: {bool(settings.DOWNLOAD_TORRENT_FILES)}"
)
comet_url = (
f" - {settings.COMET_URL}"
comet/core/models.py
+38 -1
View File
@@ -54,7 +54,17 @@ class AppSettings(BaseSettings):
INDEXER_MANAGER_MODE: Union[bool, str] = "both"
INDEXER_MANAGER_TIMEOUT: Optional[int] = 30
INDEXER_MANAGER_INDEXERS: List[str] = []
SCRAPE_JACKETT: Union[bool, str] = False
JACKETT_URL: Optional[str] = "http://127.0.0.1:9117"
JACKETT_API_KEY: Optional[str] = None
JACKETT_INDEXERS: List[str] = []
SCRAPE_PROWLARR: Union[bool, str] = False
PROWLARR_URL: Optional[str] = "http://127.0.0.1:9696"
PROWLARR_API_KEY: Optional[str] = None
PROWLARR_INDEXERS: List[str] = []
GET_TORRENT_TIMEOUT: Optional[int] = 5
MAGNET_RESOLVE_TIMEOUT: Optional[int] = 60
CATALOG_TIMEOUT: Optional[int] = 30
DOWNLOAD_TORRENT_FILES: Optional[bool] = False
SCRAPE_COMET: Union[bool, str] = False
COMET_URL: Union[str, List[str]] = "https://comet.elfhosted.com"
@@ -143,7 +153,9 @@ class AppSettings(BaseSettings):
return "sqlite"
return value
@field_validator("INDEXER_MANAGER_INDEXERS")
@field_validator(
"INDEXER_MANAGER_INDEXERS", "JACKETT_INDEXERS", "PROWLARR_INDEXERS"
)
def indexer_manager_indexers_normalization(cls, v, values):
v = [indexer.replace(" ", "").lower() for indexer in v]
return v
@@ -159,6 +171,8 @@ class AppSettings(BaseSettings):
"MEDIAFUSION_URL",
"AIOSTREAMS_URL",
"JACKETTIO_URL",
"JACKETT_URL",
"PROWLARR_URL",
)
def normalize_urls(cls, v):
if isinstance(v, str):
@@ -201,6 +215,29 @@ class AppSettings(BaseSettings):
scraper_setting = scraper_setting.lower()
return scraper_setting in ["true", "both", "live", "background"]
def model_post_init(self, __context):
if self.INDEXER_MANAGER_TYPE == "jackett":
if not self.SCRAPE_JACKETT:
self.SCRAPE_JACKETT = self.INDEXER_MANAGER_MODE
if self.JACKETT_URL == "http://127.0.0.1:9117" and self.INDEXER_MANAGER_URL:
self.JACKETT_URL = self.INDEXER_MANAGER_URL
if not self.JACKETT_API_KEY and self.INDEXER_MANAGER_API_KEY:
self.JACKETT_API_KEY = self.INDEXER_MANAGER_API_KEY
if not self.JACKETT_INDEXERS and self.INDEXER_MANAGER_INDEXERS:
self.JACKETT_INDEXERS = self.INDEXER_MANAGER_INDEXERS
elif self.INDEXER_MANAGER_TYPE == "prowlarr":
if not self.SCRAPE_PROWLARR:
self.SCRAPE_PROWLARR = self.INDEXER_MANAGER_MODE
if (
self.PROWLARR_URL == "http://127.0.0.1:9696"
and self.INDEXER_MANAGER_URL
):
self.PROWLARR_URL = self.INDEXER_MANAGER_URL
if not self.PROWLARR_API_KEY and self.INDEXER_MANAGER_API_KEY:
self.PROWLARR_API_KEY = self.INDEXER_MANAGER_API_KEY
if not self.PROWLARR_INDEXERS and self.INDEXER_MANAGER_INDEXERS:
self.PROWLARR_INDEXERS = self.INDEXER_MANAGER_INDEXERS
settings = AppSettings()
comet/scrapers/jackett.py
+6 -6
View File
@@ -3,6 +3,7 @@ from typing import List, Set
import aiohttp
from comet.core.constants import INDEXER_TIMEOUT
from comet.core.logger import logger
from comet.core.models import settings
from comet.scrapers.base import BaseScraper
@@ -14,9 +15,8 @@ from comet.services.torrent_manager import (add_torrent_queue,
class JackettScraper(BaseScraper):
def __init__(self, manager, session: aiohttp.ClientSession):
super().__init__(manager, session)
self.url = settings.INDEXER_MANAGER_URL
def __init__(self, manager, session: aiohttp.ClientSession, url: str):
super().__init__(manager, session, url)
async def process_torrent(self, result: dict, media_id: str, season: int):
base_torrent = {
@@ -88,8 +88,8 @@ class JackettScraper(BaseScraper):
async def fetch_jackett_results(self, indexer: str, query: str):
try:
response = await self.session.get(
f"{self.url}/api/v2.0/indexers/all/results?apikey={settings.INDEXER_MANAGER_API_KEY}&Query={query}&Tracker[]={indexer}",
timeout=aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT),
f"{self.url}/api/v2.0/indexers/all/results?apikey={settings.JACKETT_API_KEY}&Query={query}&Tracker[]={indexer}",
timeout=INDEXER_TIMEOUT,
)
response = await response.json()
return response.get("Results", [])
@@ -116,7 +116,7 @@ class JackettScraper(BaseScraper):
tasks.extend(
[
self.fetch_jackett_results(indexer, query)
for indexer in settings.INDEXER_MANAGER_INDEXERS
for indexer in settings.JACKETT_INDEXERS
]
)
comet/scrapers/manager.py
+15 -34
View File
@@ -60,44 +60,25 @@ class ScraperManager:
setting_name = scraper_name_clean.upper()
setting_key = f"SCRAPE_{setting_name}"
if scraper_name == "JackettScraper":
if not (
settings.INDEXER_MANAGER_API_KEY
and settings.is_scraper_enabled(
settings.INDEXER_MANAGER_MODE, request.context
)
and settings.INDEXER_MANAGER_TYPE == "jackett"
):
continue
elif scraper_name == "ProwlarrScraper":
if not (
settings.INDEXER_MANAGER_API_KEY
and settings.is_scraper_enabled(
settings.INDEXER_MANAGER_MODE, request.context
)
and settings.INDEXER_MANAGER_TYPE == "prowlarr"
if hasattr(settings, setting_key):
if not settings.is_scraper_enabled(
getattr(settings, setting_key), request.context
):
continue
else:
if hasattr(settings, setting_key):
if not settings.is_scraper_enabled(
getattr(settings, setting_key), request.context
):
continue
else:
logger.debug(
f"No {setting_key} found for {scraper_name_clean}, disabling"
)
continue
logger.debug(
f"No {setting_key} found for {scraper_name_clean}, disabling"
)
continue
if (
scraper_name == "NyaaScraper"
and settings.NYAA_ANIME_ONLY
and not anime_mapper.is_anime_content(
request.media_id, request.media_only_id
)
):
continue
if (
scraper_name == "NyaaScraper"
and settings.NYAA_ANIME_ONLY
and not anime_mapper.is_anime_content(
request.media_id, request.media_only_id
)
):
continue
if scraper_name == "MediaFusionScraper":
url_credentials_pairs = associate_urls_credentials(
comet/scrapers/prowlarr.py
+10 -9
View File
@@ -3,6 +3,7 @@ from typing import List, Set
import aiohttp
from comet.core.constants import INDEXER_TIMEOUT
from comet.core.logger import logger
from comet.core.models import settings
from comet.scrapers.base import BaseScraper
@@ -14,8 +15,8 @@ from comet.services.torrent_manager import (add_torrent_queue,
class ProwlarrScraper(BaseScraper):
def __init__(self, manager, session: aiohttp.ClientSession):
super().__init__(manager, session)
def __init__(self, manager, session: aiohttp.ClientSession, url: str):
super().__init__(manager, session, url)
async def process_torrent(self, result: dict, media_id: str, season: int):
base_torrent = {
@@ -94,13 +95,12 @@ class ProwlarrScraper(BaseScraper):
)
try:
indexers = [
indexer.lower() for indexer in settings.INDEXER_MANAGER_INDEXERS
]
indexers = [indexer.lower() for indexer in settings.PROWLARR_INDEXERS]
get_indexers = await self.session.get(
f"{settings.INDEXER_MANAGER_URL}/api/v1/indexer",
headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY},
f"{self.url}/api/v1/indexer",
headers={"X-Api-Key": settings.PROWLARR_API_KEY},
timeout=INDEXER_TIMEOUT,
)
get_indexers = await get_indexers.json()
@@ -116,8 +116,9 @@ class ProwlarrScraper(BaseScraper):
for query in queries:
tasks.append(
self.session.get(
f"{settings.INDEXER_MANAGER_URL}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search",
headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY},
f"{self.url}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search",
headers={"X-Api-Key": settings.PROWLARR_API_KEY},
timeout=INDEXER_TIMEOUT,
)
)
comet/services/torrent_manager.py
+5 -3
View File
@@ -15,6 +15,7 @@ from demagnetize.core import Demagnetizer
from RTN import ParsedData, parse
from torf import Magnet
from comet.core.constants import TORRENT_TIMEOUT
from comet.core.logger import logger
from comet.core.models import database, settings
from comet.utils.parsing import default_dump, is_video
@@ -35,8 +36,9 @@ def extract_trackers_from_magnet(magnet_uri: str):
async def download_torrent(session: aiohttp.ClientSession, url: str):
try:
timeout = aiohttp.ClientTimeout(total=settings.GET_TORRENT_TIMEOUT)
async with session.get(url, allow_redirects=False, timeout=timeout) as response:
async with session.get(
url, allow_redirects=False, timeout=TORRENT_TIMEOUT
) as response:
if response.status == 200:
return (await response.read(), None, None)
@@ -64,7 +66,7 @@ demagnetizer = Demagnetizer()
async def get_torrent_from_magnet(magnet_uri: str):
try:
magnet = Magnet.from_string(magnet_uri)
with anyio.fail_after(60):
with anyio.fail_after(settings.MAGNET_RESOLVE_TIMEOUT):
torrent_data = await demagnetizer.demagnetize(magnet)
if torrent_data:
return torrent_data.dump()