From 1b87b1e5017d06f59c63990d0ccb1c714a8494b5 Mon Sep 17 00:00:00 2001 From: g0ldyy <153996346+g0ldyy@users.noreply.github.com> Date: Tue, 6 Jan 2026 20:31:17 +0100 Subject: [PATCH 1/2] feat: add AnimeTosho scraper --- .env-sample | 8 ++- comet/api/endpoints/stream.py | 5 +- comet/core/logger.py | 30 ++++++++-- comet/core/models.py | 3 + comet/scrapers/animetosho.py | 110 ++++++++++++++++++++++++++++++++++ comet/scrapers/bitmagnet.py | 9 ++- comet/scrapers/manager.py | 9 +++ 7 files changed, 158 insertions(+), 16 deletions(-) create mode 100644 comet/scrapers/animetosho.py diff --git a/.env-sample b/.env-sample index 8b91abb..add83db 100644 --- a/.env-sample +++ b/.env-sample @@ -72,7 +72,7 @@ TORRENT_CACHE_TTL=2592000 # 30 days LIVE_TORRENT_CACHE_TTL=604800 # 7 days DEBRID_CACHE_TTL=86400 # 1 day -DEBRID_CACHE_CHECK_RATIO=0.05 # Minimum ratio (0.05 = 5%) of cached torrents/total torrents required to skip re-checking availability on the debrid service. +DEBRID_CACHE_CHECK_RATIO=0.0 # Minimum ratio (0.5 = 5%) of cached torrents/total torrents required to skip re-checking availability on the debrid service. METRICS_CACHE_TTL=60 # 1 minute SCRAPE_LOCK_TTL=300 # 5 minutes - Duration for distributed scraping locks SCRAPE_WAIT_TIMEOUT=30 # 30 seconds - Max time to wait for other instance to complete scraping @@ -167,6 +167,10 @@ SCRAPE_NYAA=False NYAA_ANIME_ONLY=True # Only scrape Nyaa if the content is anime NYAA_MAX_CONCURRENT_PAGES=5 # Maximum number of concurrent requests to Nyaa (consider reducing if you are often ratelimited by Nyaa) +SCRAPE_ANIMETOSHO=False +ANIMETOSHO_ANIME_ONLY=True # Only scrape AnimeTosho if the content is anime +ANIMETOSHO_MAX_CONCURRENT_PAGES=10 # Maximum number of concurrent requests to AnimeTosho (consider reducing if you are often ratelimited by AnimeTosho) + SCRAPE_ZILEAN=False ZILEAN_URL=https://zileanfortheweebs.midnightignite.me # Multi-instance example: @@ -237,7 +241,7 @@ PROXY_DEBRID_STREAM_INACTIVITY_THRESHOLD=300 # Seconds to wait before cleaning u # ============================== # DISABLE_TORRENT_STREAMS=False # When true, torrent-only requests return a friendly message instead of magnets TORRENT_DISABLED_STREAM_NAME=[INFO] Comet # Stremio stream name shown when torrents are disabled -TORRENT_DISABLED_STREAM_DESCRIPTION=Direct torrent playback is disabled on this server. Please configure a debrid provider. # Description shown to users in Stremio +TORRENT_DISABLED_STREAM_DESCRIPTION=Direct torrent playback is disabled on this server. # Description shown to users in Stremio TORRENT_DISABLED_STREAM_URL=https://comet.looks.legal # Optional URL included in the placeholder stream response # ============================== # diff --git a/comet/api/endpoints/stream.py b/comet/api/endpoints/stream.py index 1a23dce..8a36dfe 100644 --- a/comet/api/endpoints/stream.py +++ b/comet/api/endpoints/stream.py @@ -159,9 +159,8 @@ async def stream( if settings.DISABLE_TORRENT_STREAMS and config["debridService"] == "torrent": placeholder_stream = { - "name": settings.TORRENT_DISABLED_STREAM_NAME or "[INFO] Comet", - "description": settings.TORRENT_DISABLED_STREAM_DESCRIPTION - or "Direct torrent playback is disabled on this server.", + "name": settings.TORRENT_DISABLED_STREAM_NAME, + "description": settings.TORRENT_DISABLED_STREAM_DESCRIPTION, } if settings.TORRENT_DISABLED_STREAM_URL: placeholder_stream["url"] = settings.TORRENT_DISABLED_STREAM_URL diff --git a/comet/core/logger.py b/comet/core/logger.py index 6a47c00..2f83db2 100644 --- a/comet/core/logger.py +++ b/comet/core/logger.py @@ -195,7 +195,7 @@ def log_startup_info(settings): replicas = f" - Read Replicas: {settings.DATABASE_READ_REPLICA_URLS}" logger.log( "COMET", - f"Database ({settings.DATABASE_TYPE}): {settings.DATABASE_PATH if settings.DATABASE_TYPE == 'sqlite' else settings.DATABASE_URL} - TTL: metadata={settings.METADATA_CACHE_TTL}s, torrents={settings.TORRENT_CACHE_TTL}s, live_torrents={settings.LIVE_TORRENT_CACHE_TTL}s, debrid={settings.DEBRID_CACHE_TTL}s, metrics={settings.METRICS_CACHE_TTL}s - Debrid Ratio: {settings.DEBRID_CACHE_CHECK_RATIO} - Startup Cleanup Interval: {settings.DATABASE_STARTUP_CLEANUP_INTERVAL}s{replicas}", + f"Database ({settings.DATABASE_TYPE}): {settings.DATABASE_PATH if settings.DATABASE_TYPE == 'sqlite' else settings.DATABASE_URL} - Batch Size: {settings.DATABASE_BATCH_SIZE} - TTL: metadata={settings.METADATA_CACHE_TTL}s, torrents={settings.TORRENT_CACHE_TTL}s, live_torrents={settings.LIVE_TORRENT_CACHE_TTL}s, debrid={settings.DEBRID_CACHE_TTL}s, metrics={settings.METRICS_CACHE_TTL}s - Debrid Ratio: {settings.DEBRID_CACHE_CHECK_RATIO} - Startup Cleanup Interval: {settings.DATABASE_STARTUP_CLEANUP_INTERVAL}s{replicas}", ) # SQLite concurrency warnings @@ -266,6 +266,9 @@ def log_startup_info(settings): ) logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s") logger.log("COMET", f"Magnet Resolve Timeout: {settings.MAGNET_RESOLVE_TIMEOUT}s") + logger.log("COMET", f"Catalog Timeout: {settings.CATALOG_TIMEOUT}s") + logger.log("COMET", f"Scrape Lock TTL: {settings.SCRAPE_LOCK_TTL}s") + logger.log("COMET", f"Scrape Wait Timeout: {settings.SCRAPE_WAIT_TIMEOUT}s") logger.log( "COMET", f"Download Torrent Files: {bool(settings.DOWNLOAD_TORRENT_FILES)}" ) @@ -281,7 +284,7 @@ def log_startup_info(settings): ) nyaa_anime_only = ( - f" - Anime Only: {bool(settings.NYAA_ANIME_ONLY)}" + f" - Anime Only: {bool(settings.NYAA_ANIME_ONLY)} - Concurrent Pages: {settings.NYAA_MAX_CONCURRENT_PAGES}" if settings.is_any_context_enabled(settings.SCRAPE_NYAA) else "" ) @@ -290,6 +293,16 @@ def log_startup_info(settings): f"Nyaa Scraper: {settings.format_scraper_mode(settings.SCRAPE_NYAA)}{nyaa_anime_only}", ) + animetosho_anime_only = ( + f" - Anime Only: {bool(settings.ANIMETOSHO_ANIME_ONLY)} - Concurrent Pages: {settings.ANIMETOSHO_MAX_CONCURRENT_PAGES}" + if settings.is_any_context_enabled(settings.SCRAPE_ANIMETOSHO) + else "" + ) + logger.log( + "COMET", + f"AnimeTosho Scraper: {settings.format_scraper_mode(settings.SCRAPE_ANIMETOSHO)}{animetosho_anime_only}", + ) + zilean_url = ( f" - {settings.ZILEAN_URL}" if settings.is_any_context_enabled(settings.SCRAPE_ZILEAN) @@ -310,14 +323,14 @@ def log_startup_info(settings): f"StremThru Scraper: {settings.format_scraper_mode(settings.SCRAPE_STREMTHRU)}{stremthru_scrape_url}", ) - bitmagnet_url = ( - f" - {settings.BITMAGNET_URL}" + bitmagnet_info = ( + f" - {settings.BITMAGNET_URL} - Concurrent Pages: {settings.BITMAGNET_MAX_CONCURRENT_PAGES} - Max Offset: {settings.BITMAGNET_MAX_OFFSET}" if settings.is_any_context_enabled(settings.SCRAPE_BITMAGNET) else "" ) logger.log( "COMET", - f"Bitmagnet Scraper: {settings.format_scraper_mode(settings.SCRAPE_BITMAGNET)}{bitmagnet_url}", + f"Bitmagnet Scraper: {settings.format_scraper_mode(settings.SCRAPE_BITMAGNET)}{bitmagnet_info}", ) torrentio_url = ( @@ -397,9 +410,14 @@ def log_startup_info(settings): logger.log("COMET", f"StremThru URL: {settings.STREMTHRU_URL}") + disabled_streams_info = ( + f" - Name: {settings.TORRENT_DISABLED_STREAM_NAME} - URL: {settings.TORRENT_DISABLED_STREAM_URL} - Description: {settings.TORRENT_DISABLED_STREAM_DESCRIPTION}" + if settings.DISABLE_TORRENT_STREAMS + else "" + ) logger.log( "COMET", - f"Disable Torrent Streams: {bool(settings.DISABLE_TORRENT_STREAMS)}", + f"Disable Torrent Streams: {bool(settings.DISABLE_TORRENT_STREAMS)}{disabled_streams_info}", ) logger.log("COMET", f"Remove Adult Content: {bool(settings.REMOVE_ADULT_CONTENT)}") diff --git a/comet/core/models.py b/comet/core/models.py index a5f870f..41098e5 100644 --- a/comet/core/models.py +++ b/comet/core/models.py @@ -74,6 +74,9 @@ class AppSettings(BaseSettings): SCRAPE_NYAA: Union[bool, str] = False NYAA_ANIME_ONLY: Optional[bool] = True NYAA_MAX_CONCURRENT_PAGES: Optional[int] = 5 + SCRAPE_ANIMETOSHO: Union[bool, str] = False + ANIMETOSHO_ANIME_ONLY: Optional[bool] = True + ANIMETOSHO_MAX_CONCURRENT_PAGES: Optional[int] = 10 SCRAPE_ZILEAN: Union[bool, str] = False ZILEAN_URL: Union[str, List[str]] = "https://zileanfortheweebs.midnightignite.me" SCRAPE_STREMTHRU: Union[bool, str] = False diff --git a/comet/scrapers/animetosho.py b/comet/scrapers/animetosho.py new file mode 100644 index 0000000..65d1c8b --- /dev/null +++ b/comet/scrapers/animetosho.py @@ -0,0 +1,110 @@ +import asyncio +import xml.etree.ElementTree as ET + +from comet.core.logger import logger +from comet.core.models import settings +from comet.scrapers.base import BaseScraper +from comet.scrapers.models import ScrapeRequest +from comet.services.torrent_manager import extract_trackers_from_magnet + + +class AnimeToshoScraper(BaseScraper): + def __init__(self, manager, session): + super().__init__(manager, session) + + def parse_items(self, root): + torrents = [] + for item in root.findall(".//item"): + try: + title = item.find("title").text + + size = None + info_hash = None + seeders = None + + magnet = None + + for attr in item.findall( + ".//torznab:attr", + {"torznab": "http://torznab.com/schemas/2015/feed"}, + ): + attr_name = attr.get("name") + attr_value = attr.get("value") + + if attr_name == "size": + size = int(attr_value) + elif attr_name == "infohash": + info_hash = attr_value + elif attr_name == "seeders": + seeders = int(attr_value) + elif attr_name == "magneturl": + magnet = attr_value + + if info_hash is None: + continue + + torrents.append( + { + "title": title, + "infoHash": info_hash, + "fileIndex": None, + "seeders": seeders, + "size": size, + "tracker": "AnimeTosho", + "sources": extract_trackers_from_magnet(magnet), + } + ) + except Exception as e: + logger.warning(f"Error parsing torrent item from AnimeTosho: {e}") + continue + return torrents + + async def scrape_page(self, query, offset, limit): + try: + async with self.session.get( + f"https://feed.animetosho.org/api?t=search&q={query}&offset={offset}&limit={limit}" + ) as response: + content = await response.text() + if not content.strip(): + return [], 0 + + root = ET.fromstring(content) + + response_node = root.find( + ".//newznab:response", + {"newznab": "http://www.newznab.com/DTD/2010/feeds/attributes/"}, + ) + total = int(response_node.get("total", 0)) + items = self.parse_items(root) + return items, total + except Exception as e: + logger.warning(f"Error scraping AnimeTosho offset={offset}: {e}") + return [], 0 + + async def scrape(self, request: ScrapeRequest): + torrents = [] + query = request.title + limit = 150 + + initial_items, total = await self.scrape_page(query, 0, limit) + torrents.extend(initial_items) + + if total > limit: + batch_size = settings.ANIMETOSHO_MAX_CONCURRENT_PAGES + current_offset = limit + + while current_offset < total: + tasks = [] + for _ in range(batch_size): + if current_offset >= total: + break + + tasks.append(self.scrape_page(query, current_offset, limit)) + current_offset += limit + + if tasks: + results = await asyncio.gather(*tasks) + for batch_items, _ in results: + torrents.extend(batch_items) + + return torrents diff --git a/comet/scrapers/bitmagnet.py b/comet/scrapers/bitmagnet.py index d51e625..dffceed 100644 --- a/comet/scrapers/bitmagnet.py +++ b/comet/scrapers/bitmagnet.py @@ -11,7 +11,7 @@ class BitmagnetScraper(BaseScraper): def __init__(self, manager, session, url: str): super().__init__(manager, session, url) - def parse_bitmagnet_items(self, root): + def parse_items(self, root): torrents = [] for item in root.findall(".//item"): try: @@ -49,13 +49,12 @@ class BitmagnetScraper(BaseScraper): "sources": [], } ) - except Exception as e: logger.warning(f"Error parsing torrent item from BitMagnet: {e}") continue return torrents - async def scrape_bitmagnet_page( + async def scrape_page( self, imdb_id, scrape_type, offset, limit, season=None, episode=None ): try: @@ -76,7 +75,7 @@ class BitmagnetScraper(BaseScraper): if not data_text.strip(): return [] root = ET.fromstring(data_text) - return self.parse_bitmagnet_items(root) + return self.parse_items(root) except ET.ParseError: return [] except Exception as e: @@ -104,7 +103,7 @@ class BitmagnetScraper(BaseScraper): if current_offset >= settings.BITMAGNET_MAX_OFFSET: break tasks.append( - self.scrape_bitmagnet_page( + self.scrape_page( imdb_id, scrape_type, current_offset, limit, season, episode ) ) diff --git a/comet/scrapers/manager.py b/comet/scrapers/manager.py index d618b54..bed2b43 100644 --- a/comet/scrapers/manager.py +++ b/comet/scrapers/manager.py @@ -76,6 +76,15 @@ class ScraperManager: ): continue + if ( + scraper_name == "AnimeToshoScraper" + and settings.ANIMETOSHO_ANIME_ONLY + and not anime_mapper.is_anime_content( + request.media_id, request.media_only_id + ) + ): + continue + # Get client wrapper client = network_manager.get_client( scraper_name=scraper_name_clean, impersonate=scraper_class.impersonate From dd9a2336b72a7814b460c87f07d0be6487301c0a Mon Sep 17 00:00:00 2001 From: g0ldyy <153996346+g0ldyy@users.noreply.github.com> Date: Tue, 6 Jan 2026 20:36:17 +0100 Subject: [PATCH 2/2] chore: remove newline --- comet/scrapers/animetosho.py | 1 - 1 file changed, 1 deletion(-) diff --git a/comet/scrapers/animetosho.py b/comet/scrapers/animetosho.py index 65d1c8b..2bcd36a 100644 --- a/comet/scrapers/animetosho.py +++ b/comet/scrapers/animetosho.py @@ -21,7 +21,6 @@ class AnimeToshoScraper(BaseScraper): size = None info_hash = None seeders = None - magnet = None for attr in item.findall(