g0ldyy/comet
1
0
Fork 0
mirror of https://github.com/g0ldyy/comet.git synced 2026-01-12 01:16:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: add AnimeTosho scraper

Browse Source
This commit is contained in:
g0ldyy
2026-01-06 20:31:17 +01:00
parent 9137096288
commit 1b87b1e501
7 changed files with 158 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env-sample
+6 -2
View File
@@ -72,7 +72,7 @@ TORRENT_CACHE_TTL=2592000 # 30 days
LIVE_TORRENT_CACHE_TTL=604800 # 7 days
DEBRID_CACHE_TTL=86400 # 1 day
DEBRID_CACHE_CHECK_RATIO=0.05 # Minimum ratio (0.05 = 5%) of cached torrents/total torrents required to skip re-checking availability on the debrid service.
DEBRID_CACHE_CHECK_RATIO=0.0 # Minimum ratio (0.5 = 5%) of cached torrents/total torrents required to skip re-checking availability on the debrid service.
METRICS_CACHE_TTL=60 # 1 minute
SCRAPE_LOCK_TTL=300 # 5 minutes - Duration for distributed scraping locks
SCRAPE_WAIT_TIMEOUT=30 # 30 seconds - Max time to wait for other instance to complete scraping
@@ -167,6 +167,10 @@ SCRAPE_NYAA=False
NYAA_ANIME_ONLY=True # Only scrape Nyaa if the content is anime
NYAA_MAX_CONCURRENT_PAGES=5 # Maximum number of concurrent requests to Nyaa (consider reducing if you are often ratelimited by Nyaa)
SCRAPE_ANIMETOSHO=False
ANIMETOSHO_ANIME_ONLY=True # Only scrape AnimeTosho if the content is anime
ANIMETOSHO_MAX_CONCURRENT_PAGES=10 # Maximum number of concurrent requests to AnimeTosho (consider reducing if you are often ratelimited by AnimeTosho)
SCRAPE_ZILEAN=False
ZILEAN_URL=https://zileanfortheweebs.midnightignite.me
# Multi-instance example:
@@ -237,7 +241,7 @@ PROXY_DEBRID_STREAM_INACTIVITY_THRESHOLD=300 # Seconds to wait before cleaning u
# ============================== #
DISABLE_TORRENT_STREAMS=False # When true, torrent-only requests return a friendly message instead of magnets
TORRENT_DISABLED_STREAM_NAME=[INFO] Comet # Stremio stream name shown when torrents are disabled
TORRENT_DISABLED_STREAM_DESCRIPTION=Direct torrent playback is disabled on this server. Please configure a debrid provider. # Description shown to users in Stremio
TORRENT_DISABLED_STREAM_DESCRIPTION=Direct torrent playback is disabled on this server. # Description shown to users in Stremio
TORRENT_DISABLED_STREAM_URL=https://comet.looks.legal # Optional URL included in the placeholder stream response
# ============================== #
comet/api/endpoints/stream.py
+2 -3
View File
@@ -159,9 +159,8 @@ async def stream(
if settings.DISABLE_TORRENT_STREAMS and config["debridService"] == "torrent":
placeholder_stream = {
"name": settings.TORRENT_DISABLED_STREAM_NAME or "[INFO] Comet",
"description": settings.TORRENT_DISABLED_STREAM_DESCRIPTION
or "Direct torrent playback is disabled on this server.",
"name": settings.TORRENT_DISABLED_STREAM_NAME,
"description": settings.TORRENT_DISABLED_STREAM_DESCRIPTION,
}
if settings.TORRENT_DISABLED_STREAM_URL:
placeholder_stream["url"] = settings.TORRENT_DISABLED_STREAM_URL
comet/core/logger.py
+24 -6
View File
@@ -195,7 +195,7 @@ def log_startup_info(settings):
replicas = f" - Read Replicas: {settings.DATABASE_READ_REPLICA_URLS}"
logger.log(
"COMET",
f"Database ({settings.DATABASE_TYPE}): {settings.DATABASE_PATH if settings.DATABASE_TYPE == 'sqlite' else settings.DATABASE_URL} - TTL: metadata={settings.METADATA_CACHE_TTL}s, torrents={settings.TORRENT_CACHE_TTL}s, live_torrents={settings.LIVE_TORRENT_CACHE_TTL}s, debrid={settings.DEBRID_CACHE_TTL}s, metrics={settings.METRICS_CACHE_TTL}s - Debrid Ratio: {settings.DEBRID_CACHE_CHECK_RATIO} - Startup Cleanup Interval: {settings.DATABASE_STARTUP_CLEANUP_INTERVAL}s{replicas}",
f"Database ({settings.DATABASE_TYPE}): {settings.DATABASE_PATH if settings.DATABASE_TYPE == 'sqlite' else settings.DATABASE_URL} - Batch Size: {settings.DATABASE_BATCH_SIZE} - TTL: metadata={settings.METADATA_CACHE_TTL}s, torrents={settings.TORRENT_CACHE_TTL}s, live_torrents={settings.LIVE_TORRENT_CACHE_TTL}s, debrid={settings.DEBRID_CACHE_TTL}s, metrics={settings.METRICS_CACHE_TTL}s - Debrid Ratio: {settings.DEBRID_CACHE_CHECK_RATIO} - Startup Cleanup Interval: {settings.DATABASE_STARTUP_CLEANUP_INTERVAL}s{replicas}",
)
# SQLite concurrency warnings
@@ -266,6 +266,9 @@ def log_startup_info(settings):
)
logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s")
logger.log("COMET", f"Magnet Resolve Timeout: {settings.MAGNET_RESOLVE_TIMEOUT}s")
logger.log("COMET", f"Catalog Timeout: {settings.CATALOG_TIMEOUT}s")
logger.log("COMET", f"Scrape Lock TTL: {settings.SCRAPE_LOCK_TTL}s")
logger.log("COMET", f"Scrape Wait Timeout: {settings.SCRAPE_WAIT_TIMEOUT}s")
logger.log(
"COMET", f"Download Torrent Files: {bool(settings.DOWNLOAD_TORRENT_FILES)}"
)
@@ -281,7 +284,7 @@ def log_startup_info(settings):
)
nyaa_anime_only = (
f" - Anime Only: {bool(settings.NYAA_ANIME_ONLY)}"
f" - Anime Only: {bool(settings.NYAA_ANIME_ONLY)} - Concurrent Pages: {settings.NYAA_MAX_CONCURRENT_PAGES}"
if settings.is_any_context_enabled(settings.SCRAPE_NYAA)
else ""
)
@@ -290,6 +293,16 @@ def log_startup_info(settings):
f"Nyaa Scraper: {settings.format_scraper_mode(settings.SCRAPE_NYAA)}{nyaa_anime_only}",
)
animetosho_anime_only = (
f" - Anime Only: {bool(settings.ANIMETOSHO_ANIME_ONLY)} - Concurrent Pages: {settings.ANIMETOSHO_MAX_CONCURRENT_PAGES}"
if settings.is_any_context_enabled(settings.SCRAPE_ANIMETOSHO)
else ""
)
logger.log(
"COMET",
f"AnimeTosho Scraper: {settings.format_scraper_mode(settings.SCRAPE_ANIMETOSHO)}{animetosho_anime_only}",
)
zilean_url = (
f" - {settings.ZILEAN_URL}"
if settings.is_any_context_enabled(settings.SCRAPE_ZILEAN)
@@ -310,14 +323,14 @@ def log_startup_info(settings):
f"StremThru Scraper: {settings.format_scraper_mode(settings.SCRAPE_STREMTHRU)}{stremthru_scrape_url}",
)
bitmagnet_url = (
f" - {settings.BITMAGNET_URL}"
bitmagnet_info = (
f" - {settings.BITMAGNET_URL} - Concurrent Pages: {settings.BITMAGNET_MAX_CONCURRENT_PAGES} - Max Offset: {settings.BITMAGNET_MAX_OFFSET}"
if settings.is_any_context_enabled(settings.SCRAPE_BITMAGNET)
else ""
)
logger.log(
"COMET",
f"Bitmagnet Scraper: {settings.format_scraper_mode(settings.SCRAPE_BITMAGNET)}{bitmagnet_url}",
f"Bitmagnet Scraper: {settings.format_scraper_mode(settings.SCRAPE_BITMAGNET)}{bitmagnet_info}",
)
torrentio_url = (
@@ -397,9 +410,14 @@ def log_startup_info(settings):
logger.log("COMET", f"StremThru URL: {settings.STREMTHRU_URL}")
disabled_streams_info = (
f" - Name: {settings.TORRENT_DISABLED_STREAM_NAME} - URL: {settings.TORRENT_DISABLED_STREAM_URL} - Description: {settings.TORRENT_DISABLED_STREAM_DESCRIPTION}"
if settings.DISABLE_TORRENT_STREAMS
else ""
)
logger.log(
"COMET",
f"Disable Torrent Streams: {bool(settings.DISABLE_TORRENT_STREAMS)}",
f"Disable Torrent Streams: {bool(settings.DISABLE_TORRENT_STREAMS)}{disabled_streams_info}",
)
logger.log("COMET", f"Remove Adult Content: {bool(settings.REMOVE_ADULT_CONTENT)}")
comet/core/models.py
+3
View File
@@ -74,6 +74,9 @@ class AppSettings(BaseSettings):
SCRAPE_NYAA: Union[bool, str] = False
NYAA_ANIME_ONLY: Optional[bool] = True
NYAA_MAX_CONCURRENT_PAGES: Optional[int] = 5
SCRAPE_ANIMETOSHO: Union[bool, str] = False
ANIMETOSHO_ANIME_ONLY: Optional[bool] = True
ANIMETOSHO_MAX_CONCURRENT_PAGES: Optional[int] = 10
SCRAPE_ZILEAN: Union[bool, str] = False
ZILEAN_URL: Union[str, List[str]] = "https://zileanfortheweebs.midnightignite.me"
SCRAPE_STREMTHRU: Union[bool, str] = False
comet/scrapers/animetosho.py
+110
View File
@@ -0,0 +1,110 @@
import asyncio
import xml.etree.ElementTree as ET
from comet.core.logger import logger
from comet.core.models import settings
from comet.scrapers.base import BaseScraper
from comet.scrapers.models import ScrapeRequest
from comet.services.torrent_manager import extract_trackers_from_magnet
class AnimeToshoScraper(BaseScraper):
def __init__(self, manager, session):
super().__init__(manager, session)
def parse_items(self, root):
torrents = []
for item in root.findall(".//item"):
try:
title = item.find("title").text
size = None
info_hash = None
seeders = None
magnet = None
for attr in item.findall(
".//torznab:attr",
{"torznab": "http://torznab.com/schemas/2015/feed"},
):
attr_name = attr.get("name")
attr_value = attr.get("value")
if attr_name == "size":
size = int(attr_value)
elif attr_name == "infohash":
info_hash = attr_value
elif attr_name == "seeders":
seeders = int(attr_value)
elif attr_name == "magneturl":
magnet = attr_value
if info_hash is None:
continue
torrents.append(
{
"title": title,
"infoHash": info_hash,
"fileIndex": None,
"seeders": seeders,
"size": size,
"tracker": "AnimeTosho",
"sources": extract_trackers_from_magnet(magnet),
}
)
except Exception as e:
logger.warning(f"Error parsing torrent item from AnimeTosho: {e}")
continue
return torrents
async def scrape_page(self, query, offset, limit):
try:
async with self.session.get(
f"https://feed.animetosho.org/api?t=search&q={query}&offset={offset}&limit={limit}"
) as response:
content = await response.text()
if not content.strip():
return [], 0
root = ET.fromstring(content)
response_node = root.find(
".//newznab:response",
{"newznab": "http://www.newznab.com/DTD/2010/feeds/attributes/"},
)
total = int(response_node.get("total", 0))
items = self.parse_items(root)
return items, total
except Exception as e:
logger.warning(f"Error scraping AnimeTosho offset={offset}: {e}")
return [], 0
async def scrape(self, request: ScrapeRequest):
torrents = []
query = request.title
limit = 150
initial_items, total = await self.scrape_page(query, 0, limit)
torrents.extend(initial_items)
if total > limit:
batch_size = settings.ANIMETOSHO_MAX_CONCURRENT_PAGES
current_offset = limit
while current_offset < total:
tasks = []
for _ in range(batch_size):
if current_offset >= total:
break
tasks.append(self.scrape_page(query, current_offset, limit))
current_offset += limit
if tasks:
results = await asyncio.gather(*tasks)
for batch_items, _ in results:
torrents.extend(batch_items)
return torrents
comet/scrapers/bitmagnet.py
+4 -5
View File
@@ -11,7 +11,7 @@ class BitmagnetScraper(BaseScraper):
def __init__(self, manager, session, url: str):
super().__init__(manager, session, url)
def parse_bitmagnet_items(self, root):
def parse_items(self, root):
torrents = []
for item in root.findall(".//item"):
try:
@@ -49,13 +49,12 @@ class BitmagnetScraper(BaseScraper):
"sources": [],
}
)
except Exception as e:
logger.warning(f"Error parsing torrent item from BitMagnet: {e}")
continue
return torrents
async def scrape_bitmagnet_page(
async def scrape_page(
self, imdb_id, scrape_type, offset, limit, season=None, episode=None
):
try:
@@ -76,7 +75,7 @@ class BitmagnetScraper(BaseScraper):
if not data_text.strip():
return []
root = ET.fromstring(data_text)
return self.parse_bitmagnet_items(root)
return self.parse_items(root)
except ET.ParseError:
return []
except Exception as e:
@@ -104,7 +103,7 @@ class BitmagnetScraper(BaseScraper):
if current_offset >= settings.BITMAGNET_MAX_OFFSET:
break
tasks.append(
self.scrape_bitmagnet_page(
self.scrape_page(
imdb_id, scrape_type, current_offset, limit, season, episode
)
)
comet/scrapers/manager.py
+9
View File
@@ -76,6 +76,15 @@ class ScraperManager:
):
continue
if (
scraper_name == "AnimeToshoScraper"
and settings.ANIMETOSHO_ANIME_ONLY
and not anime_mapper.is_anime_content(
request.media_id, request.media_only_id
)
):
continue
# Get client wrapper
client = network_manager.get_client(
scraper_name=scraper_name_clean, impersonate=scraper_class.impersonate