g0ldyy/comet
1
0
Fork 0
mirror of https://github.com/g0ldyy/comet.git synced 2026-01-12 01:16:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: Nyaa scraper

Browse Source
This commit is contained in:
g0ldyy
2025-09-07 00:55:01 +02:00
parent b9259319c3
commit a96986cec7
7 changed files with 156 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env-sample
+3
View File
@@ -89,6 +89,9 @@ COMET_URL=https://comet.elfhosted.com
# Multi-instance example:
# COMET_URL='["https://comet1.example.com", "https://comet2.example.com"]'
SCRAPE_NYAA=True
NYAA_MAX_CONCURRENT_PAGES=5 # Maximum number of concurrent requests to Nyaa (consider reducing if you are often ratelimited by Nyaa)
SCRAPE_ZILEAN=True
ZILEAN_URL=https://zilean.elfhosted.com
# Multi-instance example:
comet/main.py
+5
View File
@@ -195,6 +195,11 @@ def start_log():
f"Comet Scraper: {bool(settings.SCRAPE_COMET)}{comet_url}",
)
logger.log(
"COMET",
f"Nyaa Scraper: {bool(settings.SCRAPE_NYAA)}",
)
zilean_url = f" - {settings.ZILEAN_URL}" if settings.SCRAPE_ZILEAN else ""
logger.log(
"COMET",
comet/scrapers/manager.py
+3
View File
@@ -30,6 +30,7 @@ from .aiostreams import get_aiostreams
from .jackettio import get_jackettio
from .debridio import get_debridio
from .torbox import get_torbox
from .nyaa import get_nyaa
class TorrentManager:
@@ -79,6 +80,8 @@ class TorrentManager:
tasks.extend(get_all_torrentio_tasks(self))
if settings.SCRAPE_MEDIAFUSION:
tasks.extend(get_all_mediafusion_tasks(self))
if settings.SCRAPE_NYAA:
tasks.extend(get_nyaa(self))
if settings.SCRAPE_ZILEAN:
tasks.extend(get_all_zilean_tasks(self, session))
if settings.SCRAPE_STREMTHRU:
comet/scrapers/nyaa.py
+129
View File
@@ -0,0 +1,129 @@
import re
import asyncio
from curl_cffi import requests
from comet.utils.general import log_scraper_error, size_to_bytes
from comet.utils.torrent import extract_trackers_from_magnet
from comet.utils.logger import logger
from comet.utils.models import settings
PAGE_PATTERN = re.compile(r'(\d+)(?=">\d+<\/a><\/li><li class="next">)')
MAGNET_PATTERN = re.compile(r'href="(magnet:[^"]+)"')
SIZE_PATTERN = re.compile(r'<td class="text-center">([\d.]+ (?:KiB|MiB|GiB|TiB))</td>')
SEEDERS_PATTERN = re.compile(
r'<td class="text-center">(\d+)</td>\s*<td class="text-center">(\d+)</td>\s*<td class="text-center">(\d+)</td>'
)
TITLE_PATTERN = re.compile(r'href="/view/\d+" title="([^"]+)"')
INFO_HASH_PATTERN = re.compile(r"btih:([a-fA-F0-9]{40}|[a-zA-Z0-9]{32})")
def extract_torrent_data(html_content: str):
torrents = []
magnet_links = MAGNET_PATTERN.findall(html_content)
sizes = SIZE_PATTERN.findall(html_content)
seeders_data = SEEDERS_PATTERN.findall(html_content)
seeders = [int(match[0]) for match in seeders_data]
titles = TITLE_PATTERN.findall(html_content)
for i in range(len(magnet_links)):
magnet = magnet_links[i]
info_hash = INFO_HASH_PATTERN.search(magnet).group(1)
size_str = sizes[i]
try:
size_bytes = size_to_bytes(size_str.replace("iB", "B"))
except Exception:
size_bytes = 0
torrent = {
"title": titles[i],
"infoHash": info_hash,
"fileIndex": None,
"seeders": seeders[i],
"size": size_bytes,
"tracker": "Nyaa",
"sources": extract_trackers_from_magnet(magnet),
}
torrents.append(torrent)
return torrents
async def scrape_nyaa_page(
session: requests.AsyncSession, semaphore: asyncio.Semaphore, query: str, page: int
):
async with semaphore:
url = f"https://nyaa.si/?q={query}"
if page > 1:
url += f"&p={page}"
response = await session.get(url)
if response.status_code != 200:
logger.warning(
f"Failed to scrape Nyaa page {page} (consider reducing NYAA_MAX_CONCURRENT_PAGES): {response.status_code}"
)
return []
html_content = response.text
return extract_torrent_data(html_content)
async def get_all_nyaa_pages(session: requests.AsyncSession, query: str):
all_torrents = []
max_concurrent = settings.NYAA_MAX_CONCURRENT_PAGES
semaphore = asyncio.Semaphore(max_concurrent)
first_page_url = f"https://nyaa.si/?q={query}"
response = await session.get(first_page_url)
if response.status_code != 200:
logger.warning(f"Failed to scrape Nyaa page 1: {response.status_code}")
return []
first_page_text = response.text
first_page_torrents = extract_torrent_data(first_page_text)
all_torrents.extend(first_page_torrents)
last_page_matches = PAGE_PATTERN.findall(first_page_text)
if len(last_page_matches) == 0:
return all_torrents
last_page_number = int(last_page_matches[0])
if last_page_number > 1:
tasks = []
for page_number in range(2, last_page_number + 1):
tasks.append(scrape_nyaa_page(session, semaphore, query, page_number))
page_results = await asyncio.gather(*tasks, return_exceptions=True)
for result in page_results:
if isinstance(result, list):
all_torrents.extend(result)
return all_torrents
def get_nyaa(manager):
async def nyaa_scraper_task():
torrents = []
try:
async with requests.AsyncSession() as session:
query = manager.title
all_torrents = await get_all_nyaa_pages(session, query)
torrents.extend(all_torrents)
except Exception as e:
log_scraper_error("Nyaa", "https://nyaa.si", manager.media_id, e)
await manager.filter_manager(torrents)
return [nyaa_scraper_task()]
comet/utils/database.py
+11 -11
View File
@@ -323,7 +323,7 @@ async def setup_database():
# =============================================================================
# TORRENTS TABLE INDEXES - Most critical for performance
# =============================================================================
# Primary lookup index: media_id + season + episode + timestamp (cache TTL filter)
await database.execute(
"""
@@ -365,9 +365,9 @@ async def setup_database():
)
# =============================================================================
# DEBRID_AVAILABILITY TABLE INDEXES - Critical for cache performance
# DEBRID_AVAILABILITY TABLE INDEXES - Critical for cache performance
# =============================================================================
# Primary cache lookup: service + info_hash list + timestamp
await database.execute(
"""
@@ -403,7 +403,7 @@ async def setup_database():
# =============================================================================
# DOWNLOAD_LINKS_CACHE TABLE INDEXES - Playback performance
# =============================================================================
# Primary playback lookup: debrid_key + info_hash + season + episode
await database.execute(
"""
@@ -423,7 +423,7 @@ async def setup_database():
# =============================================================================
# METADATA_CACHE TABLE INDEXES - Metadata performance
# =============================================================================
# Primary cache lookup: media_id + timestamp
await database.execute(
"""
@@ -443,7 +443,7 @@ async def setup_database():
# =============================================================================
# FIRST_SEARCHES TABLE INDEXES - Search optimization
# =============================================================================
# Primary search check: media_id (already PRIMARY KEY, but explicit for clarity)
# Media ID is already PRIMARY KEY, so focusing on timestamp for TTL cleanup
await database.execute(
@@ -456,7 +456,7 @@ async def setup_database():
# =============================================================================
# ACTIVE_CONNECTIONS TABLE INDEXES - Admin dashboard performance
# =============================================================================
# Admin dashboard ordering: timestamp DESC (most recent first)
await database.execute(
"""
@@ -484,7 +484,7 @@ async def setup_database():
# =============================================================================
# SCRAPE_LOCKS TABLE INDEXES - Lock management
# =============================================================================
# Expired locks cleanup: expires_at < current_time
await database.execute(
"""
@@ -504,7 +504,7 @@ async def setup_database():
# =============================================================================
# ADMIN_SESSIONS TABLE INDEXES - Authentication performance
# =============================================================================
# Session cleanup: expires_at < current_time
await database.execute(
"""
@@ -516,7 +516,7 @@ async def setup_database():
# =============================================================================
# BACKGROUND_SCRAPER_STATE TABLE INDEXES - Scraper performance
# =============================================================================
# Media type filtering for scraper analytics
await database.execute(
"""
@@ -544,7 +544,7 @@ async def setup_database():
# =============================================================================
# COMPOSITE INDEXES FOR COMPLEX QUERIES
# =============================================================================
# Torrents: media + quality filtering + cache validity
await database.execute(
"""
comet/utils/models.py
+2
View File
@@ -56,6 +56,8 @@ class AppSettings(BaseSettings):
DOWNLOAD_TORRENT_FILES: Optional[bool] = False
SCRAPE_COMET: Optional[bool] = False
COMET_URL: Union[str, List[str]] = "https://comet.elfhosted.com"
SCRAPE_NYAA: Optional[bool] = False
NYAA_MAX_CONCURRENT_PAGES: Optional[int] = 5
SCRAPE_ZILEAN: Optional[bool] = False
ZILEAN_URL: Union[str, List[str]] = "https://zilean.elfhosted.com"
SCRAPE_STREMTHRU: Optional[bool] = False
comet/utils/torrent.py
+3 -1
View File
@@ -7,6 +7,7 @@ import asyncio
import orjson
import time
import base64
import html
from urllib.parse import parse_qs, urlparse
from demagnetize.core import Demagnetizer
@@ -22,7 +23,8 @@ info_hash_pattern = re.compile(r"btih:([a-fA-F0-9]{40}|[a-zA-Z0-9]{32})")
def extract_trackers_from_magnet(magnet_uri: str):
try:
parsed = urlparse(magnet_uri)
decoded_uri = html.unescape(magnet_uri)
parsed = urlparse(decoded_uri)
params = parse_qs(parsed.query)
return params.get("tr", [])
except Exception as e: