From a96986cec7a2266bf07685e1a09a63b0646bfff4 Mon Sep 17 00:00:00 2001
From: g0ldyy <153996346+g0ldyy@users.noreply.github.com>
Date: Sun, 7 Sep 2025 00:55:01 +0200
Subject: [PATCH] feat: Nyaa scraper

---
 .env-sample               |   3 +
 comet/main.py             |   5 ++
 comet/scrapers/manager.py |   3 +
 comet/scrapers/nyaa.py    | 129 ++++++++++++++++++++++++++++++++++++++
 comet/utils/database.py   |  22 +++----
 comet/utils/models.py     |   2 +
 comet/utils/torrent.py    |   4 +-
 7 files changed, 156 insertions(+), 12 deletions(-)
 create mode 100644 comet/scrapers/nyaa.py

diff --git a/.env-sample b/.env-sample
index 1c16a7d..6d33fab 100644
--- a/.env-sample
+++ b/.env-sample
@@ -89,6 +89,9 @@ COMET_URL=https://comet.elfhosted.com
 # Multi-instance example:
 # COMET_URL='["https://comet1.example.com", "https://comet2.example.com"]'
 
+SCRAPE_NYAA=True
+NYAA_MAX_CONCURRENT_PAGES=5 # Maximum number of concurrent requests to Nyaa (consider reducing if you are often ratelimited by Nyaa)
+
 SCRAPE_ZILEAN=True
 ZILEAN_URL=https://zilean.elfhosted.com
 # Multi-instance example:
diff --git a/comet/main.py b/comet/main.py
index 11639e1..b664b8e 100644
--- a/comet/main.py
+++ b/comet/main.py
@@ -195,6 +195,11 @@ def start_log():
         f"Comet Scraper: {bool(settings.SCRAPE_COMET)}{comet_url}",
     )
 
+    logger.log(
+        "COMET",
+        f"Nyaa Scraper: {bool(settings.SCRAPE_NYAA)}",
+    )
+
     zilean_url = f" - {settings.ZILEAN_URL}" if settings.SCRAPE_ZILEAN else ""
     logger.log(
         "COMET",
diff --git a/comet/scrapers/manager.py b/comet/scrapers/manager.py
index 65765ec..7a4b8f1 100644
--- a/comet/scrapers/manager.py
+++ b/comet/scrapers/manager.py
@@ -30,6 +30,7 @@ from .aiostreams import get_aiostreams
 from .jackettio import get_jackettio
 from .debridio import get_debridio
 from .torbox import get_torbox
+from .nyaa import get_nyaa
 
 
 class TorrentManager:
@@ -79,6 +80,8 @@ class TorrentManager:
             tasks.extend(get_all_torrentio_tasks(self))
         if settings.SCRAPE_MEDIAFUSION:
             tasks.extend(get_all_mediafusion_tasks(self))
+        if settings.SCRAPE_NYAA:
+            tasks.extend(get_nyaa(self))
         if settings.SCRAPE_ZILEAN:
             tasks.extend(get_all_zilean_tasks(self, session))
         if settings.SCRAPE_STREMTHRU:
diff --git a/comet/scrapers/nyaa.py b/comet/scrapers/nyaa.py
new file mode 100644
index 0000000..527492b
--- /dev/null
+++ b/comet/scrapers/nyaa.py
@@ -0,0 +1,129 @@
+import re
+import asyncio
+
+from curl_cffi import requests
+
+from comet.utils.general import log_scraper_error, size_to_bytes
+from comet.utils.torrent import extract_trackers_from_magnet
+from comet.utils.logger import logger
+from comet.utils.models import settings
+
+PAGE_PATTERN = re.compile(r'(\d+)(?=">\d+<\/a><\/li><li class="next">)')
+MAGNET_PATTERN = re.compile(r'href="(magnet:[^"]+)"')
+SIZE_PATTERN = re.compile(r'<td class="text-center">([\d.]+ (?:KiB|MiB|GiB|TiB))</td>')
+SEEDERS_PATTERN = re.compile(
+    r'<td class="text-center">(\d+)</td>\s*<td class="text-center">(\d+)</td>\s*<td class="text-center">(\d+)</td>'
+)
+TITLE_PATTERN = re.compile(r'href="/view/\d+" title="([^"]+)"')
+INFO_HASH_PATTERN = re.compile(r"btih:([a-fA-F0-9]{40}|[a-zA-Z0-9]{32})")
+
+
+def extract_torrent_data(html_content: str):
+    torrents = []
+
+    magnet_links = MAGNET_PATTERN.findall(html_content)
+
+    sizes = SIZE_PATTERN.findall(html_content)
+
+    seeders_data = SEEDERS_PATTERN.findall(html_content)
+    seeders = [int(match[0]) for match in seeders_data]
+
+    titles = TITLE_PATTERN.findall(html_content)
+
+    for i in range(len(magnet_links)):
+        magnet = magnet_links[i]
+        info_hash = INFO_HASH_PATTERN.search(magnet).group(1)
+
+        size_str = sizes[i]
+        try:
+            size_bytes = size_to_bytes(size_str.replace("iB", "B"))
+        except Exception:
+            size_bytes = 0
+
+        torrent = {
+            "title": titles[i],
+            "infoHash": info_hash,
+            "fileIndex": None,
+            "seeders": seeders[i],
+            "size": size_bytes,
+            "tracker": "Nyaa",
+            "sources": extract_trackers_from_magnet(magnet),
+        }
+        torrents.append(torrent)
+
+    return torrents
+
+
+async def scrape_nyaa_page(
+    session: requests.AsyncSession, semaphore: asyncio.Semaphore, query: str, page: int
+):
+    async with semaphore:
+        url = f"https://nyaa.si/?q={query}"
+        if page > 1:
+            url += f"&p={page}"
+
+        response = await session.get(url)
+        if response.status_code != 200:
+            logger.warning(
+                f"Failed to scrape Nyaa page {page} (consider reducing NYAA_MAX_CONCURRENT_PAGES): {response.status_code}"
+            )
+            return []
+
+        html_content = response.text
+        return extract_torrent_data(html_content)
+
+
+async def get_all_nyaa_pages(session: requests.AsyncSession, query: str):
+    all_torrents = []
+
+    max_concurrent = settings.NYAA_MAX_CONCURRENT_PAGES
+    semaphore = asyncio.Semaphore(max_concurrent)
+
+    first_page_url = f"https://nyaa.si/?q={query}"
+    response = await session.get(first_page_url)
+    if response.status_code != 200:
+        logger.warning(f"Failed to scrape Nyaa page 1: {response.status_code}")
+        return []
+
+    first_page_text = response.text
+
+    first_page_torrents = extract_torrent_data(first_page_text)
+    all_torrents.extend(first_page_torrents)
+
+    last_page_matches = PAGE_PATTERN.findall(first_page_text)
+    if len(last_page_matches) == 0:
+        return all_torrents
+
+    last_page_number = int(last_page_matches[0])
+
+    if last_page_number > 1:
+        tasks = []
+        for page_number in range(2, last_page_number + 1):
+            tasks.append(scrape_nyaa_page(session, semaphore, query, page_number))
+
+        page_results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        for result in page_results:
+            if isinstance(result, list):
+                all_torrents.extend(result)
+
+    return all_torrents
+
+
+def get_nyaa(manager):
+    async def nyaa_scraper_task():
+        torrents = []
+
+        try:
+            async with requests.AsyncSession() as session:
+                query = manager.title
+
+                all_torrents = await get_all_nyaa_pages(session, query)
+                torrents.extend(all_torrents)
+
+        except Exception as e:
+            log_scraper_error("Nyaa", "https://nyaa.si", manager.media_id, e)
+
+        await manager.filter_manager(torrents)
+
+    return [nyaa_scraper_task()]
diff --git a/comet/utils/database.py b/comet/utils/database.py
index c3318f2..41c5eff 100644
--- a/comet/utils/database.py
+++ b/comet/utils/database.py
@@ -323,7 +323,7 @@ async def setup_database():
         # =============================================================================
         # TORRENTS TABLE INDEXES - Most critical for performance
         # =============================================================================
-        
+
         # Primary lookup index: media_id + season + episode + timestamp (cache TTL filter)
         await database.execute(
             """
@@ -365,9 +365,9 @@ async def setup_database():
         )
 
         # =============================================================================
-        # DEBRID_AVAILABILITY TABLE INDEXES - Critical for cache performance  
+        # DEBRID_AVAILABILITY TABLE INDEXES - Critical for cache performance
         # =============================================================================
-        
+
         # Primary cache lookup: service + info_hash list + timestamp
         await database.execute(
             """
@@ -403,7 +403,7 @@ async def setup_database():
         # =============================================================================
         # DOWNLOAD_LINKS_CACHE TABLE INDEXES - Playback performance
         # =============================================================================
-        
+
         # Primary playback lookup: debrid_key + info_hash + season + episode
         await database.execute(
             """
@@ -423,7 +423,7 @@ async def setup_database():
         # =============================================================================
         # METADATA_CACHE TABLE INDEXES - Metadata performance
         # =============================================================================
-        
+
         # Primary cache lookup: media_id + timestamp
         await database.execute(
             """
@@ -443,7 +443,7 @@ async def setup_database():
         # =============================================================================
         # FIRST_SEARCHES TABLE INDEXES - Search optimization
         # =============================================================================
-        
+
         # Primary search check: media_id (already PRIMARY KEY, but explicit for clarity)
         # Media ID is already PRIMARY KEY, so focusing on timestamp for TTL cleanup
         await database.execute(
@@ -456,7 +456,7 @@ async def setup_database():
         # =============================================================================
         # ACTIVE_CONNECTIONS TABLE INDEXES - Admin dashboard performance
         # =============================================================================
-        
+
         # Admin dashboard ordering: timestamp DESC (most recent first)
         await database.execute(
             """
@@ -484,7 +484,7 @@ async def setup_database():
         # =============================================================================
         # SCRAPE_LOCKS TABLE INDEXES - Lock management
         # =============================================================================
-        
+
         # Expired locks cleanup: expires_at < current_time
         await database.execute(
             """
@@ -504,7 +504,7 @@ async def setup_database():
         # =============================================================================
         # ADMIN_SESSIONS TABLE INDEXES - Authentication performance
         # =============================================================================
-        
+
         # Session cleanup: expires_at < current_time
         await database.execute(
             """
@@ -516,7 +516,7 @@ async def setup_database():
         # =============================================================================
         # BACKGROUND_SCRAPER_STATE TABLE INDEXES - Scraper performance
         # =============================================================================
-        
+
         # Media type filtering for scraper analytics
         await database.execute(
             """
@@ -544,7 +544,7 @@ async def setup_database():
         # =============================================================================
         # COMPOSITE INDEXES FOR COMPLEX QUERIES
         # =============================================================================
-        
+
         # Torrents: media + quality filtering + cache validity
         await database.execute(
             """
diff --git a/comet/utils/models.py b/comet/utils/models.py
index 4c06459..6702618 100644
--- a/comet/utils/models.py
+++ b/comet/utils/models.py
@@ -56,6 +56,8 @@ class AppSettings(BaseSettings):
     DOWNLOAD_TORRENT_FILES: Optional[bool] = False
     SCRAPE_COMET: Optional[bool] = False
     COMET_URL: Union[str, List[str]] = "https://comet.elfhosted.com"
+    SCRAPE_NYAA: Optional[bool] = False
+    NYAA_MAX_CONCURRENT_PAGES: Optional[int] = 5
     SCRAPE_ZILEAN: Optional[bool] = False
     ZILEAN_URL: Union[str, List[str]] = "https://zilean.elfhosted.com"
     SCRAPE_STREMTHRU: Optional[bool] = False
diff --git a/comet/utils/torrent.py b/comet/utils/torrent.py
index cacec16..c28da50 100644
--- a/comet/utils/torrent.py
+++ b/comet/utils/torrent.py
@@ -7,6 +7,7 @@ import asyncio
 import orjson
 import time
 import base64
+import html
 
 from urllib.parse import parse_qs, urlparse
 from demagnetize.core import Demagnetizer
@@ -22,7 +23,8 @@ info_hash_pattern = re.compile(r"btih:([a-fA-F0-9]{40}|[a-zA-Z0-9]{32})")
 
 def extract_trackers_from_magnet(magnet_uri: str):
     try:
-        parsed = urlparse(magnet_uri)
+        decoded_uri = html.unescape(magnet_uri)
+        parsed = urlparse(decoded_uri)
         params = parse_qs(parsed.query)
         return params.get("tr", [])
     except Exception as e: