diff --git a/.env-sample b/.env-sample index 4c609ed..1848473 100644 --- a/.env-sample +++ b/.env-sample @@ -190,6 +190,7 @@ TORBOX_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX SCRAPE_YGGTORRENT=False YGGTORRENT_USERNAME=username YGGTORRENT_PASSWORD=password +YGGTORRENT_PASSKEY=passkey YGGTORRENT_MAX_CONCURRENT_PAGES=5 # ============================== # diff --git a/comet/core/logger.py b/comet/core/logger.py index ad763aa..a434750 100644 --- a/comet/core/logger.py +++ b/comet/core/logger.py @@ -342,7 +342,7 @@ def log_startup_info(settings): ) yggtorrent_info = ( - f" - Username: {settings.YGGTORRENT_USERNAME} - Password: {settings.YGGTORRENT_PASSWORD}" + f" - Username: {settings.YGGTORRENT_USERNAME} - Password: {settings.YGGTORRENT_PASSWORD} - Passkey: {settings.YGGTORRENT_PASSKEY}" if settings.is_any_context_enabled(settings.SCRAPE_YGGTORRENT) else "" ) diff --git a/comet/core/models.py b/comet/core/models.py index 857fb83..461d2b2 100644 --- a/comet/core/models.py +++ b/comet/core/models.py @@ -103,6 +103,7 @@ class AppSettings(BaseSettings): SCRAPE_YGGTORRENT: Union[bool, str] = False YGGTORRENT_USERNAME: Optional[str] = None YGGTORRENT_PASSWORD: Optional[str] = None + YGGTORRENT_PASSKEY: Optional[str] = None YGGTORRENT_MAX_CONCURRENT_PAGES: Optional[int] = 5 CUSTOM_HEADER_HTML: Optional[str] = None PROXY_DEBRID_STREAM: Optional[bool] = False diff --git a/comet/scrapers/yggtorrent.py b/comet/scrapers/yggtorrent.py index 6bb8d98..52314a2 100644 --- a/comet/scrapers/yggtorrent.py +++ b/comet/scrapers/yggtorrent.py @@ -1,6 +1,5 @@ import asyncio import re -from urllib.parse import urlparse import aiohttp from curl_cffi import requests @@ -9,7 +8,6 @@ from comet.core.logger import logger from comet.core.models import settings from comet.scrapers.base import BaseScraper from comet.scrapers.models import ScrapeRequest -from comet.services.torrent_manager import extract_torrent_metadata from comet.utils.formatting import size_to_bytes LOGIN_PAGE = "/auth/login" @@ -34,22 +32,25 @@ class YGGTorrentScraper(BaseScraper): if cls._domain: return cls._domain - try: - async with requests.AsyncSession(impersonate="chrome") as session: - response = await session.get("https://ygg.re", allow_redirects=False) - if "location" in response.headers: - location = response.headers["location"] - domain = urlparse(location).netloc - logger.info(f"Resolved YGG domain to: {domain}") - cls._domain = domain - return domain - elif response.status_code == 200: - cls._domain = urlparse(response.url).netloc - return cls._domain - except Exception as e: - logger.error(f"Error resolving YGG domain: {e}") - return None - return None + cls._domain = "www.yggtorrent.org" + return cls._domain + + # try: + # async with requests.AsyncSession(impersonate="chrome") as session: + # response = await session.get("https://ygg.re", allow_redirects=False) + # if "location" in response.headers: + # location = response.headers["location"] + # domain = urlparse(location).netloc + # logger.info(f"Resolved YGG domain to: {domain}") + # cls._domain = domain + # return domain + # elif response.status_code == 200: + # cls._domain = urlparse(response.url).netloc + # return cls._domain + # except Exception as e: + # logger.error(f"Error resolving YGG domain: {e}") + # return None + # return None @classmethod async def _ensure_session(cls): @@ -112,50 +113,60 @@ class YGGTorrentScraper(BaseScraper): await session.close() return False - async def _download_torrent(self, url): + async def _process_torrent(self, domain, url, title, seeders, size): try: response = await self._session.get(url) - if response.status_code == 200: - return response.content - logger.warning(f"Failed to download torrent file: {response.status_code}") - except Exception as e: - logger.warning(f"Exception downloading torrent file: {e}") - return None - - async def _process_torrent(self, domain, torrent_id, title, seeders, size): - download_url = f"https://{domain}/engine/download_torrent?id={torrent_id}" - - torrent_content = await self._download_torrent(download_url) - - results = [] - if torrent_content: - try: - metadata = extract_torrent_metadata(torrent_content) - if metadata: - for file in metadata["files"]: - results.append( - { - "title": file["name"], - "infoHash": metadata["info_hash"].lower(), - "fileIndex": file["index"], - "seeders": seeders, - "size": file["size"], - "tracker": "YGGTorrent", - "sources": metadata["announce_list"], - } - ) - else: - logger.warning( - f"Failed to extract metadata for torrent {torrent_id}" - ) - except Exception as e: + if response.status_code != 200: logger.warning( - f"Exception extracting metadata for torrent {torrent_id}: {e}" + f"Failed to fetch torrent page {url}: {response.status_code}" ) - else: - logger.warning(f"Failed to download torrent content for {torrent_id}") + return [] - return results + html_content = response.text + + hash_match = re.search( + r"Hash\s*\s*]*>(.*?)", + html_content, + re.IGNORECASE | re.DOTALL, + ) + + if not hash_match: + logger.warning(f"Could not find hash in page {url}") + return [] + + info_hash = ( + hash_match.group(1).strip().replace("Tester", "").strip().lower() + ) + + info_hash = re.sub(r"<[^>]+>", "", info_hash).strip() + + if not info_hash or len(info_hash) != 40: + logger.warning(f"Invalid hash found: {info_hash} in {url}") + return [] + + if not settings.YGGTORRENT_PASSKEY: + logger.warning( + "YGGTORRENT_PASSKEY not set, cannot construct source URL." + ) + return [] + + source = f"http://tracker.p2p-world.net:8080/{settings.YGGTORRENT_PASSKEY}/announce" + + return [ + { + "title": title, + "infoHash": info_hash, + "fileIndex": None, + "seeders": seeders, + "size": size, + "tracker": "YGGTorrent", + "sources": [source], + } + ] + + except Exception as e: + logger.warning(f"Exception processing torrent {url}: {e}") + return [] async def _scrape_page(self, domain, query, offset, semaphore): url = f"https://{domain}/engine/search?name={query}&do=search&page={offset}&category=2145" @@ -193,10 +204,8 @@ class YGGTorrentScraper(BaseScraper): name_match = NAME_PATTERN.search(name_html) href = name_match.group(1) - id_match = re.search(r"/torrent/.*/(\d+)-", href) - torrent_id = id_match.group(1) - title = name_match.group(2) + title = name_match.group(2).split("")[0].strip() seeders = int(tds[7]) @@ -205,9 +214,7 @@ class YGGTorrentScraper(BaseScraper): clean_size = re.sub(r"(\d)([a-zA-Z])", r"\1 \2", clean_size) size = size_to_bytes(clean_size) - tasks.append( - self._process_torrent(domain, torrent_id, title, seeders, size) - ) + tasks.append(self._process_torrent(domain, href, title, seeders, size)) results = await asyncio.gather(*tasks) return [r for res in results for r in res], total_results