g0ldyy/comet
1
0
Fork 0
mirror of https://github.com/g0ldyy/comet.git synced 2026-01-12 01:16:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #399 from g0ldyy/feat/fuck-ygg

Browse Source 
feat: add YGGTORRENT_PASSKEY setting and update Yggtorrent scraper to parse info hash from page HTML and use passkey for tracker sources
This commit is contained in:
Goldy
2025-12-21 18:19:45 +01:00
committed by GitHub
parent beacb2a966 6e44eb260d
commit c442b0c269
4 changed files with 73 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env-sample
+1
View File
@@ -190,6 +190,7 @@ TORBOX_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
SCRAPE_YGGTORRENT=False
YGGTORRENT_USERNAME=username
YGGTORRENT_PASSWORD=password
YGGTORRENT_PASSKEY=passkey
YGGTORRENT_MAX_CONCURRENT_PAGES=5
# ============================== #
comet/core/logger.py
+1 -1
View File
@@ -342,7 +342,7 @@ def log_startup_info(settings):
)
yggtorrent_info = (
f" - Username: {settings.YGGTORRENT_USERNAME} - Password: {settings.YGGTORRENT_PASSWORD}"
f" - Username: {settings.YGGTORRENT_USERNAME} - Password: {settings.YGGTORRENT_PASSWORD} - Passkey: {settings.YGGTORRENT_PASSKEY}"
if settings.is_any_context_enabled(settings.SCRAPE_YGGTORRENT)
else ""
)
comet/core/models.py
+1
View File
@@ -103,6 +103,7 @@ class AppSettings(BaseSettings):
SCRAPE_YGGTORRENT: Union[bool, str] = False
YGGTORRENT_USERNAME: Optional[str] = None
YGGTORRENT_PASSWORD: Optional[str] = None
YGGTORRENT_PASSKEY: Optional[str] = None
YGGTORRENT_MAX_CONCURRENT_PAGES: Optional[int] = 5
CUSTOM_HEADER_HTML: Optional[str] = None
PROXY_DEBRID_STREAM: Optional[bool] = False
comet/scrapers/yggtorrent.py
+70 -63
View File
@@ -1,6 +1,5 @@
import asyncio
import re
from urllib.parse import urlparse
import aiohttp
from curl_cffi import requests
@@ -9,7 +8,6 @@ from comet.core.logger import logger
from comet.core.models import settings
from comet.scrapers.base import BaseScraper
from comet.scrapers.models import ScrapeRequest
from comet.services.torrent_manager import extract_torrent_metadata
from comet.utils.formatting import size_to_bytes
LOGIN_PAGE = "/auth/login"
@@ -34,22 +32,25 @@ class YGGTorrentScraper(BaseScraper):
if cls._domain:
return cls._domain
try:
async with requests.AsyncSession(impersonate="chrome") as session:
response = await session.get("https://ygg.re", allow_redirects=False)
if "location" in response.headers:
location = response.headers["location"]
domain = urlparse(location).netloc
logger.info(f"Resolved YGG domain to: {domain}")
cls._domain = domain
return domain
elif response.status_code == 200:
cls._domain = urlparse(response.url).netloc
return cls._domain
except Exception as e:
logger.error(f"Error resolving YGG domain: {e}")
return None
return None
cls._domain = "www.yggtorrent.org"
return cls._domain
# try:
# async with requests.AsyncSession(impersonate="chrome") as session:
# response = await session.get("https://ygg.re", allow_redirects=False)
# if "location" in response.headers:
# location = response.headers["location"]
# domain = urlparse(location).netloc
# logger.info(f"Resolved YGG domain to: {domain}")
# cls._domain = domain
# return domain
# elif response.status_code == 200:
# cls._domain = urlparse(response.url).netloc
# return cls._domain
# except Exception as e:
# logger.error(f"Error resolving YGG domain: {e}")
# return None
# return None
@classmethod
async def _ensure_session(cls):
@@ -112,50 +113,60 @@ class YGGTorrentScraper(BaseScraper):
await session.close()
return False
async def _download_torrent(self, url):
async def _process_torrent(self, domain, url, title, seeders, size):
try:
response = await self._session.get(url)
if response.status_code == 200:
return response.content
logger.warning(f"Failed to download torrent file: {response.status_code}")
except Exception as e:
logger.warning(f"Exception downloading torrent file: {e}")
return None
async def _process_torrent(self, domain, torrent_id, title, seeders, size):
download_url = f"https://{domain}/engine/download_torrent?id={torrent_id}"
torrent_content = await self._download_torrent(download_url)
results = []
if torrent_content:
try:
metadata = extract_torrent_metadata(torrent_content)
if metadata:
for file in metadata["files"]:
results.append(
{
"title": file["name"],
"infoHash": metadata["info_hash"].lower(),
"fileIndex": file["index"],
"seeders": seeders,
"size": file["size"],
"tracker": "YGGTorrent",
"sources": metadata["announce_list"],
}
)
else:
logger.warning(
f"Failed to extract metadata for torrent {torrent_id}"
)
except Exception as e:
if response.status_code != 200:
logger.warning(
f"Exception extracting metadata for torrent {torrent_id}: {e}"
f"Failed to fetch torrent page {url}: {response.status_code}"
)
else:
logger.warning(f"Failed to download torrent content for {torrent_id}")
return []
return results
html_content = response.text
hash_match = re.search(
r"Hash\s*</td>\s*<td[^>]*>(.*?)</td>",
html_content,
re.IGNORECASE | re.DOTALL,
)
if not hash_match:
logger.warning(f"Could not find hash in page {url}")
return []
info_hash = (
hash_match.group(1).strip().replace("Tester", "").strip().lower()
)
info_hash = re.sub(r"<[^>]+>", "", info_hash).strip()
if not info_hash or len(info_hash) != 40:
logger.warning(f"Invalid hash found: {info_hash} in {url}")
return []
if not settings.YGGTORRENT_PASSKEY:
logger.warning(
"YGGTORRENT_PASSKEY not set, cannot construct source URL."
)
return []
source = f"http://tracker.p2p-world.net:8080/{settings.YGGTORRENT_PASSKEY}/announce"
return [
{
"title": title,
"infoHash": info_hash,
"fileIndex": None,
"seeders": seeders,
"size": size,
"tracker": "YGGTorrent",
"sources": [source],
}
]
except Exception as e:
logger.warning(f"Exception processing torrent {url}: {e}")
return []
async def _scrape_page(self, domain, query, offset, semaphore):
url = f"https://{domain}/engine/search?name={query}&do=search&page={offset}&category=2145"
@@ -193,10 +204,8 @@ class YGGTorrentScraper(BaseScraper):
name_match = NAME_PATTERN.search(name_html)
href = name_match.group(1)
id_match = re.search(r"/torrent/.*/(\d+)-", href)
torrent_id = id_match.group(1)
title = name_match.group(2)
title = name_match.group(2).split("</a>")[0].strip()
seeders = int(tds[7])
@@ -205,9 +214,7 @@ class YGGTorrentScraper(BaseScraper):
clean_size = re.sub(r"(\d)([a-zA-Z])", r"\1 \2", clean_size)
size = size_to_bytes(clean_size)
tasks.append(
self._process_torrent(domain, torrent_id, title, seeders, size)
)
tasks.append(self._process_torrent(domain, href, title, seeders, size))
results = await asyncio.gather(*tasks)
return [r for res in results for r in res], total_results