g0ldyy/comet
1
0
Fork 0
mirror of https://github.com/g0ldyy/comet.git synced 2026-01-12 01:16:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: new admin dashboard for debrid proxy stream + title year check + ip-based max connections for debrid proxy stream

Browse Source
This commit is contained in:
Goldy
2024-08-27 12:36:07 +02:00
parent 7533ecb4b0
commit 1d11ac46bd
10 changed files with 635 additions and 926 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env-sample
+8 -4
View File
@@ -2,20 +2,24 @@ ADDON_ID=stremio.comet.fast # for Stremio
ADDON_NAME=Comet # for Stremio
FASTAPI_HOST=0.0.0.0
FASTAPI_PORT=8000
FASTAPI_WORKERS=1 # remove to destroy CPU -> max performances :)
FASTAPI_WORKERS=1 # remove to destroy CPU -> max performances :D
DASHBOARD_ADMIN_PASSWORD=CHANGE_ME # The password to access the dashboard with active connections and soon more...
DATABASE_PATH=data/comet.db # only change it if you know what it is - folders in path must exist
CACHE_TTL=86400 # cache duration in seconds
DEBRID_PROXY_URL=http://127.0.0.1:1080 # https://github.com/cmj2002/warp-docker to bypass Debrid Services and Torrentio server IP blacklist
INDEXER_MANAGER_TYPE=jackett # or prowlarr
INDEXER_MANAGER_TYPE=jackett # or prowlarr or None if you want to disable it completely and use Zilean or Torrentio
INDEXER_MANAGER_URL=http://127.0.0.1:9117
INDEXER_MANAGER_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
INDEXER_MANAGER_TIMEOUT=60 # maximum time to obtain search results from indexer manager in seconds
INDEXER_MANAGER_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]'
GET_TORRENT_TIMEOUT=5 # maximum time to obtain the torrent info hash in seconds
ZILEAN_URL=None # for DMM search - https://github.com/iPromKnight/zilean
ZILEAN_URL=None # for DMM search - https://github.com/iPromKnight/zilean - ex: http://127.0.0.1:8181
ZILEAN_TAKE_FIRST=500 # only change it if you know what it is
SCRAPE_TORRENTIO=False # scrape Torrentio
CUSTOM_HEADER_HTML=None # only set it if you know what it is
PROXY_DEBRID_STREAM=False # Proxy Debrid Streams (very useful to use your debrid service on multiple IPs at same time)
PROXY_DEBRID_STREAM_PASSWORD=CHANGE_ME # Secret password to enter on configuration page to prevent people from abusing your debrid stream proxy
PROXY_DEBRID_STREAM_MAX_CONNECTIONS=100 # IP-Based connection limit for the Debrid Stream Proxy
PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE=realdebrid # if you want your users who use the Debrid Stream Proxy not to have to specify Debrid information, but to use the default one instead
PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY=CHANGE_ME # if you want your users who use the Debrid Stream Proxy not to have to specify Debrid information, but to use the default one instead
TITLE_MATCH_CHECK=True # disable if you only use Jackett / Prowlarr / Torrentio and are sure you're only scraping good titles, for example (keep it True if Zilean is enabled)
CUSTOM_HEADER_HTML=None # only set it if you know what it is
comet/api/core.py
+1 -1
View File
@@ -51,7 +51,7 @@ async def configure(request: Request):
{
"request": request,
"CUSTOM_HEADER_HTML": settings.CUSTOM_HEADER_HTML
if settings.CUSTOM_HEADER_HTML and settings.CUSTOM_HEADER_HTML != "None"
if settings.CUSTOM_HEADER_HTML
else "",
"webConfig": web_config,
"proxyDebridStream": settings.PROXY_DEBRID_STREAM,
comet/api/stream.py
+124 -13
View File
@@ -4,15 +4,21 @@ import json
import time
import aiohttp
import httpx
import uuid
import orjson
from fastapi import APIRouter, Request
from fastapi.responses import RedirectResponse, StreamingResponse, FileResponse
from fastapi.responses import (
RedirectResponse,
StreamingResponse,
FileResponse,
Response,
)
from starlette.background import BackgroundTask
from RTN import Torrent, sort_torrents
from comet.debrid.manager import getDebrid
from comet.utils.general import (
get_language_emoji,
config_check,
get_debrid_extension,
get_indexer_manager,
@@ -22,7 +28,7 @@ from comet.utils.general import (
get_torrent_hash,
translate,
get_balanced_hashes,
format_title
format_title,
)
from comet.utils.logger import logger
from comet.utils.models import database, rtn, settings
@@ -67,14 +73,17 @@ async def stream(request: Request, b64config: str, type: str, id: str):
get_metadata = await get_metadata.json()
name = get_metadata["data"]["attributes"]["canonicalTitle"]
season = 1
year = None
else:
get_metadata = await session.get(
f"https://v3.sg.media-imdb.com/suggestion/a/{id}.json"
)
metadata = await get_metadata.json()
name = metadata["d"][
element = metadata["d"][
0 if metadata["d"][0]["l"] != "Summer Watch Guide" else 1
]["l"]
]
name = element["l"]
year = element["y"]
except Exception as e:
logger.warning(f"Exception while getting metadata for {id}: {e}")
@@ -175,6 +184,17 @@ async def stream(request: Request, b64config: str, type: str, id: str):
else:
logger.info(f"No cache found for {log_name} with user configuration")
if (
settings.PROXY_DEBRID_STREAM
and settings.PROXY_DEBRID_STREAM_PASSWORD
== config["debridStreamProxyPassword"]
and config["debridApiKey"] == ""
):
config["debridService"] = (
settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE
)
config["debridApiKey"] = settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY
debrid = getDebrid(session, config)
check_premium = await debrid.check_premium()
@@ -198,7 +218,7 @@ async def stream(request: Request, b64config: str, type: str, id: str):
search_indexer = len(config["indexers"]) != 0
torrents = []
tasks = []
if search_indexer:
if indexer_manager_type and search_indexer:
logger.info(
f"Start of {indexer_manager_type} search for {log_name} with indexers {config['indexers']}"
)
@@ -216,7 +236,9 @@ async def stream(request: Request, b64config: str, type: str, id: str):
for term in search_terms
)
else:
logger.info(f"No indexer selected by user for {log_name}")
logger.info(
f"No indexer {'manager ' if not indexer_manager_type else ' '}{'selected by user' if indexer_manager_type else 'defined'} for {log_name}"
)
if settings.ZILEAN_URL:
tasks.append(get_zilean(session, name, log_name, season, episode))
@@ -230,7 +252,27 @@ async def stream(request: Request, b64config: str, type: str, id: str):
torrents.append(result)
logger.info(
f"{len(torrents)} torrents found for {log_name} with {indexer_manager_type}{' and Zilean' if settings.ZILEAN_URL else ''}{' and Torrentio' if settings.SCRAPE_TORRENTIO else ''}"
f"{len(torrents)} torrents found for {log_name}"
+ (
" with "
+ ", ".join(
part
for part in [
indexer_manager_type,
"Zilean" if settings.ZILEAN_URL else None,
"Torrentio" if settings.SCRAPE_TORRENTIO else None,
]
if part
)
if any(
[
indexer_manager_type,
settings.ZILEAN_URL,
settings.SCRAPE_TORRENTIO,
]
)
else ""
)
)
if len(torrents) == 0:
@@ -246,7 +288,7 @@ async def stream(request: Request, b64config: str, type: str, id: str):
tasks = []
for chunk in chunks:
tasks.append(filter(chunk, name))
tasks.append(filter(chunk, name, year))
filtered_torrents = await asyncio.gather(*tasks)
index_less = 0
@@ -333,7 +375,7 @@ async def stream(request: Request, b64config: str, type: str, id: str):
json_data = json.dumps(sorted_ranked_files).replace("'", "''")
await database.execute(
f"INSERT OR IGNORE INTO cache (cacheKey, results, timestamp) VALUES (:cache_key, :json_data, :timestamp)",
"INSERT OR IGNORE INTO cache (cacheKey, results, timestamp) VALUES (:cache_key, :json_data, :timestamp)",
{"cache_key": cache_key, "json_data": json_data, "timestamp": time.time()},
)
logger.info(f"Results have been cached for {log_name}")
@@ -381,6 +423,27 @@ async def playback(b64config: str, hash: str, index: str):
return RedirectResponse("https://stremio.fast", status_code=302)
class CustomORJSONResponse(Response):
media_type = "application/json"
def render(self, content) -> bytes:
assert orjson is not None, "orjson must be installed"
return orjson.dumps(content, option=orjson.OPT_INDENT_2)
@streams.get("/active-connections", response_class=CustomORJSONResponse)
async def active_connections(request: Request, password: str):
if password != settings.DASHBOARD_ADMIN_PASSWORD:
return "Invalid Password"
active_connections = await database.fetch_all("SELECT * FROM active_connections")
return {
"total_connections": len(active_connections),
"active_connections": active_connections,
}
@streams.get("/{b64config}/playback/{hash}/{index}")
async def playback(request: Request, b64config: str, hash: str, index: str):
config = config_check(b64config)
@@ -408,6 +471,19 @@ async def playback(request: Request, b64config: str, hash: str, index: str):
)
if not download_link:
if (
settings.PROXY_DEBRID_STREAM
and settings.PROXY_DEBRID_STREAM_PASSWORD
== config["debridStreamProxyPassword"]
and config["debridApiKey"] == ""
):
config["debridService"] = (
settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE
)
config["debridApiKey"] = (
settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY
)
debrid = getDebrid(session, config)
download_link = await debrid.generate_download_link(hash, index)
if not download_link:
@@ -430,10 +506,28 @@ async def playback(request: Request, b64config: str, hash: str, index: str):
and settings.PROXY_DEBRID_STREAM_PASSWORD
== config["debridStreamProxyPassword"]
):
active_ip_connections = await database.fetch_all(
"SELECT ip, COUNT(*) as connections FROM active_connections GROUP BY ip"
)
ip = (
request.headers["cf-connecting-ip"]
if "cf-connecting-ip" in request.headers
else request.client.host
)
if any(
connection["ip"] == ip
and connection["connections"]
>= settings.PROXY_DEBRID_STREAM_MAX_CONNECTIONS
for connection in active_ip_connections
):
return FileResponse("comet/assets/proxylimit.mp4")
proxy = None
class Streamer:
def __init__(self):
def __init__(self, id: str):
self.id = id
self.client = httpx.AsyncClient(proxy=proxy)
self.response = None
@@ -445,6 +539,10 @@ async def playback(request: Request, b64config: str, hash: str, index: str):
yield chunk
async def close(self):
await database.execute(
f"DELETE FROM active_connections WHERE id = '{self.id}'"
)
if self.response is not None:
await self.response.aclose()
if self.client is not None:
@@ -456,14 +554,27 @@ async def playback(request: Request, b64config: str, hash: str, index: str):
download_link, headers={"Range": range_header}
)
if response.status == 503 and config["debridService"] == "alldebrid":
proxy = settings.DEBRID_PROXY_URL # proxy is not needed to proxy realdebrid stream
proxy = (
settings.DEBRID_PROXY_URL
) # proxy is not needed to proxy realdebrid stream
response = await session.head(
download_link, headers={"Range": range_header}, proxy=proxy
)
if response.status == 206:
streamer = Streamer()
id = str(uuid.uuid4())
await database.execute(
"INSERT OR REPLACE INTO active_connections (id, ip, content, timestamp) VALUES (:id, :ip, :content, :timestamp)",
{
"id": id,
"ip": ip,
"content": str(response.url),
"timestamp": current_time,
},
)
streamer = Streamer(id)
return StreamingResponse(
streamer.stream_content({"Range": range_header}),
comet/assets/proxylimit.mp4
BIN
View File
Binary file not shown.
comet/main.py
+20 -6
View File
@@ -102,6 +102,7 @@ config = uvicorn.Config(
app,
host=settings.FASTAPI_HOST,
port=settings.FASTAPI_PORT,
proxy_headers=True,
workers=settings.FASTAPI_WORKERS,
log_config=None,
)
@@ -113,25 +114,38 @@ def start_log():
"COMET",
f"Server started on http://{settings.FASTAPI_HOST}:{settings.FASTAPI_PORT} - {settings.FASTAPI_WORKERS} workers",
)
logger.log(
"COMET", f"Dashboard Admin Password: {settings.DASHBOARD_ADMIN_PASSWORD}"
)
logger.log(
"COMET", f"Database: {settings.DATABASE_PATH} - TTL: {settings.CACHE_TTL}s"
)
logger.log("COMET", f"Debrid Proxy: {settings.DEBRID_PROXY_URL}")
logger.log(
"COMET",
f"Indexer Manager: {settings.INDEXER_MANAGER_TYPE}|{settings.INDEXER_MANAGER_URL} - Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s",
)
if settings.INDEXER_MANAGER_TYPE:
logger.log(
"COMET",
f"Indexer Manager: {settings.INDEXER_MANAGER_TYPE}|{settings.INDEXER_MANAGER_URL} - Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s",
)
else:
logger.log("COMET", "Indexer Manager: False")
logger.log("COMET", f"Indexers: {', '.join(settings.INDEXER_MANAGER_INDEXERS)}")
logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s")
if settings.ZILEAN_URL:
logger.log(
"COMET",
f"Zilean: {settings.ZILEAN_URL} - Take first: {settings.ZILEAN_TAKE_FIRST}",
)
else:
logger.log("COMET", "Zilean: Disabled")
logger.log("COMET", "Zilean: False")
logger.log("COMET", f"Torrentio Scraper: {bool(settings.SCRAPE_TORRENTIO)}")
logger.log("COMET", f"Debrid Stream Proxy: {bool(settings.PROXY_DEBRID_STREAM)}")
logger.log(
"COMET",
f"Debrid Stream Proxy: {bool(settings.PROXY_DEBRID_STREAM)} - Password: {settings.PROXY_DEBRID_STREAM_PASSWORD} - Max Connections: {settings.PROXY_DEBRID_STREAM_MAX_CONNECTIONS} - Default Debrid Service: {settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE} - Default Debrid API Key: {settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY}",
)
logger.log("COMET", f"Title Match Check: {bool(settings.TITLE_MATCH_CHECK)}")
logger.log("COMET", f"Custom Header HTML: {bool(settings.CUSTOM_HEADER_HTML)}")
comet/utils/db.py
+4
View File
@@ -18,6 +18,10 @@ async def setup_database():
await database.execute(
"CREATE TABLE IF NOT EXISTS download_links (debrid_key TEXT, hash TEXT, `index` TEXT, link TEXT, timestamp INTEGER, PRIMARY KEY (debrid_key, hash, `index`))"
)
await database.execute("DROP TABLE IF EXISTS active_connections")
await database.execute(
"CREATE TABLE IF NOT EXISTS active_connections (id TEXT PRIMARY KEY, ip TEXT, content TEXT, timestamp INTEGER)"
)
except Exception as e:
logger.error(f"Error setting up the database: {e}")
comet/utils/general.py
+12 -6
View File
@@ -419,7 +419,7 @@ async def get_torrentio(log_name: str, type: str, full_id: str):
return results
async def filter(torrents: list, name: str):
async def filter(torrents: list, name: str, year: int):
results = []
for torrent in torrents:
index = torrent[0]
@@ -428,11 +428,16 @@ async def filter(torrents: list, name: str):
if "\n" in title: # Torrentio title parsing
title = title.split("\n")[1]
if title_match(name, parse(title).parsed_title):
results.append((index, True))
parsed = parse(title)
if not title_match(name, parsed.parsed_title):
results.append((index, False))
continue
results.append((index, False))
if year and parsed.year != 0 and year != parsed.year:
results.append((index, False))
continue
results.append((index, True))
return results
@@ -565,7 +570,6 @@ def format_metadata(data: dict):
def format_title(data: dict, config: dict):
title = ""
logger.info(config)
if "Title" in config["resultFormat"] or "All" in config["resultFormat"]:
title += f"{data['title']}\n"
if "Metadata" in config["resultFormat"] or "All" in config["resultFormat"]:
@@ -581,7 +585,9 @@ def format_title(data: dict, config: dict):
formatted_languages = (
"/".join(get_language_emoji(language) for language in languages)
if languages
else get_language_emoji("multi_audio") if data["is_multi_audio"] else None
else get_language_emoji("multi_audio")
if data["is_multi_audio"]
else None
)
languages_str = "\n" + formatted_languages if formatted_languages else ""
title += f"{languages_str}"
comet/utils/models.py
+28 -4
View File
@@ -1,4 +1,6 @@
import os
import random
import string
from typing import List, Optional
from databases import Database
@@ -15,12 +17,13 @@ class AppSettings(BaseSettings):
FASTAPI_HOST: Optional[str] = "0.0.0.0"
FASTAPI_PORT: Optional[int] = 8000
FASTAPI_WORKERS: Optional[int] = 2 * (os.cpu_count() or 1)
DASHBOARD_ADMIN_PASSWORD: Optional[str] = None
DATABASE_PATH: Optional[str] = "data/comet.db"
CACHE_TTL: Optional[int] = 86400
DEBRID_PROXY_URL: Optional[str] = None
INDEXER_MANAGER_TYPE: str = "jackett"
INDEXER_MANAGER_URL: str = "http://127.0.0.1:9117"
INDEXER_MANAGER_API_KEY: str
INDEXER_MANAGER_TYPE: Optional[str] = "jackett"
INDEXER_MANAGER_URL: Optional[str] = "http://127.0.0.1:9117"
INDEXER_MANAGER_API_KEY: Optional[str] = None
INDEXER_MANAGER_TIMEOUT: Optional[int] = 30
INDEXER_MANAGER_INDEXERS: List[str] = ["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]
GET_TORRENT_TIMEOUT: Optional[int] = 5
@@ -29,9 +32,30 @@ class AppSettings(BaseSettings):
SCRAPE_TORRENTIO: Optional[bool] = False
CUSTOM_HEADER_HTML: Optional[str] = None
PROXY_DEBRID_STREAM: Optional[bool] = False
PROXY_DEBRID_STREAM_PASSWORD: Optional[str] = "CHANGE_ME"
PROXY_DEBRID_STREAM_PASSWORD: Optional[str] = None
PROXY_DEBRID_STREAM_MAX_CONNECTIONS: Optional[int] = 100
PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE: Optional[str] = None
PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY: Optional[str] = None
TITLE_MATCH_CHECK: Optional[bool] = True
@field_validator("DASHBOARD_ADMIN_PASSWORD")
def set_dashboard_admin_password(cls, v, values):
if v is None:
return "".join(random.choices(string.ascii_letters + string.digits, k=16))
return v
@field_validator("INDEXER_MANAGER_TYPE")
def set_indexer_manager_type(cls, v, values):
if v == "None":
return None
return v
@field_validator("PROXY_DEBRID_STREAM_PASSWORD")
def set_debrid_stream_proxy_password(cls, v, values):
if v is None:
return "".join(random.choices(string.ascii_letters + string.digits, k=16))
return v
settings = AppSettings()
poetry.lock
Generated
+437 -892
View File
File diff suppressed because it is too large Load Diff
pyproject.toml
+1
View File
@@ -21,6 +21,7 @@ pydantic-settings = "*"
bencode-py = "*"
httpx = "*"
curl-cffi = "*"
orjson = "*"
[tool.poetry.group.dev.dependencies]
isort = "*"