mirror of
https://github.com/g0ldyy/comet.git
synced 2026-01-12 01:16:12 +01:00
Merge rewrite into main by overwriting files
This commit is contained in:
Goldy
+12
-8
@@ -2,22 +2,27 @@ ADDON_ID=stremio.comet.fast # for Stremio
|
||||
ADDON_NAME=Comet # for Stremio
|
||||
FASTAPI_HOST=0.0.0.0
|
||||
FASTAPI_PORT=8000
|
||||
FASTAPI_WORKERS=1 # remove to destroy CPU -> max performances :)
|
||||
FASTAPI_WORKERS=1
|
||||
USE_GUNICORN=True # will use uvicorn if False or if on Windows
|
||||
DASHBOARD_ADMIN_PASSWORD=CHANGE_ME # The password to access the dashboard with active connections and soon more...
|
||||
DATABASE_TYPE=sqlite # or postgresql if you know what you're doing
|
||||
DATABASE_TYPE=sqlite # or postgresql if you're making a Comet cluster
|
||||
DATABASE_URL=username:password@hostname:port # to connect to PostgreSQL
|
||||
DATABASE_PATH=data/comet.db # only change it if you know what it is - folders in path must exist - ignored if PostgreSQL used
|
||||
CACHE_TTL=86400 # cache duration in seconds
|
||||
METADATA_CACHE_TTL=2592000 # metadata cache duration in seconds (30 days by default)
|
||||
TORRENT_CACHE_TTL=1296000 # torrent cache duration in seconds (15 days by default)
|
||||
DEBRID_CACHE_TTL=86400 # debrid availability cache duration in seconds (1 day by default)
|
||||
DEBRID_PROXY_URL=http://127.0.0.1:1080 # https://github.com/cmj2002/warp-docker to bypass Debrid Services and Torrentio server IP blacklist
|
||||
INDEXER_MANAGER_TYPE=None # jackett or prowlarr or None if you want to disable it completely and use Zilean or Torrentio
|
||||
INDEXER_MANAGER_TYPE=none # jackett or prowlarr or none if you want to disable it completely and use Zilean or Torrentio
|
||||
INDEXER_MANAGER_URL=http://127.0.0.1:9117
|
||||
INDEXER_MANAGER_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
||||
INDEXER_MANAGER_TIMEOUT=60 # maximum time to obtain search results from indexer manager in seconds
|
||||
INDEXER_MANAGER_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' # for jackett, get the names from https://github.com/Jackett/Jackett/tree/master/src/Jackett.Common/Definitions - for prowlarr you can write them like on the web dashboard
|
||||
GET_TORRENT_TIMEOUT=5 # maximum time to obtain the torrent info hash in seconds
|
||||
ZILEAN_URL=None # for DMM search - https://github.com/iPromKnight/zilean - ex: http://127.0.0.1:8181
|
||||
ZILEAN_TAKE_FIRST=500 # only change it if you know what it is
|
||||
DOWNLOAD_TORRENT_FILES=False # set to True to enable torrent file retrieval instead of using only magnet link info (infohash and sources, ensuring file index is included in results for Jackett and Prowlarr torrents)
|
||||
SCRAPE_ZILEAN=False # scrape Zilean/DMM
|
||||
ZILEAN_URL=https://zilean.elfhosted.com # for DMM search - https://github.com/iPromKnight/zilean - ex: http://127.0.0.1:8181
|
||||
SCRAPE_TORRENTIO=False # scrape Torrentio
|
||||
TORRENTIO_URL=https://torrentio.strem.fun # or https://knightcrawler.elfhosted.com if you prefer to scrape the ElfHosted KnightCrawler instance
|
||||
SCRAPE_MEDIAFUSION=False # scrape MediaFusion - has better results for Indian content
|
||||
MEDIAFUSION_URL=https://mediafusion.elfhosted.com # Allows you to scrape custom instances of MediaFusion
|
||||
PROXY_DEBRID_STREAM=False # Proxy Debrid Streams (very useful to use your debrid service on multiple IPs at same time)
|
||||
@@ -25,7 +30,6 @@ PROXY_DEBRID_STREAM_PASSWORD=CHANGE_ME # Secret password to enter on configurati
|
||||
PROXY_DEBRID_STREAM_MAX_CONNECTIONS=-1 # IP-Based connection limit for the Debrid Stream Proxy (-1 = disabled)
|
||||
PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE=realdebrid # if you want your users who use the Debrid Stream Proxy not to have to specify Debrid information, but to use the default one instead
|
||||
PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY=CHANGE_ME # if you want your users who use the Debrid Stream Proxy not to have to specify Debrid information, but to use the default one instead
|
||||
TITLE_MATCH_CHECK=True # disable if you only use Torrentio / MediaFusion and are sure you're only scraping good titles, for example (keep it True if Zilean is enabled)
|
||||
REMOVE_ADULT_CONTENT=False # detect and remove adult content
|
||||
STREMTHRU_DEFAULT_URL=None # if you want your users to use StremThru without having to specify it
|
||||
CUSTOM_HEADER_HTML=None # only set it if you know what it is
|
||||
STREMTHRU_URL=https://stremthru.13377001.xyz # StremThru acts as a proxy between Comet and debrid services to support them all, so you must have it
|
||||
@@ -4,8 +4,7 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
# release:
|
||||
# types: [created]
|
||||
- rewrite
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
@@ -42,6 +41,19 @@ jobs:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
|
||||
|
||||
- name: Get branch name
|
||||
id: branch-name
|
||||
run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Set Docker tags
|
||||
id: docker_tags
|
||||
run: |
|
||||
if [ "${{ steps.branch-name.outputs.branch }}" = "main" ]; then
|
||||
echo "tags=ghcr.io/g0ldyy/comet:latest,docker.io/g0ldyy/comet:latest" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tags=ghcr.io/g0ldyy/comet:${{ steps.branch-name.outputs.branch }},docker.io/g0ldyy/comet:${{ steps.branch-name.outputs.branch }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5.3.0
|
||||
with:
|
||||
@@ -51,6 +63,4 @@ jobs:
|
||||
push: true
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
tags: |
|
||||
ghcr.io/g0ldyy/comet:latest
|
||||
docker.io/g0ldyy/comet:latest
|
||||
tags: ${{ steps.docker_tags.outputs.tags }}
|
||||
+13
-3
@@ -5,8 +5,6 @@
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
logs/
|
||||
.vscode/
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
@@ -99,12 +97,18 @@ ipython_config.py
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# UV
|
||||
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
uv.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
poetry.lock
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
@@ -165,3 +169,9 @@ cython_debug/
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
# Ruff stuff:
|
||||
.ruff_cache/
|
||||
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
@@ -1,29 +1,5 @@
|
||||
# Changelog
|
||||
|
||||
## [1.53.0](https://github.com/g0ldyy/comet/compare/v1.52.0...v1.53.0) (2025-02-07)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **debrid/stremthru:** forward client ip ([982c398](https://github.com/g0ldyy/comet/commit/982c398774480badbb60d7214a91c3a62260fee6))
|
||||
* **debrid/stremthru:** match behavior with mediafusion ([24611a2](https://github.com/g0ldyy/comet/commit/24611a2940dec82cd3cd09aab757cef06a9beb8a))
|
||||
* **debrid/stremthru:** pass stremio video id for magnet cache check ([786a298](https://github.com/g0ldyy/comet/commit/786a2985ca3ef2463b65c684bc8cef6a86ce772f))
|
||||
* **debrid/stremthru:** use default url only for specific services ([f0b029d](https://github.com/g0ldyy/comet/commit/f0b029d7933bc85a7ca794c3edea1afccc8d4f28))
|
||||
* **debrid:** add support for stremthru ([64ba2b3](https://github.com/g0ldyy/comet/commit/64ba2b39fb0faab69e16e8755f024eb82c14f888))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **debrid/stremthru:** handle missing file index ([32ae49e](https://github.com/g0ldyy/comet/commit/32ae49ed1dd7ef0ab52ec8f6247250b3a8e888c7))
|
||||
* direct torrent results ([dea8a90](https://github.com/g0ldyy/comet/commit/dea8a9041cb54245e2fdadd9d914c0285efebc56))
|
||||
|
||||
## [1.52.0](https://github.com/g0ldyy/comet/compare/v1.51.0...v1.52.0) (2025-01-03)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* revert random manifest id ([f792529](https://github.com/g0ldyy/comet/commit/f7925298993e904a635041248079cf578d602818))
|
||||
|
||||
## [1.51.0](https://github.com/g0ldyy/comet/compare/v1.50.1...v1.51.0) (2024-11-28)
|
||||
|
||||
|
||||
|
||||
+5
-18
@@ -1,4 +1,4 @@
|
||||
FROM python:3.11-alpine
|
||||
FROM ghcr.io/astral-sh/uv:python3.11-alpine
|
||||
LABEL name="Comet" \
|
||||
description="Stremio's fastest torrent/debrid search add-on." \
|
||||
url="https://github.com/g0ldyy/comet"
|
||||
@@ -7,23 +7,10 @@ WORKDIR /app
|
||||
|
||||
ARG DATABASE_PATH
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
POETRY_NO_INTERACTION=1 \
|
||||
POETRY_HOME="/usr/local" \
|
||||
FORCE_COLOR=1 \
|
||||
TERM=xterm-256color
|
||||
|
||||
# Fix python-alpine gcc
|
||||
RUN apk add --no-cache \
|
||||
gcc \
|
||||
musl-dev \
|
||||
libffi-dev \
|
||||
make
|
||||
|
||||
RUN pip install poetry
|
||||
COPY pyproject.toml .
|
||||
RUN poetry install --no-cache --no-root --without dev
|
||||
|
||||
RUN uv sync
|
||||
|
||||
COPY . .
|
||||
|
||||
ENTRYPOINT ["poetry", "run", "python", "-m", "comet.main"]
|
||||
ENTRYPOINT ["uv", "run", "python", "-m", "comet.main"]
|
||||
|
||||
@@ -16,13 +16,14 @@
|
||||
- Smart Torrent Ranking powered by [RTN](https://github.com/dreulavelle/rank-torrent-name)
|
||||
- Proxy support to bypass debrid restrictions
|
||||
- Real-Debrid, All-Debrid, Premiumize, TorBox and Debrid-Link supported
|
||||
- Direct Torrent supported (do not specify a Debrid API Key on the configuration page (webui) to activate it - it will use the cached results of other users using debrid service)
|
||||
- Direct Torrent supported
|
||||
- [Kitsu](https://kitsu.io/) support (anime)
|
||||
- Adult Content Filter
|
||||
- [StremThru](https://github.com/MunifTanjim/stremthru) support
|
||||
|
||||
# Installation
|
||||
To customize your Comet experience to suit your needs, please first take a look at all the [environment variables](https://github.com/g0ldyy/comet/blob/main/.env-sample)!
|
||||
|
||||
## ElfHosted
|
||||
A free, public Comet instance is available at https://comet.elfhosted.com
|
||||
|
||||
@@ -44,58 +45,17 @@ ElfHosted offer "one-click" [private Comet instances](https://elfhosted.com/app/
|
||||
```
|
||||
- Install dependencies
|
||||
```sh
|
||||
pip install poetry
|
||||
poetry install
|
||||
pip install uv
|
||||
uv sync
|
||||
````
|
||||
- Start Comet
|
||||
```sh
|
||||
poetry run python -m comet.main
|
||||
uv run python -m comet.main
|
||||
````
|
||||
|
||||
### With Docker
|
||||
- Simply run the Docker image after modifying the environment variables
|
||||
```sh
|
||||
docker run --name comet -p 8000:8000 -d \
|
||||
-e FASTAPI_HOST=0.0.0.0 \
|
||||
-e FASTAPI_PORT=8000 \
|
||||
-e FASTAPI_WORKERS=1 \
|
||||
-e CACHE_TTL=86400 \
|
||||
-e DEBRID_PROXY_URL=http://127.0.0.1:1080 \
|
||||
-e INDEXER_MANAGER_TYPE=jackett \
|
||||
-e INDEXER_MANAGER_URL=http://127.0.0.1:9117 \
|
||||
-e INDEXER_MANAGER_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \
|
||||
-e INDEXER_MANAGER_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' \
|
||||
-e INDEXER_MANAGER_TIMEOUT=30 \
|
||||
-e GET_TORRENT_TIMEOUT=5 \
|
||||
g0ldyy/comet
|
||||
```
|
||||
- To update your container
|
||||
|
||||
- Find your existing container name
|
||||
```sh
|
||||
docker ps
|
||||
```
|
||||
|
||||
- Stop your existing container
|
||||
```sh
|
||||
docker stop <CONTAINER_ID>
|
||||
```
|
||||
|
||||
- Remove your existing container
|
||||
```sh
|
||||
docker rm <CONTAINER_ID>
|
||||
```
|
||||
|
||||
- Pull the latest version from docker hub
|
||||
```sh
|
||||
docker pull g0ldyy/comet
|
||||
```
|
||||
|
||||
- Finally, re-run the docker run command
|
||||
|
||||
### With Docker Compose
|
||||
- Copy *compose.yaml* in a directory
|
||||
- Copy *env-sample* to *.env* in the same directory
|
||||
- Copy *deployment/docker-compose.yml* in a directory
|
||||
- Copy *.env-sample* to *.env* in the same directory and keep only the variables you wish to modify, also remove all comments
|
||||
- Pull the latest version from docker hub
|
||||
```sh
|
||||
docker compose pull
|
||||
@@ -105,8 +65,21 @@ ElfHosted offer "one-click" [private Comet instances](https://elfhosted.com/app/
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
## Debrid IP Blacklist
|
||||
To bypass Real-Debrid's (or AllDebrid) IP blacklist, start a cloudflare-warp container: https://github.com/cmj2002/warp-docker
|
||||
### Nginx Reverse Proxy
|
||||
If you want to serve Comet via a Nginx Reverse Proxy, here's the configuration you should use.
|
||||
```
|
||||
server {
|
||||
server_name example.com;
|
||||
|
||||
location / {
|
||||
proxy_pass http://localhost:8000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Web UI Showcase
|
||||
<img src="https://i.imgur.com/SaD365F.png" />
|
||||
|
||||
+56
-32
@@ -1,39 +1,32 @@
|
||||
import PTT
|
||||
import RTN
|
||||
import random
|
||||
import string
|
||||
import secrets
|
||||
import orjson
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import RedirectResponse
|
||||
from fastapi import APIRouter, Request, Depends, HTTPException
|
||||
from fastapi.responses import RedirectResponse, Response
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from fastapi.security import HTTPBasic, HTTPBasicCredentials
|
||||
|
||||
from comet.utils.models import settings
|
||||
from comet.utils.general import config_check, get_debrid_extension
|
||||
from comet.utils.models import settings, web_config, database
|
||||
from comet.utils.general import config_check
|
||||
from comet.debrid.manager import get_debrid_extension
|
||||
|
||||
templates = Jinja2Templates("comet/templates")
|
||||
main = APIRouter()
|
||||
security = HTTPBasic()
|
||||
|
||||
|
||||
@main.get("/", status_code=200)
|
||||
@main.get("/")
|
||||
async def root():
|
||||
return RedirectResponse("/configure")
|
||||
|
||||
|
||||
@main.get("/health", status_code=200)
|
||||
@main.get("/health")
|
||||
async def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
indexers = settings.INDEXER_MANAGER_INDEXERS
|
||||
languages = [language for language in PTT.parse.LANGUAGES_TRANSLATION_TABLE.values()]
|
||||
languages.insert(0, "Unknown")
|
||||
languages.insert(1, "Multi")
|
||||
web_config = {
|
||||
"indexers": [indexer.replace(" ", "_").lower() for indexer in indexers],
|
||||
"languages": languages,
|
||||
"resolutions": [resolution.value for resolution in RTN.models.Resolution],
|
||||
"resultFormat": ["Title", "Metadata", "Size", "Tracker", "Languages"],
|
||||
}
|
||||
|
||||
|
||||
@main.get("/configure")
|
||||
@main.get("/{b64config}/configure")
|
||||
async def configure(request: Request):
|
||||
@@ -45,25 +38,16 @@ async def configure(request: Request):
|
||||
if settings.CUSTOM_HEADER_HTML
|
||||
else "",
|
||||
"webConfig": web_config,
|
||||
"indexerManager": settings.INDEXER_MANAGER_TYPE,
|
||||
"proxyDebridStream": settings.PROXY_DEBRID_STREAM,
|
||||
"stremthruDefaultUrl": settings.STREMTHRU_DEFAULT_URL or "",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@main.get("/manifest.json")
|
||||
@main.get("/{b64config}/manifest.json")
|
||||
async def manifest(b64config: str = None):
|
||||
config = config_check(b64config)
|
||||
if not config:
|
||||
config = {"debridService": None}
|
||||
|
||||
debrid_extension = get_debrid_extension(config["debridService"], config["debridApiKey"])
|
||||
|
||||
return {
|
||||
"id": settings.ADDON_ID,
|
||||
"name": f"{settings.ADDON_NAME}{(' | ' + debrid_extension) if debrid_extension is not None else ''}",
|
||||
async def manifest(request: Request, b64config: str = None):
|
||||
base_manifest = {
|
||||
"id": f"{settings.ADDON_ID}.{''.join(random.choice(string.ascii_letters) for _ in range(4))}",
|
||||
"description": "Stremio's fastest torrent/debrid search add-on.",
|
||||
"version": "1.0.0",
|
||||
"catalogs": [],
|
||||
@@ -79,3 +63,43 @@ async def manifest(b64config: str = None):
|
||||
"background": "https://i.imgur.com/WwnXB3k.jpeg",
|
||||
"behaviorHints": {"configurable": True, "configurationRequired": False},
|
||||
}
|
||||
|
||||
config = config_check(b64config)
|
||||
if not config:
|
||||
base_manifest["name"] = "❌ | Comet"
|
||||
base_manifest["description"] = f"⚠️ OBSOLETE CONFIGURATION, PLEASE RE-CONFIGURE ON {request.url.scheme}://{request.url.netloc} ⚠️"
|
||||
return base_manifest
|
||||
|
||||
debrid_extension = get_debrid_extension(config["debridService"])
|
||||
base_manifest["name"] = f"{settings.ADDON_NAME}{(' | ' + debrid_extension) if debrid_extension is not None else ''}"
|
||||
|
||||
return base_manifest
|
||||
|
||||
|
||||
class CustomORJSONResponse(Response):
|
||||
media_type = "application/json"
|
||||
|
||||
def render(self, content) -> bytes:
|
||||
assert orjson is not None, "orjson must be installed"
|
||||
return orjson.dumps(content, option=orjson.OPT_INDENT_2)
|
||||
|
||||
|
||||
def verify_dashboard_auth(credentials: HTTPBasicCredentials = Depends(security)):
|
||||
is_correct = secrets.compare_digest(
|
||||
credentials.password, settings.DASHBOARD_ADMIN_PASSWORD
|
||||
)
|
||||
|
||||
if not is_correct:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail="Incorrect password",
|
||||
headers={"WWW-Authenticate": "Basic"},
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
@main.get("/dashboard", response_class=CustomORJSONResponse)
|
||||
async def dashboard(authenticated: bool = Depends(verify_dashboard_auth)):
|
||||
rows = await database.fetch_all("SELECT * FROM active_connections")
|
||||
return rows
|
||||
|
||||
+292
-596
@@ -1,480 +1,248 @@
|
||||
import asyncio
|
||||
import time
|
||||
import aiohttp
|
||||
import httpx
|
||||
import uuid
|
||||
import orjson
|
||||
import time
|
||||
import mediaflow_proxy.utils.http_utils
|
||||
|
||||
from fastapi import APIRouter, Request, BackgroundTasks
|
||||
from fastapi.responses import (
|
||||
RedirectResponse,
|
||||
StreamingResponse,
|
||||
FileResponse,
|
||||
Response,
|
||||
RedirectResponse,
|
||||
)
|
||||
from starlette.background import BackgroundTask
|
||||
from RTN import Torrent, sort_torrents
|
||||
|
||||
from comet.debrid.manager import getDebrid
|
||||
from comet.utils.general import (
|
||||
config_check,
|
||||
get_debrid_extension,
|
||||
get_indexer_manager,
|
||||
get_zilean,
|
||||
get_torrentio,
|
||||
get_mediafusion,
|
||||
filter,
|
||||
get_torrent_hash,
|
||||
translate,
|
||||
get_balanced_hashes,
|
||||
format_title,
|
||||
get_client_ip,
|
||||
get_aliases,
|
||||
add_torrent_to_cache,
|
||||
)
|
||||
from comet.utils.config import is_proxy_stream_authed, is_proxy_stream_enabled, prepare_debrid_config, should_skip_proxy_stream
|
||||
from comet.utils.models import settings, database, trackers
|
||||
from comet.utils.general import parse_media_id
|
||||
from comet.metadata.manager import MetadataScraper
|
||||
from comet.scrapers.manager import TorrentManager
|
||||
from comet.utils.general import config_check, format_title, get_client_ip
|
||||
from comet.debrid.manager import get_debrid_extension, get_debrid
|
||||
from comet.utils.streaming import custom_handle_stream_request
|
||||
from comet.utils.logger import logger
|
||||
from comet.utils.models import database, rtn, settings, trackers
|
||||
|
||||
streams = APIRouter()
|
||||
|
||||
|
||||
@streams.get("/stream/{type}/{id}.json")
|
||||
async def stream_noconfig(request: Request, type: str, id: str):
|
||||
return {
|
||||
"streams": [
|
||||
{
|
||||
"name": "[⚠️] Comet",
|
||||
"description": f"{request.url.scheme}://{request.url.netloc}/configure",
|
||||
"url": "https://comet.fast",
|
||||
}
|
||||
]
|
||||
}
|
||||
async def remove_ongoing_search_from_database(media_id: str):
|
||||
await database.execute(
|
||||
"DELETE FROM ongoing_searches WHERE media_id = :media_id",
|
||||
{"media_id": media_id},
|
||||
)
|
||||
|
||||
|
||||
@streams.get("/{b64config}/stream/{type}/{id}.json")
|
||||
async def is_first_search(media_id: str) -> bool:
|
||||
try:
|
||||
await database.execute(
|
||||
"INSERT INTO first_searches VALUES (:media_id, :timestamp)",
|
||||
{"media_id": media_id, "timestamp": time.time()},
|
||||
)
|
||||
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
async def background_scrape(
|
||||
torrent_manager: TorrentManager, media_id: str, debrid_service: str
|
||||
):
|
||||
try:
|
||||
async with aiohttp.ClientSession() as new_session:
|
||||
await torrent_manager.scrape_torrents(new_session)
|
||||
|
||||
if debrid_service != "torrent" and len(torrent_manager.torrents) > 0:
|
||||
await torrent_manager.get_and_cache_debrid_availability(new_session)
|
||||
|
||||
logger.log(
|
||||
"SCRAPER",
|
||||
"📥 Background scrape + availability check complete!",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.log("SCRAPER", f"❌ Background scrape + availability check failed: {e}")
|
||||
finally:
|
||||
await remove_ongoing_search_from_database(media_id)
|
||||
|
||||
|
||||
@streams.get("/stream/{media_type}/{media_id}.json")
|
||||
@streams.get("/{b64config}/stream/{media_type}/{media_id}.json")
|
||||
async def stream(
|
||||
request: Request,
|
||||
b64config: str,
|
||||
type: str,
|
||||
id: str,
|
||||
media_type: str,
|
||||
media_id: str,
|
||||
background_tasks: BackgroundTasks,
|
||||
b64config: str = None,
|
||||
):
|
||||
config = config_check(b64config)
|
||||
if not config:
|
||||
return {
|
||||
"streams": [
|
||||
{
|
||||
"name": "[⚠️] Comet",
|
||||
"description": "Invalid Comet config.",
|
||||
"name": "[❌] Comet",
|
||||
"description": f"⚠️ OBSOLETE CONFIGURATION, PLEASE RE-CONFIGURE ON {request.url.scheme}://{request.url.netloc} ⚠️",
|
||||
"url": "https://comet.fast",
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
ongoing_search = await database.fetch_one(
|
||||
"SELECT timestamp FROM ongoing_searches WHERE media_id = :media_id",
|
||||
{"media_id": media_id},
|
||||
)
|
||||
|
||||
if ongoing_search:
|
||||
return {
|
||||
"streams": [
|
||||
{
|
||||
"name": "[🔄] Comet",
|
||||
"description": "Search in progress, please try again in a few seconds...",
|
||||
"url": "https://comet.fast",
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
connector = aiohttp.TCPConnector(limit=0)
|
||||
async with aiohttp.ClientSession(
|
||||
connector=connector, raise_for_status=True
|
||||
) as session:
|
||||
full_id = id
|
||||
season = None
|
||||
episode = None
|
||||
if type == "series":
|
||||
info = id.split(":")
|
||||
id = info[0]
|
||||
season = int(info[1])
|
||||
episode = int(info[2])
|
||||
|
||||
year = None
|
||||
year_end = None
|
||||
try:
|
||||
kitsu = False
|
||||
if id == "kitsu":
|
||||
kitsu = True
|
||||
get_metadata = await session.get(
|
||||
f"https://kitsu.io/api/edge/anime/{season}"
|
||||
async with aiohttp.ClientSession(connector=connector) as session:
|
||||
metadata, aliases = await MetadataScraper(session).fetch_metadata_and_aliases(
|
||||
media_type, media_id
|
||||
)
|
||||
metadata = await get_metadata.json()
|
||||
name = metadata["data"]["attributes"]["canonicalTitle"]
|
||||
season = 1
|
||||
else:
|
||||
get_metadata = await session.get(
|
||||
f"https://v3.sg.media-imdb.com/suggestion/a/{id}.json"
|
||||
)
|
||||
metadata = await get_metadata.json()
|
||||
element = metadata["d"][
|
||||
0
|
||||
if metadata["d"][0]["id"]
|
||||
not in ["/imdbpicks/summer-watch-guide", "/emmys"]
|
||||
else 1
|
||||
]
|
||||
|
||||
for element in metadata["d"]:
|
||||
if "/" not in element["id"]:
|
||||
break
|
||||
|
||||
name = element["l"]
|
||||
year = element.get("y")
|
||||
|
||||
if "yr" in element:
|
||||
year_end = int(element["yr"].split("-")[1])
|
||||
except Exception as e:
|
||||
logger.warning(f"Exception while getting metadata for {id}: {e}")
|
||||
|
||||
if metadata is None:
|
||||
logger.log("SCRAPER", f"❌ Failed to fetch metadata for {media_id}")
|
||||
return {
|
||||
"streams": [
|
||||
{
|
||||
"name": "[⚠️] Comet",
|
||||
"description": f"Can't get metadata for {id}",
|
||||
"description": "Unable to get metadata.",
|
||||
"url": "https://comet.fast",
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
name = translate(name)
|
||||
log_name = name
|
||||
if type == "series":
|
||||
log_name = f"{name} S{season:02d}E{episode:02d}"
|
||||
title = metadata["title"]
|
||||
year = metadata["year"]
|
||||
year_end = metadata["year_end"]
|
||||
season = metadata["season"]
|
||||
episode = metadata["episode"]
|
||||
|
||||
prepare_debrid_config(config)
|
||||
log_title = f"({media_id}) {title}"
|
||||
if media_type == "series":
|
||||
log_title += f" S{season:02d}E{episode:02d}"
|
||||
|
||||
if config["debridApiKey"] == "":
|
||||
services = ["realdebrid", "alldebrid", "premiumize", "torbox", "debridlink", "stremthru"]
|
||||
debrid_emoji = "⬇️"
|
||||
else:
|
||||
services = [config["debridService"]]
|
||||
debrid_emoji = "⚡"
|
||||
logger.log("SCRAPER", f"🔍 Starting search for {log_title}")
|
||||
|
||||
results = []
|
||||
if (
|
||||
is_proxy_stream_enabled(config)
|
||||
and not is_proxy_stream_authed(config)
|
||||
):
|
||||
results.append(
|
||||
{
|
||||
"name": "[⚠️] Comet",
|
||||
"description": "Debrid Stream Proxy Password incorrect.\nStreams will not be proxied.",
|
||||
"url": "https://comet.fast",
|
||||
}
|
||||
)
|
||||
id, season, episode = parse_media_id(media_type, media_id)
|
||||
media_only_id = id
|
||||
|
||||
indexers = config["indexers"].copy()
|
||||
if settings.SCRAPE_TORRENTIO:
|
||||
indexers.append("torrentio")
|
||||
if settings.SCRAPE_MEDIAFUSION:
|
||||
indexers.append("mediafusion")
|
||||
if settings.ZILEAN_URL:
|
||||
indexers.append("dmm")
|
||||
indexers_json = orjson.dumps(indexers).decode("utf-8")
|
||||
|
||||
all_sorted_ranked_files = {}
|
||||
trackers_found = (
|
||||
set()
|
||||
) # we want to check that we have a cache for each of the user's trackers
|
||||
the_time = time.time()
|
||||
cache_ttl = settings.CACHE_TTL
|
||||
|
||||
for debrid_service in services:
|
||||
cached_results = await database.fetch_all(
|
||||
f"""
|
||||
SELECT info_hash, tracker, data
|
||||
FROM cache
|
||||
WHERE debridService = :debrid_service
|
||||
AND name = :name
|
||||
AND ((cast(:season as INTEGER) IS NULL AND season IS NULL) OR season = cast(:season as INTEGER))
|
||||
AND ((cast(:episode as INTEGER) IS NULL AND episode IS NULL) OR episode = cast(:episode as INTEGER))
|
||||
AND tracker IN (SELECT cast(value as TEXT) FROM {'json_array_elements_text' if settings.DATABASE_TYPE == 'postgresql' else 'json_each'}(:indexers))
|
||||
AND timestamp + :cache_ttl >= :current_time
|
||||
""",
|
||||
{
|
||||
"debrid_service": debrid_service,
|
||||
"name": name,
|
||||
"season": season,
|
||||
"episode": episode,
|
||||
"indexers": indexers_json,
|
||||
"cache_ttl": cache_ttl,
|
||||
"current_time": the_time,
|
||||
},
|
||||
)
|
||||
for result in cached_results:
|
||||
trackers_found.add(result["tracker"].lower())
|
||||
|
||||
hash = result["info_hash"]
|
||||
if "searched" in hash:
|
||||
continue
|
||||
|
||||
all_sorted_ranked_files[hash] = orjson.loads(result["data"])
|
||||
|
||||
if len(all_sorted_ranked_files) != 0 and set(indexers).issubset(trackers_found):
|
||||
debrid_extension = get_debrid_extension(
|
||||
debrid_service, config["debridApiKey"]
|
||||
)
|
||||
balanced_hashes = get_balanced_hashes(all_sorted_ranked_files, config)
|
||||
|
||||
for resolution in balanced_hashes:
|
||||
for hash in balanced_hashes[resolution]:
|
||||
data = all_sorted_ranked_files[hash]["data"]
|
||||
the_stream = {
|
||||
"name": f"[{debrid_extension}{debrid_emoji}] Comet {data['resolution']}",
|
||||
"description": format_title(data, config),
|
||||
"torrentTitle": (
|
||||
data["torrent_title"] if "torrent_title" in data else None
|
||||
),
|
||||
"torrentSize": (
|
||||
data["torrent_size"] if "torrent_size" in data else None
|
||||
),
|
||||
"behaviorHints": {
|
||||
"filename": data["raw_title"],
|
||||
"bingeGroup": "comet|" + hash,
|
||||
},
|
||||
}
|
||||
|
||||
if config["debridApiKey"] != "":
|
||||
the_stream["url"] = (
|
||||
f"{request.url.scheme}://{request.url.netloc}/{b64config}/playback/{hash}/{data['index']}"
|
||||
)
|
||||
else:
|
||||
the_stream["infoHash"] = hash
|
||||
index = str(data["index"])
|
||||
the_stream["fileIdx"] = (
|
||||
1 if "|" in index else int(index)
|
||||
) # 1 because for Premiumize it's impossible to get the file index
|
||||
the_stream["sources"] = trackers
|
||||
|
||||
results.append(the_stream)
|
||||
|
||||
logger.info(
|
||||
f"{len(all_sorted_ranked_files)} cached results found for {log_name}"
|
||||
)
|
||||
|
||||
return {"streams": results}
|
||||
|
||||
if config["debridApiKey"] == "":
|
||||
return {
|
||||
"streams": [
|
||||
{
|
||||
"name": "[⚠️] Comet",
|
||||
"description": "No cache found for Direct Torrenting.",
|
||||
"url": "https://comet.fast",
|
||||
}
|
||||
]
|
||||
}
|
||||
logger.info(f"No cache found for {log_name} with user configuration")
|
||||
|
||||
debrid = getDebrid(session, config, get_client_ip(request))
|
||||
|
||||
check_premium = await debrid.check_premium()
|
||||
if not check_premium:
|
||||
additional_info = ""
|
||||
if config["debridService"] == "alldebrid":
|
||||
additional_info = "\nCheck your email!"
|
||||
|
||||
return {
|
||||
"streams": [
|
||||
{
|
||||
"name": "[⚠️] Comet",
|
||||
"description": f"Invalid {config['debridService']} account.{additional_info}",
|
||||
"url": "https://comet.fast",
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
indexer_manager_type = settings.INDEXER_MANAGER_TYPE
|
||||
|
||||
search_indexer = len(config["indexers"]) != 0
|
||||
torrents = []
|
||||
tasks = []
|
||||
if indexer_manager_type and search_indexer:
|
||||
logger.info(
|
||||
f"Start of {indexer_manager_type} search for {log_name} with indexers {config['indexers']}"
|
||||
)
|
||||
|
||||
search_terms = [name]
|
||||
if type == "series":
|
||||
if not kitsu:
|
||||
search_terms.append(f"{name} S{season:02d}E{episode:02d}")
|
||||
else:
|
||||
search_terms.append(f"{name} {episode}")
|
||||
tasks.extend(
|
||||
get_indexer_manager(
|
||||
session, indexer_manager_type, config["indexers"], term
|
||||
)
|
||||
for term in search_terms
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"No indexer {'manager ' if not indexer_manager_type else ''}{'selected by user' if indexer_manager_type else 'defined'} for {log_name}"
|
||||
)
|
||||
|
||||
if settings.ZILEAN_URL:
|
||||
tasks.append(get_zilean(session, name, log_name, season, episode))
|
||||
|
||||
if settings.SCRAPE_TORRENTIO:
|
||||
tasks.append(get_torrentio(log_name, type, full_id))
|
||||
|
||||
if settings.SCRAPE_MEDIAFUSION:
|
||||
tasks.append(get_mediafusion(log_name, type, full_id))
|
||||
|
||||
search_response = await asyncio.gather(*tasks)
|
||||
for results in search_response:
|
||||
for result in results:
|
||||
torrents.append(result)
|
||||
|
||||
logger.info(
|
||||
f"{len(torrents)} unique torrents found for {log_name}"
|
||||
+ (
|
||||
" with "
|
||||
+ ", ".join(
|
||||
part
|
||||
for part in [
|
||||
indexer_manager_type,
|
||||
"Zilean" if settings.ZILEAN_URL else None,
|
||||
"Torrentio" if settings.SCRAPE_TORRENTIO else None,
|
||||
"MediaFusion" if settings.SCRAPE_MEDIAFUSION else None,
|
||||
]
|
||||
if part
|
||||
)
|
||||
if any(
|
||||
[
|
||||
indexer_manager_type,
|
||||
settings.ZILEAN_URL,
|
||||
settings.SCRAPE_TORRENTIO,
|
||||
settings.SCRAPE_MEDIAFUSION,
|
||||
]
|
||||
)
|
||||
else ""
|
||||
)
|
||||
)
|
||||
|
||||
if len(torrents) == 0:
|
||||
return {"streams": []}
|
||||
|
||||
if settings.TITLE_MATCH_CHECK:
|
||||
aliases = await get_aliases(
|
||||
session, "movies" if type == "movie" else "shows", id
|
||||
)
|
||||
|
||||
indexed_torrents = [(i, torrents[i]["Title"]) for i in range(len(torrents))]
|
||||
chunk_size = 50
|
||||
chunks = [
|
||||
indexed_torrents[i : i + chunk_size]
|
||||
for i in range(0, len(indexed_torrents), chunk_size)
|
||||
]
|
||||
|
||||
remove_adult_content = (
|
||||
settings.REMOVE_ADULT_CONTENT and config["removeTrash"]
|
||||
)
|
||||
tasks = []
|
||||
for chunk in chunks:
|
||||
tasks.append(
|
||||
filter(chunk, name, year, year_end, aliases, remove_adult_content)
|
||||
)
|
||||
|
||||
filtered_torrents = await asyncio.gather(*tasks)
|
||||
index_less = 0
|
||||
for result in filtered_torrents:
|
||||
for filtered in result:
|
||||
if not filtered[1]:
|
||||
del torrents[filtered[0] - index_less]
|
||||
index_less += 1
|
||||
continue
|
||||
|
||||
logger.info(
|
||||
f"{len(torrents)} torrents passed title match check for {log_name}"
|
||||
)
|
||||
|
||||
if len(torrents) == 0:
|
||||
return {"streams": []}
|
||||
|
||||
tasks = []
|
||||
for i in range(len(torrents)):
|
||||
tasks.append(get_torrent_hash(session, (i, torrents[i])))
|
||||
|
||||
torrent_hashes = await asyncio.gather(*tasks)
|
||||
index_less = 0
|
||||
for hash in torrent_hashes:
|
||||
if not hash[1]:
|
||||
del torrents[hash[0] - index_less]
|
||||
index_less += 1
|
||||
continue
|
||||
|
||||
torrents[hash[0] - index_less]["InfoHash"] = hash[1]
|
||||
|
||||
logger.info(f"{len(torrents)} info hashes found for {log_name}")
|
||||
|
||||
if len(torrents) == 0:
|
||||
return {"streams": []}
|
||||
|
||||
files = await debrid.get_files(
|
||||
list({hash[1] for hash in torrent_hashes if hash[1] is not None}),
|
||||
type,
|
||||
debrid_service = config["debridService"]
|
||||
torrent_manager = TorrentManager(
|
||||
debrid_service,
|
||||
config["debridApiKey"],
|
||||
get_client_ip(request),
|
||||
media_type,
|
||||
media_id,
|
||||
media_only_id,
|
||||
title,
|
||||
year,
|
||||
year_end,
|
||||
season,
|
||||
episode,
|
||||
kitsu,
|
||||
video_id=full_id,
|
||||
aliases,
|
||||
settings.REMOVE_ADULT_CONTENT and config["removeTrash"],
|
||||
)
|
||||
|
||||
ranked_files = set()
|
||||
torrents_by_hash = {torrent["InfoHash"]: torrent for torrent in torrents}
|
||||
for hash in files:
|
||||
try:
|
||||
ranked_file = rtn.rank(
|
||||
torrents_by_hash[hash]["Title"],
|
||||
hash,
|
||||
remove_trash=False, # user can choose if he wants to remove it
|
||||
await torrent_manager.get_cached_torrents()
|
||||
logger.log(
|
||||
"SCRAPER", f"📦 Found cached torrents: {len(torrent_manager.torrents)}"
|
||||
)
|
||||
|
||||
ranked_files.add(ranked_file)
|
||||
except:
|
||||
pass
|
||||
is_first = await is_first_search(media_id)
|
||||
has_cached_results = len(torrent_manager.torrents) > 0
|
||||
|
||||
sorted_ranked_files = sort_torrents(ranked_files)
|
||||
cached_results = []
|
||||
non_cached_results = []
|
||||
|
||||
len_sorted_ranked_files = len(sorted_ranked_files)
|
||||
logger.info(
|
||||
f"{len_sorted_ranked_files} cached files found on {config['debridService']} for {log_name}"
|
||||
if not has_cached_results:
|
||||
logger.log("SCRAPER", f"🔎 Starting new search for {log_title}")
|
||||
await database.execute(
|
||||
f"INSERT {'OR IGNORE ' if settings.DATABASE_TYPE == 'sqlite' else ''}INTO ongoing_searches VALUES (:media_id, :timestamp){' ON CONFLICT DO NOTHING' if settings.DATABASE_TYPE == 'postgresql' else ''}",
|
||||
{"media_id": media_id, "timestamp": time.time()},
|
||||
)
|
||||
background_tasks.add_task(remove_ongoing_search_from_database, media_id)
|
||||
|
||||
if len_sorted_ranked_files == 0:
|
||||
return {"streams": []}
|
||||
|
||||
sorted_ranked_files = {
|
||||
key: (value.model_dump() if isinstance(value, Torrent) else value)
|
||||
for key, value in sorted_ranked_files.items()
|
||||
}
|
||||
for hash in sorted_ranked_files: # needed for caching
|
||||
sorted_ranked_files[hash]["data"]["title"] = files[hash]["title"]
|
||||
sorted_ranked_files[hash]["data"]["torrent_title"] = torrents_by_hash[hash][
|
||||
"Title"
|
||||
]
|
||||
sorted_ranked_files[hash]["data"]["tracker"] = torrents_by_hash[hash][
|
||||
"Tracker"
|
||||
]
|
||||
torrent_size = torrents_by_hash[hash]["Size"]
|
||||
sorted_ranked_files[hash]["data"]["size"] = (
|
||||
files[hash]["size"]
|
||||
await torrent_manager.scrape_torrents(session)
|
||||
logger.log(
|
||||
"SCRAPER",
|
||||
f"📥 Scraped torrents: {len(torrent_manager.torrents)}",
|
||||
)
|
||||
sorted_ranked_files[hash]["data"]["torrent_size"] = (
|
||||
torrent_size if torrent_size else files[hash]["size"]
|
||||
elif is_first:
|
||||
logger.log(
|
||||
"SCRAPER",
|
||||
f"🔄 Starting background scrape + availability check for {log_title}",
|
||||
)
|
||||
await database.execute(
|
||||
f"INSERT {'OR IGNORE ' if settings.DATABASE_TYPE == 'sqlite' else ''}INTO ongoing_searches VALUES (:media_id, :timestamp){' ON CONFLICT DO NOTHING' if settings.DATABASE_TYPE == 'postgresql' else ''}",
|
||||
{"media_id": media_id, "timestamp": time.time()},
|
||||
)
|
||||
sorted_ranked_files[hash]["data"]["index"] = files[hash]["index"]
|
||||
|
||||
background_tasks.add_task(
|
||||
add_torrent_to_cache, config, name, season, episode, sorted_ranked_files
|
||||
background_scrape, torrent_manager, media_id, debrid_service
|
||||
)
|
||||
|
||||
logger.info(f"Results have been cached for {log_name}")
|
||||
cached_results.append(
|
||||
{
|
||||
"name": "[🔄] Comet",
|
||||
"description": "First search for this media - More results will be available in a few seconds...",
|
||||
"url": "https://comet.fast",
|
||||
}
|
||||
)
|
||||
|
||||
debrid_extension = get_debrid_extension(config["debridService"], config["debridApiKey"])
|
||||
|
||||
balanced_hashes = get_balanced_hashes(sorted_ranked_files, config)
|
||||
|
||||
results = []
|
||||
await torrent_manager.get_cached_availability()
|
||||
if (
|
||||
is_proxy_stream_enabled(config)
|
||||
and not is_proxy_stream_authed(config)
|
||||
(
|
||||
not has_cached_results
|
||||
or sum(
|
||||
1
|
||||
for torrent in torrent_manager.torrents.values()
|
||||
if torrent["cached"]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
and len(torrent_manager.torrents) > 0
|
||||
and debrid_service != "torrent"
|
||||
):
|
||||
results.append(
|
||||
logger.log("SCRAPER", "🔄 Checking availability on debrid service...")
|
||||
await torrent_manager.get_and_cache_debrid_availability(session)
|
||||
|
||||
if debrid_service != "torrent":
|
||||
cached_count = sum(
|
||||
1 for torrent in torrent_manager.torrents.values() if torrent["cached"]
|
||||
)
|
||||
|
||||
logger.log(
|
||||
"SCRAPER",
|
||||
f"💾 Available cached torrents on {debrid_service}: {cached_count}/{len(torrent_manager.torrents)}",
|
||||
)
|
||||
|
||||
initial_torrent_count = len(torrent_manager.torrents)
|
||||
|
||||
torrent_manager.rank_torrents(
|
||||
config["rtnSettings"],
|
||||
config["rtnRanking"],
|
||||
config["maxResultsPerResolution"],
|
||||
config["maxSize"],
|
||||
config["cachedOnly"],
|
||||
config["removeTrash"],
|
||||
)
|
||||
logger.log(
|
||||
"SCRAPER",
|
||||
f"⚖️ Torrents after RTN filtering: {len(torrent_manager.ranked_torrents)}/{initial_torrent_count}",
|
||||
)
|
||||
|
||||
debrid_extension = get_debrid_extension(debrid_service)
|
||||
|
||||
if (
|
||||
config["debridStreamProxyPassword"] != ""
|
||||
and settings.PROXY_DEBRID_STREAM
|
||||
and settings.PROXY_DEBRID_STREAM_PASSWORD
|
||||
!= config["debridStreamProxyPassword"]
|
||||
):
|
||||
cached_results.append(
|
||||
{
|
||||
"name": "[⚠️] Comet",
|
||||
"description": "Debrid Stream Proxy Password incorrect.\nStreams will not be proxied.",
|
||||
@@ -482,209 +250,137 @@ async def stream(
|
||||
}
|
||||
)
|
||||
|
||||
for resolution in balanced_hashes:
|
||||
for hash in balanced_hashes[resolution]:
|
||||
data = sorted_ranked_files[hash]["data"]
|
||||
index = data['index']
|
||||
if index == -1:
|
||||
index = data['title']
|
||||
url = f"{request.url.scheme}://{request.url.netloc}/{b64config}/playback/{hash}/{index}"
|
||||
results.append(
|
||||
{
|
||||
"name": f"[{debrid_extension}⚡] Comet {data['resolution']}",
|
||||
"description": format_title(data, config),
|
||||
"torrentTitle": data["torrent_title"],
|
||||
"torrentSize": data["torrent_size"],
|
||||
"url": url,
|
||||
result_season = season if season is not None else "n"
|
||||
result_episode = episode if episode is not None else "n"
|
||||
|
||||
torrents = torrent_manager.torrents
|
||||
for info_hash in torrent_manager.ranked_torrents:
|
||||
torrent = torrents[info_hash]
|
||||
rtn_data = torrent["parsed"]
|
||||
|
||||
debrid_emoji = (
|
||||
"🧲"
|
||||
if debrid_service == "torrent"
|
||||
else ("⚡" if torrent["cached"] else "⬇️")
|
||||
)
|
||||
|
||||
the_stream = {
|
||||
"name": f"[{debrid_extension}{debrid_emoji}] Comet {rtn_data.resolution}",
|
||||
"description": format_title(
|
||||
rtn_data,
|
||||
torrent["title"],
|
||||
torrent["seeders"],
|
||||
torrent["size"],
|
||||
torrent["tracker"],
|
||||
config["resultFormat"],
|
||||
),
|
||||
"behaviorHints": {
|
||||
"filename": data["raw_title"],
|
||||
"bingeGroup": "comet|" + hash,
|
||||
"bingeGroup": "comet|" + info_hash,
|
||||
"videoSize": torrent["size"],
|
||||
"filename": rtn_data.raw_title,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
return {"streams": results}
|
||||
if debrid_service == "torrent":
|
||||
the_stream["infoHash"] = info_hash
|
||||
|
||||
if torrent["fileIndex"] is not None:
|
||||
the_stream["fileIdx"] = torrent["fileIndex"]
|
||||
|
||||
@streams.head("/{b64config}/playback/{hash}/{index}")
|
||||
async def playback(b64config: str, hash: str, index: str):
|
||||
return RedirectResponse("https://stremio.fast", status_code=302)
|
||||
|
||||
|
||||
class CustomORJSONResponse(Response):
|
||||
media_type = "application/json"
|
||||
|
||||
def render(self, content) -> bytes:
|
||||
assert orjson is not None, "orjson must be installed"
|
||||
return orjson.dumps(content, option=orjson.OPT_INDENT_2)
|
||||
|
||||
|
||||
@streams.get("/active-connections", response_class=CustomORJSONResponse)
|
||||
async def active_connections(request: Request, password: str):
|
||||
if password != settings.DASHBOARD_ADMIN_PASSWORD:
|
||||
return "Invalid Password"
|
||||
|
||||
active_connections = await database.fetch_all("SELECT * FROM active_connections")
|
||||
|
||||
return {
|
||||
"total_connections": len(active_connections),
|
||||
"active_connections": active_connections,
|
||||
}
|
||||
|
||||
|
||||
@streams.get("/{b64config}/playback/{hash}/{index}")
|
||||
async def playback(request: Request, b64config: str, hash: str, index: str):
|
||||
config = config_check(b64config)
|
||||
if not config:
|
||||
return FileResponse("comet/assets/invalidconfig.mp4")
|
||||
|
||||
prepare_debrid_config(config)
|
||||
|
||||
async with aiohttp.ClientSession(raise_for_status=True) as session:
|
||||
# Check for cached download link
|
||||
cached_link = await database.fetch_one(
|
||||
f"SELECT link, timestamp FROM download_links WHERE debrid_key = '{config['debridApiKey']}' AND hash = '{hash}' AND file_index = '{index}'"
|
||||
)
|
||||
|
||||
current_time = time.time()
|
||||
download_link = None
|
||||
if cached_link:
|
||||
link = cached_link["link"]
|
||||
timestamp = cached_link["timestamp"]
|
||||
|
||||
if current_time - timestamp < 3600:
|
||||
download_link = link
|
||||
if len(torrent["sources"]) == 0:
|
||||
the_stream["sources"] = trackers
|
||||
else:
|
||||
# Cache expired, remove old entry
|
||||
await database.execute(
|
||||
f"DELETE FROM download_links WHERE debrid_key = '{config['debridApiKey']}' AND hash = '{hash}' AND file_index = '{index}'"
|
||||
the_stream["sources"] = torrent["sources"]
|
||||
else:
|
||||
the_stream["url"] = (
|
||||
f"{request.url.scheme}://{request.url.netloc}/{b64config}/playback/{info_hash}/{torrent['fileIndex'] if torrent['cached'] and torrent['fileIndex'] is not None else 'n'}/{title}/{result_season}/{result_episode}"
|
||||
)
|
||||
|
||||
if torrent["cached"]:
|
||||
cached_results.append(the_stream)
|
||||
else:
|
||||
non_cached_results.append(the_stream)
|
||||
|
||||
return {"streams": cached_results + non_cached_results}
|
||||
|
||||
|
||||
@streams.get("/{b64config}/playback/{hash}/{index}/{name}/{season}/{episode}")
|
||||
async def playback(
|
||||
request: Request,
|
||||
b64config: str,
|
||||
hash: str,
|
||||
index: str,
|
||||
name: str,
|
||||
season: str,
|
||||
episode: str,
|
||||
):
|
||||
config = config_check(b64config)
|
||||
# if not config:
|
||||
# return FileResponse("comet/assets/invalidconfig.mp4")
|
||||
|
||||
season = int(season) if season != "n" else None
|
||||
episode = int(episode) if episode != "n" else None
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
cached_link = await database.fetch_one(
|
||||
f"SELECT download_url FROM download_links_cache WHERE debrid_key = '{config['debridApiKey']}' AND info_hash = '{hash}' AND ((cast(:season as INTEGER) IS NULL AND season IS NULL) OR season = cast(:season as INTEGER)) AND ((cast(:episode as INTEGER) IS NULL AND episode IS NULL) OR episode = cast(:episode as INTEGER)) AND timestamp + 3600 >= :current_time",
|
||||
{
|
||||
"current_time": time.time(),
|
||||
"season": season,
|
||||
"episode": season,
|
||||
},
|
||||
)
|
||||
|
||||
download_url = None
|
||||
if cached_link:
|
||||
download_url = cached_link["download_url"]
|
||||
|
||||
ip = get_client_ip(request)
|
||||
|
||||
if not download_link:
|
||||
debrid = getDebrid(
|
||||
if download_url is None:
|
||||
debrid = get_debrid(
|
||||
session,
|
||||
config,
|
||||
ip
|
||||
if (
|
||||
not is_proxy_stream_enabled(config)
|
||||
or not is_proxy_stream_authed(config)
|
||||
None,
|
||||
None,
|
||||
config["debridService"],
|
||||
config["debridApiKey"],
|
||||
ip,
|
||||
)
|
||||
else "",
|
||||
download_url = await debrid.generate_download_link(
|
||||
hash, index, name, season, episode
|
||||
)
|
||||
download_link = await debrid.generate_download_link(hash, index)
|
||||
if not download_link:
|
||||
if not download_url:
|
||||
return FileResponse("comet/assets/uncached.mp4")
|
||||
|
||||
# Cache the new download link
|
||||
query = f"""
|
||||
INSERT {"OR IGNORE " if settings.DATABASE_TYPE == "sqlite" else ""}
|
||||
INTO download_links_cache
|
||||
VALUES (:debrid_key, :info_hash, :season, :episode, :download_url, :timestamp)
|
||||
{" ON CONFLICT DO NOTHING" if settings.DATABASE_TYPE == "postgresql" else ""}
|
||||
"""
|
||||
|
||||
await database.execute(
|
||||
f"INSERT {'OR IGNORE ' if settings.DATABASE_TYPE == 'sqlite' else ''}INTO download_links (debrid_key, hash, file_index, link, timestamp) VALUES (:debrid_key, :hash, :file_index, :link, :timestamp){' ON CONFLICT DO NOTHING' if settings.DATABASE_TYPE == 'postgresql' else ''}",
|
||||
query,
|
||||
{
|
||||
"debrid_key": config["debridApiKey"],
|
||||
"hash": hash,
|
||||
"file_index": index,
|
||||
"link": download_link,
|
||||
"timestamp": current_time,
|
||||
"info_hash": hash,
|
||||
"season": season,
|
||||
"episode": episode,
|
||||
"download_url": download_url,
|
||||
"timestamp": time.time(),
|
||||
},
|
||||
)
|
||||
|
||||
if should_skip_proxy_stream(config):
|
||||
return RedirectResponse(download_link, status_code=302)
|
||||
|
||||
if (
|
||||
is_proxy_stream_enabled(config)
|
||||
and is_proxy_stream_authed(config)
|
||||
settings.PROXY_DEBRID_STREAM
|
||||
and settings.PROXY_DEBRID_STREAM_PASSWORD
|
||||
== config["debridStreamProxyPassword"]
|
||||
):
|
||||
if settings.PROXY_DEBRID_STREAM_MAX_CONNECTIONS != -1:
|
||||
active_ip_connections = await database.fetch_all(
|
||||
"SELECT ip, COUNT(*) as connections FROM active_connections GROUP BY ip"
|
||||
)
|
||||
if any(
|
||||
connection["ip"] == ip
|
||||
and connection["connections"]
|
||||
>= settings.PROXY_DEBRID_STREAM_MAX_CONNECTIONS
|
||||
for connection in active_ip_connections
|
||||
):
|
||||
return FileResponse("comet/assets/proxylimit.mp4")
|
||||
|
||||
proxy = None
|
||||
|
||||
class Streamer:
|
||||
def __init__(self, id: str):
|
||||
self.id = id
|
||||
|
||||
self.client = httpx.AsyncClient(proxy=proxy, timeout=None)
|
||||
self.response = None
|
||||
|
||||
async def stream_content(self, headers: dict):
|
||||
async with self.client.stream(
|
||||
"GET", download_link, headers=headers
|
||||
) as self.response:
|
||||
async for chunk in self.response.aiter_raw():
|
||||
yield chunk
|
||||
|
||||
async def close(self):
|
||||
await database.execute(
|
||||
f"DELETE FROM active_connections WHERE id = '{self.id}'"
|
||||
return await custom_handle_stream_request(
|
||||
request.method,
|
||||
download_url,
|
||||
mediaflow_proxy.utils.http_utils.get_proxy_headers(request),
|
||||
media_id=hash,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
if self.response is not None:
|
||||
await self.response.aclose()
|
||||
if self.client is not None:
|
||||
await self.client.aclose()
|
||||
|
||||
range_header = request.headers.get("range", "bytes=0-")
|
||||
|
||||
try:
|
||||
if config["debridService"] != "torbox":
|
||||
response = await session.head(
|
||||
download_link, headers={"Range": range_header}
|
||||
)
|
||||
else:
|
||||
response = await session.get(
|
||||
download_link, headers={"Range": range_header}
|
||||
)
|
||||
except aiohttp.ClientResponseError as e:
|
||||
if e.status == 503 and config["debridService"] == "alldebrid":
|
||||
proxy = (
|
||||
settings.DEBRID_PROXY_URL
|
||||
) # proxy is needed only to proxy alldebrid streams
|
||||
|
||||
response = await session.head(
|
||||
download_link, headers={"Range": range_header}, proxy=proxy
|
||||
)
|
||||
else:
|
||||
logger.warning(f"Exception while proxying {download_link}: {e}")
|
||||
return
|
||||
|
||||
if response.status == 206 or (
|
||||
response.status == 200 and config["debridService"] == "torbox"
|
||||
):
|
||||
id = str(uuid.uuid4())
|
||||
await database.execute(
|
||||
f"INSERT {'OR IGNORE ' if settings.DATABASE_TYPE == 'sqlite' else ''}INTO active_connections (id, ip, content, timestamp) VALUES (:id, :ip, :content, :timestamp){' ON CONFLICT DO NOTHING' if settings.DATABASE_TYPE == 'postgresql' else ''}",
|
||||
{
|
||||
"id": id,
|
||||
"ip": ip,
|
||||
"content": str(response.url),
|
||||
"timestamp": current_time,
|
||||
},
|
||||
)
|
||||
|
||||
streamer = Streamer(id)
|
||||
|
||||
return StreamingResponse(
|
||||
streamer.stream_content({"Range": range_header}),
|
||||
status_code=206,
|
||||
headers={
|
||||
"Content-Range": response.headers["Content-Range"],
|
||||
"Content-Length": response.headers["Content-Length"],
|
||||
"Accept-Ranges": "bytes",
|
||||
},
|
||||
background=BackgroundTask(streamer.close),
|
||||
)
|
||||
|
||||
return FileResponse("comet/assets/uncached.mp4")
|
||||
|
||||
return RedirectResponse(download_link, status_code=302)
|
||||
return RedirectResponse(download_url, status_code=302)
|
||||
|
||||
+3
-167
@@ -1,172 +1,8 @@
|
||||
import aiohttp
|
||||
import asyncio
|
||||
|
||||
from RTN import parse
|
||||
|
||||
from comet.utils.general import is_video
|
||||
from comet.utils.logger import logger
|
||||
from comet.utils.models import settings
|
||||
|
||||
|
||||
class AllDebrid:
|
||||
def __init__(self, session: aiohttp.ClientSession, debrid_api_key: str):
|
||||
session.headers["Authorization"] = f"Bearer {debrid_api_key}"
|
||||
self.session = session
|
||||
self.proxy = None
|
||||
|
||||
self.api_url = "http://api.alldebrid.com/v4"
|
||||
self.agent = "comet"
|
||||
|
||||
async def check_premium(self):
|
||||
try:
|
||||
check_premium = await self.session.get(
|
||||
f"{self.api_url}/user?agent={self.agent}"
|
||||
)
|
||||
check_premium = await check_premium.text()
|
||||
if '"isPremium":true' in check_premium:
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while checking premium status on All-Debrid: {e}"
|
||||
)
|
||||
|
||||
return False
|
||||
|
||||
async def get_instant(self, chunk: list):
|
||||
try:
|
||||
get_instant = await self.session.get(
|
||||
f"{self.api_url}/magnet/instant?agent={self.agent}&magnets[]={'&magnets[]='.join(chunk)}"
|
||||
)
|
||||
return await get_instant.json()
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while checking hashes instant availability on All-Debrid: {e}"
|
||||
)
|
||||
|
||||
async def get_files(
|
||||
self, torrent_hashes: list, type: str, season: str, episode: str, kitsu: bool, **kwargs
|
||||
def __init__(
|
||||
self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
|
||||
):
|
||||
chunk_size = 500
|
||||
chunks = [
|
||||
torrent_hashes[i : i + chunk_size]
|
||||
for i in range(0, len(torrent_hashes), chunk_size)
|
||||
]
|
||||
|
||||
tasks = []
|
||||
for chunk in chunks:
|
||||
tasks.append(self.get_instant(chunk))
|
||||
|
||||
responses = await asyncio.gather(*tasks)
|
||||
|
||||
availability = [response for response in responses if response]
|
||||
|
||||
files = {}
|
||||
|
||||
if type == "series":
|
||||
for result in availability:
|
||||
if "status" not in result or result["status"] != "success":
|
||||
continue
|
||||
|
||||
for magnet in result["data"]["magnets"]:
|
||||
if not magnet["instant"]:
|
||||
continue
|
||||
|
||||
for file in magnet["files"]:
|
||||
filename = file["n"]
|
||||
pack = False
|
||||
if "e" in file: # PACK
|
||||
filename = file["e"][0]["n"]
|
||||
pack = True
|
||||
|
||||
if not is_video(filename):
|
||||
continue
|
||||
|
||||
if "sample" in filename.lower():
|
||||
continue
|
||||
|
||||
filename_parsed = parse(filename)
|
||||
if episode not in filename_parsed.episodes:
|
||||
continue
|
||||
|
||||
if kitsu:
|
||||
if filename_parsed.seasons:
|
||||
continue
|
||||
else:
|
||||
if season not in filename_parsed.seasons:
|
||||
continue
|
||||
|
||||
files[magnet["hash"]] = {
|
||||
"index": magnet["files"].index(file),
|
||||
"title": filename,
|
||||
"size": file["e"][0]["s"] if pack else file["s"],
|
||||
}
|
||||
|
||||
break
|
||||
else:
|
||||
for result in availability:
|
||||
if "status" not in result or result["status"] != "success":
|
||||
continue
|
||||
|
||||
for magnet in result["data"]["magnets"]:
|
||||
if not magnet["instant"]:
|
||||
continue
|
||||
|
||||
for file in magnet["files"]:
|
||||
filename = file["n"]
|
||||
|
||||
if not is_video(filename):
|
||||
continue
|
||||
|
||||
if "sample" in filename.lower():
|
||||
continue
|
||||
|
||||
files[magnet["hash"]] = {
|
||||
"index": magnet["files"].index(file),
|
||||
"title": filename,
|
||||
"size": file["s"],
|
||||
}
|
||||
|
||||
break
|
||||
|
||||
return files
|
||||
|
||||
async def generate_download_link(self, hash: str, index: str):
|
||||
try:
|
||||
check_blacklisted = await self.session.get(
|
||||
f"{self.api_url}/magnet/upload?agent=comet&magnets[]={hash}"
|
||||
)
|
||||
check_blacklisted = await check_blacklisted.text()
|
||||
if "NO_SERVER" in check_blacklisted:
|
||||
self.proxy = settings.DEBRID_PROXY_URL
|
||||
if not self.proxy:
|
||||
logger.warning(
|
||||
"All-Debrid blacklisted server's IP. No proxy found."
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"All-Debrid blacklisted server's IP. Switching to proxy {self.proxy} for {hash}|{index}"
|
||||
)
|
||||
|
||||
upload_magnet = await self.session.get(
|
||||
f"{self.api_url}/magnet/upload?agent=comet&magnets[]={hash}",
|
||||
proxy=self.proxy,
|
||||
)
|
||||
upload_magnet = await upload_magnet.json()
|
||||
|
||||
get_magnet_status = await self.session.get(
|
||||
f"{self.api_url}/magnet/status?agent=comet&id={upload_magnet['data']['magnets'][0]['id']}",
|
||||
proxy=self.proxy,
|
||||
)
|
||||
get_magnet_status = await get_magnet_status.json()
|
||||
|
||||
unlock_link = await self.session.get(
|
||||
f"{self.api_url}/link/unlock?agent=comet&link={get_magnet_status['data']['magnets']['links'][int(index)]['link']}",
|
||||
proxy=self.proxy,
|
||||
)
|
||||
unlock_link = await unlock_link.json()
|
||||
|
||||
return unlock_link["data"]["link"]
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while getting download link from All-Debrid for {hash}|{index}: {e}"
|
||||
)
|
||||
pass
|
||||
|
||||
+3
-135
@@ -1,140 +1,8 @@
|
||||
import aiohttp
|
||||
import asyncio
|
||||
|
||||
from RTN import parse
|
||||
|
||||
from comet.utils.general import is_video
|
||||
from comet.utils.logger import logger
|
||||
|
||||
|
||||
class DebridLink:
|
||||
def __init__(self, session: aiohttp.ClientSession, debrid_api_key: str):
|
||||
session.headers["Authorization"] = f"Bearer {debrid_api_key}"
|
||||
self.session = session
|
||||
self.proxy = None
|
||||
|
||||
self.api_url = "https://debrid-link.com/api/v2"
|
||||
|
||||
async def check_premium(self):
|
||||
try:
|
||||
check_premium = await self.session.get(f"{self.api_url}/account/infos")
|
||||
check_premium = await check_premium.text()
|
||||
if '"accountType":1' in check_premium:
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while checking premium status on Debrid-Link: {e}"
|
||||
)
|
||||
|
||||
return False
|
||||
|
||||
async def get_instant(self, chunk: list):
|
||||
responses = []
|
||||
for hash in chunk:
|
||||
try:
|
||||
add_torrent = await self.session.post(
|
||||
f"{self.api_url}/seedbox/add",
|
||||
data={"url": hash, "wait": True, "async": True},
|
||||
)
|
||||
add_torrent = await add_torrent.json()
|
||||
|
||||
torrent_id = add_torrent["value"]["id"]
|
||||
await self.session.delete(f"{self.api_url}/seedbox/{torrent_id}/remove")
|
||||
|
||||
responses.append(add_torrent)
|
||||
except:
|
||||
pass
|
||||
|
||||
return responses
|
||||
|
||||
async def get_files(
|
||||
self, torrent_hashes: list, type: str, season: str, episode: str, kitsu: bool, **kwargs
|
||||
def __init__(
|
||||
self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
|
||||
):
|
||||
chunk_size = 10
|
||||
chunks = [
|
||||
torrent_hashes[i : i + chunk_size]
|
||||
for i in range(0, len(torrent_hashes), chunk_size)
|
||||
]
|
||||
|
||||
tasks = []
|
||||
for chunk in chunks:
|
||||
tasks.append(self.get_instant(chunk))
|
||||
|
||||
responses = await asyncio.gather(*tasks)
|
||||
|
||||
availability = []
|
||||
for response_list in responses:
|
||||
for response in response_list:
|
||||
availability.append(response)
|
||||
|
||||
files = {}
|
||||
|
||||
if type == "series":
|
||||
for result in availability:
|
||||
torrent_files = result["value"]["files"]
|
||||
for file in torrent_files:
|
||||
if file["downloadPercent"] != 100:
|
||||
continue
|
||||
|
||||
filename = file["name"]
|
||||
|
||||
if not is_video(filename):
|
||||
continue
|
||||
|
||||
if "sample" in filename.lower():
|
||||
continue
|
||||
|
||||
filename_parsed = parse(filename)
|
||||
if episode not in filename_parsed.episodes:
|
||||
continue
|
||||
|
||||
if kitsu:
|
||||
if filename_parsed.seasons:
|
||||
continue
|
||||
else:
|
||||
if season not in filename_parsed.seasons:
|
||||
continue
|
||||
|
||||
files[result["value"]["hashString"]] = {
|
||||
"index": torrent_files.index(file),
|
||||
"title": filename,
|
||||
"size": file["size"],
|
||||
}
|
||||
|
||||
break
|
||||
else:
|
||||
for result in availability:
|
||||
value = result["value"]
|
||||
torrent_files = value["files"]
|
||||
for file in torrent_files:
|
||||
if file["downloadPercent"] != 100:
|
||||
continue
|
||||
|
||||
filename = file["name"]
|
||||
|
||||
if not is_video(filename):
|
||||
continue
|
||||
|
||||
if "sample" in filename.lower():
|
||||
continue
|
||||
|
||||
files[value["hashString"]] = {
|
||||
"index": torrent_files.index(file),
|
||||
"title": filename,
|
||||
"size": file["size"],
|
||||
}
|
||||
|
||||
return files
|
||||
|
||||
async def generate_download_link(self, hash: str, index: str):
|
||||
try:
|
||||
add_torrent = await self.session.post(
|
||||
f"{self.api_url}/seedbox/add", data={"url": hash, "async": True}
|
||||
)
|
||||
add_torrent = await add_torrent.json()
|
||||
|
||||
return add_torrent["value"]["files"][int(index)]["downloadUrl"]
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while getting download link from Debrid-Link for {hash}|{index}: {e}"
|
||||
)
|
||||
pass
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
import aiohttp
|
||||
|
||||
|
||||
class EasyDebrid:
|
||||
def __init__(
|
||||
self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
|
||||
):
|
||||
pass
|
||||
+96
-22
@@ -1,35 +1,109 @@
|
||||
import aiohttp
|
||||
|
||||
from comet.utils.config import should_use_stremthru
|
||||
|
||||
from .realdebrid import RealDebrid
|
||||
from .alldebrid import AllDebrid
|
||||
from .premiumize import Premiumize
|
||||
from .torbox import TorBox
|
||||
from .debridlink import DebridLink
|
||||
from .torrent import Torrent
|
||||
from .stremthru import StremThru
|
||||
from .easydebrid import EasyDebrid
|
||||
from .offcloud import Offcloud
|
||||
from .pikpak import PikPak
|
||||
|
||||
debrid_services = {
|
||||
"realdebrid": {
|
||||
"extension": "RD",
|
||||
"cache_availability_endpoint": False,
|
||||
"class": RealDebrid,
|
||||
},
|
||||
"alldebrid": {
|
||||
"extension": "AD",
|
||||
"cache_availability_endpoint": False,
|
||||
"class": AllDebrid,
|
||||
},
|
||||
"premiumize": {
|
||||
"extension": "PM",
|
||||
"cache_availability_endpoint": True,
|
||||
"class": Premiumize,
|
||||
},
|
||||
"torbox": {"extension": "TB", "cache_availability_endpoint": True, "class": TorBox},
|
||||
"debridlink": {
|
||||
"extension": "DL",
|
||||
"cache_availability_endpoint": False,
|
||||
"class": DebridLink,
|
||||
},
|
||||
"stremthru": {
|
||||
"extension": "ST",
|
||||
"cache_availability_endpoint": True,
|
||||
"class": StremThru,
|
||||
},
|
||||
"easydebrid": {
|
||||
"extension": "ED",
|
||||
"cache_availability_endpoint": True,
|
||||
"class": EasyDebrid,
|
||||
},
|
||||
"offcloud": {
|
||||
"extension": "OC",
|
||||
"cache_availability_endpoint": False,
|
||||
"class": Offcloud,
|
||||
},
|
||||
"pikpak": {
|
||||
"extension": "PP",
|
||||
"cache_availability_endpoint": False,
|
||||
"class": PikPak,
|
||||
},
|
||||
"torrent": {
|
||||
"extension": "TORRENT",
|
||||
"cache_availability_endpoint": False,
|
||||
"class": Torrent,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def getDebrid(session: aiohttp.ClientSession, config: dict, ip: str):
|
||||
debrid_service = config["debridService"]
|
||||
debrid_api_key = config["debridApiKey"]
|
||||
def get_debrid_extension(debrid_service: str):
|
||||
original_extension = debrid_services[debrid_service]["extension"]
|
||||
|
||||
if should_use_stremthru(config):
|
||||
return StremThru(
|
||||
session=session,
|
||||
url=config["stremthruUrl"],
|
||||
debrid_service=debrid_service,
|
||||
token=debrid_api_key,
|
||||
ip=ip,
|
||||
return original_extension
|
||||
|
||||
|
||||
def build_stremthru_token(debrid_service: str, debrid_api_key: str):
|
||||
return f"{debrid_service}:{debrid_api_key}"
|
||||
|
||||
|
||||
def get_debrid(
|
||||
session: aiohttp.ClientSession,
|
||||
video_id: str,
|
||||
media_only_id: str,
|
||||
debrid_service: str,
|
||||
debrid_api_key: str,
|
||||
ip: str,
|
||||
):
|
||||
if debrid_service != "torrent":
|
||||
return debrid_services["stremthru"]["class"](
|
||||
session,
|
||||
video_id,
|
||||
media_only_id,
|
||||
build_stremthru_token(debrid_service, debrid_api_key),
|
||||
ip,
|
||||
)
|
||||
|
||||
if debrid_service == "realdebrid":
|
||||
return RealDebrid(session, debrid_api_key, ip)
|
||||
elif debrid_service == "alldebrid":
|
||||
return AllDebrid(session, debrid_api_key)
|
||||
elif debrid_service == "premiumize":
|
||||
return Premiumize(session, debrid_api_key)
|
||||
elif debrid_service == "torbox":
|
||||
return TorBox(session, debrid_api_key)
|
||||
elif debrid_service == "debridlink":
|
||||
return DebridLink(session, debrid_api_key)
|
||||
|
||||
async def retrieve_debrid_availability(
|
||||
session: aiohttp.ClientSession,
|
||||
video_id: str,
|
||||
media_only_id: str,
|
||||
debrid_service: str,
|
||||
debrid_api_key: str,
|
||||
ip: str,
|
||||
info_hashes: list,
|
||||
seeders_map: dict,
|
||||
tracker_map: dict,
|
||||
sources_map: dict,
|
||||
):
|
||||
if debrid_service == "torrent":
|
||||
return []
|
||||
|
||||
return await get_debrid(
|
||||
session, video_id, media_only_id, debrid_service, debrid_api_key, ip
|
||||
).get_availability(info_hashes, seeders_map, tracker_map, sources_map)
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
import aiohttp
|
||||
|
||||
|
||||
class Offcloud:
|
||||
def __init__(
|
||||
self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
|
||||
):
|
||||
pass
|
||||
@@ -0,0 +1,8 @@
|
||||
import aiohttp
|
||||
|
||||
|
||||
class PikPak:
|
||||
def __init__(
|
||||
self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
|
||||
):
|
||||
pass
|
||||
+3
-172
@@ -1,177 +1,8 @@
|
||||
import aiohttp
|
||||
import asyncio
|
||||
|
||||
from RTN import parse
|
||||
|
||||
from comet.utils.general import is_video
|
||||
from comet.utils.logger import logger
|
||||
|
||||
|
||||
class Premiumize:
|
||||
def __init__(self, session: aiohttp.ClientSession, debrid_api_key: str):
|
||||
self.session = session
|
||||
self.proxy = None
|
||||
|
||||
self.api_url = "https://premiumize.me/api"
|
||||
self.debrid_api_key = debrid_api_key
|
||||
|
||||
async def check_premium(self):
|
||||
try:
|
||||
check_premium = await self.session.get(
|
||||
f"{self.api_url}/account/info?apikey={self.debrid_api_key}"
|
||||
)
|
||||
check_premium = await check_premium.text()
|
||||
if (
|
||||
'"status":"success"' in check_premium
|
||||
and '"premium_until":null' not in check_premium
|
||||
def __init__(
|
||||
self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
|
||||
):
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while checking premium status on Premiumize: {e}"
|
||||
)
|
||||
|
||||
return False
|
||||
|
||||
async def get_instant(self, chunk: list):
|
||||
try:
|
||||
response = await self.session.get(
|
||||
f"{self.api_url}/cache/check?apikey={self.debrid_api_key}&items[]={'&items[]='.join(chunk)}"
|
||||
)
|
||||
|
||||
response = await response.json()
|
||||
response["hashes"] = chunk
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while checking hash instant availability on Premiumize: {e}"
|
||||
)
|
||||
|
||||
async def get_files(
|
||||
self, torrent_hashes: list, type: str, season: str, episode: str, kitsu: bool, **kwargs
|
||||
):
|
||||
chunk_size = 100
|
||||
chunks = [
|
||||
torrent_hashes[i : i + chunk_size]
|
||||
for i in range(0, len(torrent_hashes), chunk_size)
|
||||
]
|
||||
|
||||
tasks = []
|
||||
for chunk in chunks:
|
||||
tasks.append(self.get_instant(chunk))
|
||||
|
||||
responses = await asyncio.gather(*tasks)
|
||||
|
||||
availability = []
|
||||
for response in responses:
|
||||
if not response:
|
||||
continue
|
||||
|
||||
availability.append(response)
|
||||
|
||||
files = {}
|
||||
|
||||
if type == "series":
|
||||
for result in availability:
|
||||
if result["status"] != "success":
|
||||
continue
|
||||
|
||||
responses = result["response"]
|
||||
filenames = result["filename"]
|
||||
filesizes = result["filesize"]
|
||||
hashes = result["hashes"]
|
||||
for index, response in enumerate(responses):
|
||||
if not response:
|
||||
continue
|
||||
|
||||
if not filesizes[index]:
|
||||
continue
|
||||
|
||||
filename = filenames[index]
|
||||
|
||||
if "sample" in filename.lower():
|
||||
continue
|
||||
|
||||
filename_parsed = parse(filename)
|
||||
if episode not in filename_parsed.episodes:
|
||||
continue
|
||||
|
||||
if kitsu:
|
||||
if filename_parsed.seasons:
|
||||
continue
|
||||
else:
|
||||
if season not in filename_parsed.seasons:
|
||||
continue
|
||||
|
||||
files[hashes[index]] = {
|
||||
"index": f"{season}|{episode}",
|
||||
"title": filename,
|
||||
"size": int(filesizes[index]),
|
||||
}
|
||||
else:
|
||||
for result in availability:
|
||||
if result["status"] != "success":
|
||||
continue
|
||||
|
||||
responses = result["response"]
|
||||
filenames = result["filename"]
|
||||
filesizes = result["filesize"]
|
||||
hashes = result["hashes"]
|
||||
for index, response in enumerate(responses):
|
||||
if response is False:
|
||||
continue
|
||||
|
||||
if not filesizes[index]:
|
||||
continue
|
||||
|
||||
filename = filenames[index]
|
||||
|
||||
if "sample" in filename.lower():
|
||||
continue
|
||||
|
||||
files[hashes[index]] = {
|
||||
"index": 0,
|
||||
"title": filename,
|
||||
"size": int(filesizes[index]),
|
||||
}
|
||||
|
||||
return files
|
||||
|
||||
async def generate_download_link(self, hash: str, index: str):
|
||||
try:
|
||||
add_magnet = await self.session.post(
|
||||
f"{self.api_url}/transfer/directdl?apikey={self.debrid_api_key}&src=magnet:?xt=urn:btih:{hash}",
|
||||
)
|
||||
add_magnet = await add_magnet.json()
|
||||
|
||||
season = None
|
||||
if "|" in index:
|
||||
index = index.split("|")
|
||||
season = int(index[0])
|
||||
episode = int(index[1])
|
||||
|
||||
content = add_magnet["content"]
|
||||
for file in content:
|
||||
filename = file["path"]
|
||||
if "/" in filename:
|
||||
filename = filename.split("/")[1]
|
||||
|
||||
if not is_video(filename):
|
||||
content.remove(file)
|
||||
continue
|
||||
|
||||
if season is not None:
|
||||
filename_parsed = parse(filename)
|
||||
if (
|
||||
season in filename_parsed.seasons
|
||||
and episode in filename_parsed.episodes
|
||||
):
|
||||
return file["link"]
|
||||
|
||||
max_size_item = max(content, key=lambda x: x["size"])
|
||||
return max_size_item["link"]
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while getting download link from Premiumize for {hash}|{index}: {e}"
|
||||
)
|
||||
pass
|
||||
|
||||
+3
-187
@@ -1,192 +1,8 @@
|
||||
import aiohttp
|
||||
import asyncio
|
||||
|
||||
from RTN import parse
|
||||
|
||||
from comet.utils.general import is_video
|
||||
from comet.utils.logger import logger
|
||||
from comet.utils.models import settings
|
||||
|
||||
|
||||
class RealDebrid:
|
||||
def __init__(self, session: aiohttp.ClientSession, debrid_api_key: str, ip: str):
|
||||
session.headers["Authorization"] = f"Bearer {debrid_api_key}"
|
||||
self.session = session
|
||||
self.ip = ip
|
||||
self.proxy = None
|
||||
|
||||
self.api_url = "https://api.real-debrid.com/rest/1.0"
|
||||
|
||||
async def check_premium(self):
|
||||
try:
|
||||
check_premium = await self.session.get(f"{self.api_url}/user")
|
||||
check_premium = await check_premium.text()
|
||||
if '"type": "premium"' in check_premium:
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while checking premium status on Real-Debrid: {e}"
|
||||
)
|
||||
|
||||
return False
|
||||
|
||||
async def get_instant(self, chunk: list):
|
||||
try:
|
||||
response = await self.session.get(
|
||||
f"{self.api_url}/torrents/instantAvailability/{'/'.join(chunk)}"
|
||||
)
|
||||
return await response.json()
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while checking hash instant availability on Real-Debrid: {e}"
|
||||
)
|
||||
|
||||
async def get_files(
|
||||
self, torrent_hashes: list, type: str, season: str, episode: str, kitsu: bool, **kwargs
|
||||
def __init__(
|
||||
self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
|
||||
):
|
||||
chunk_size = 100
|
||||
chunks = [
|
||||
torrent_hashes[i : i + chunk_size]
|
||||
for i in range(0, len(torrent_hashes), chunk_size)
|
||||
]
|
||||
|
||||
tasks = []
|
||||
for chunk in chunks:
|
||||
tasks.append(self.get_instant(chunk))
|
||||
|
||||
responses = await asyncio.gather(*tasks)
|
||||
|
||||
availability = {}
|
||||
for response in responses:
|
||||
if response is not None:
|
||||
availability.update(response)
|
||||
|
||||
files = {}
|
||||
|
||||
if type == "series":
|
||||
for hash, details in availability.items():
|
||||
if "rd" not in details:
|
||||
continue
|
||||
|
||||
for variants in details["rd"]:
|
||||
for index, file in variants.items():
|
||||
filename = file["filename"]
|
||||
|
||||
if not is_video(filename):
|
||||
continue
|
||||
|
||||
if "sample" in filename.lower():
|
||||
continue
|
||||
|
||||
filename_parsed = parse(filename)
|
||||
if episode not in filename_parsed.episodes:
|
||||
continue
|
||||
|
||||
if kitsu:
|
||||
if filename_parsed.seasons:
|
||||
continue
|
||||
else:
|
||||
if season not in filename_parsed.seasons:
|
||||
continue
|
||||
|
||||
files[hash] = {
|
||||
"index": index,
|
||||
"title": filename,
|
||||
"size": file["filesize"],
|
||||
}
|
||||
|
||||
break
|
||||
else:
|
||||
for hash, details in availability.items():
|
||||
if "rd" not in details:
|
||||
continue
|
||||
|
||||
for variants in details["rd"]:
|
||||
for index, file in variants.items():
|
||||
filename = file["filename"]
|
||||
|
||||
if not is_video(filename):
|
||||
continue
|
||||
|
||||
if "sample" in filename.lower():
|
||||
continue
|
||||
|
||||
files[hash] = {
|
||||
"index": index,
|
||||
"title": filename,
|
||||
"size": file["filesize"],
|
||||
}
|
||||
|
||||
break
|
||||
|
||||
return files
|
||||
|
||||
async def generate_download_link(self, hash: str, index: str):
|
||||
try:
|
||||
check_blacklisted = await self.session.get("https://real-debrid.com/vpn")
|
||||
check_blacklisted = await check_blacklisted.text()
|
||||
if (
|
||||
"Your ISP or VPN provider IP address is currently blocked on our website"
|
||||
in check_blacklisted
|
||||
):
|
||||
self.proxy = settings.DEBRID_PROXY_URL
|
||||
if not self.proxy:
|
||||
logger.warning(
|
||||
"Real-Debrid blacklisted server's IP. No proxy found."
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"Real-Debrid blacklisted server's IP. Switching to proxy {self.proxy} for {hash}|{index}"
|
||||
)
|
||||
|
||||
add_magnet = await self.session.post(
|
||||
f"{self.api_url}/torrents/addMagnet",
|
||||
data={"magnet": f"magnet:?xt=urn:btih:{hash}", "ip": self.ip},
|
||||
proxy=self.proxy,
|
||||
)
|
||||
add_magnet = await add_magnet.json()
|
||||
|
||||
get_magnet_info = await self.session.get(
|
||||
add_magnet["uri"], proxy=self.proxy
|
||||
)
|
||||
get_magnet_info = await get_magnet_info.json()
|
||||
|
||||
await self.session.post(
|
||||
f"{self.api_url}/torrents/selectFiles/{add_magnet['id']}",
|
||||
data={
|
||||
"files": ",".join(
|
||||
str(file["id"])
|
||||
for file in get_magnet_info["files"]
|
||||
if is_video(file["path"])
|
||||
),
|
||||
"ip": self.ip,
|
||||
},
|
||||
proxy=self.proxy,
|
||||
)
|
||||
|
||||
get_magnet_info = await self.session.get(
|
||||
add_magnet["uri"], proxy=self.proxy
|
||||
)
|
||||
get_magnet_info = await get_magnet_info.json()
|
||||
|
||||
index = int(index)
|
||||
realIndex = index
|
||||
for file in get_magnet_info["files"]:
|
||||
if file["id"] == realIndex:
|
||||
break
|
||||
|
||||
if file["selected"] != 1:
|
||||
index -= 1
|
||||
|
||||
unrestrict_link = await self.session.post(
|
||||
f"{self.api_url}/unrestrict/link",
|
||||
data={"link": get_magnet_info["links"][index - 1], "ip": self.ip},
|
||||
proxy=self.proxy,
|
||||
)
|
||||
unrestrict_link = await unrestrict_link.json()
|
||||
|
||||
return unrestrict_link["download"]
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while getting download link from Real-Debrid for {hash}|{index}: {e}"
|
||||
)
|
||||
pass
|
||||
|
||||
+132
-97
@@ -1,59 +1,44 @@
|
||||
import asyncio
|
||||
from typing import Optional
|
||||
|
||||
import aiohttp
|
||||
from RTN import parse
|
||||
import asyncio
|
||||
|
||||
from RTN import parse, title_match
|
||||
|
||||
from comet.utils.models import settings
|
||||
from comet.utils.general import is_video
|
||||
from comet.utils.debrid import cache_availability
|
||||
from comet.utils.logger import logger
|
||||
from comet.utils.torrent import torrent_update_queue
|
||||
|
||||
|
||||
class StremThru:
|
||||
def __init__(
|
||||
self,
|
||||
session: aiohttp.ClientSession,
|
||||
url: str,
|
||||
video_id: str,
|
||||
media_only_id: str,
|
||||
token: str,
|
||||
debrid_service: str,
|
||||
ip: str,
|
||||
):
|
||||
if not self.is_supported_store(debrid_service):
|
||||
raise ValueError(f"unsupported store: {debrid_service}")
|
||||
|
||||
store, token = self.parse_store_creds(debrid_service, token)
|
||||
if store == "stremthru":
|
||||
session.headers["Proxy-Authorization"] = f"Basic {token}"
|
||||
else:
|
||||
store, token = self.parse_store_creds(token)
|
||||
session.headers["X-StremThru-Store-Name"] = store
|
||||
session.headers["X-StremThru-Store-Authorization"] = f"Bearer {token}"
|
||||
|
||||
session.headers["User-Agent"] = "comet"
|
||||
|
||||
self.session = session
|
||||
self.base_url = f"{url}/v0/store"
|
||||
self.name = f"StremThru[{debrid_service}]" if debrid_service else "StremThru"
|
||||
self.base_url = f"{settings.STREMTHRU_URL}/v0/store"
|
||||
self.name = f"StremThru-{store}"
|
||||
self.real_debrid_name = store
|
||||
self.client_ip = ip
|
||||
self.sid = video_id
|
||||
self.media_only_id = media_only_id
|
||||
|
||||
@staticmethod
|
||||
def parse_store_creds(debrid_service, token: str = ""):
|
||||
if debrid_service != "stremthru":
|
||||
return debrid_service, token
|
||||
def parse_store_creds(self, token: str):
|
||||
if ":" in token:
|
||||
parts = token.split(":")
|
||||
parts = token.split(":", 1)
|
||||
return parts[0], parts[1]
|
||||
return debrid_service, token
|
||||
|
||||
@staticmethod
|
||||
def is_supported_store(name: Optional[str]):
|
||||
return (
|
||||
name == "stremthru"
|
||||
or name == "alldebrid"
|
||||
or name == "debridlink"
|
||||
or name == "easydebrid"
|
||||
or name == "premiumize"
|
||||
or name == "realdebrid"
|
||||
or name == "torbox"
|
||||
)
|
||||
return token, ""
|
||||
|
||||
async def check_premium(self):
|
||||
try:
|
||||
@@ -69,11 +54,9 @@ class StremThru:
|
||||
|
||||
return False
|
||||
|
||||
async def get_instant(self, magnets: list, sid: Optional[str] = None):
|
||||
async def get_instant(self, magnets: list):
|
||||
try:
|
||||
url = f"{self.base_url}/magnets/check?magnet={','.join(magnets)}&client_ip={self.client_ip}"
|
||||
if sid:
|
||||
url = f"{url}&sid={sid}"
|
||||
url = f"{self.base_url}/magnets/check?magnet={','.join(magnets)}&client_ip={self.client_ip}&sid={self.sid}"
|
||||
magnet = await self.session.get(url)
|
||||
return await magnet.json()
|
||||
except Exception as e:
|
||||
@@ -81,17 +64,17 @@ class StremThru:
|
||||
f"Exception while checking hash instant availability on {self.name}: {e}"
|
||||
)
|
||||
|
||||
async def get_files(
|
||||
async def get_availability(
|
||||
self,
|
||||
torrent_hashes: list,
|
||||
type: str,
|
||||
season: str,
|
||||
episode: str,
|
||||
kitsu: bool,
|
||||
video_id: Optional[str] = None,
|
||||
**kwargs,
|
||||
seeders_map: dict,
|
||||
tracker_map: dict,
|
||||
sources_map: dict,
|
||||
):
|
||||
chunk_size = 25
|
||||
if not await self.check_premium():
|
||||
return []
|
||||
|
||||
chunk_size = 50
|
||||
chunks = [
|
||||
torrent_hashes[i : i + chunk_size]
|
||||
for i in range(0, len(torrent_hashes), chunk_size)
|
||||
@@ -99,7 +82,7 @@ class StremThru:
|
||||
|
||||
tasks = []
|
||||
for chunk in chunks:
|
||||
tasks.append(self.get_instant(chunk, sid=video_id))
|
||||
tasks.append(self.get_instant(chunk))
|
||||
|
||||
responses = await asyncio.gather(*tasks)
|
||||
|
||||
@@ -109,62 +92,83 @@ class StremThru:
|
||||
if response and "data" in response
|
||||
]
|
||||
|
||||
files = {}
|
||||
is_offcloud = self.real_debrid_name == "offcloud"
|
||||
|
||||
if type == "series":
|
||||
for magnets in availability:
|
||||
for magnet in magnets:
|
||||
if magnet["status"] != "cached":
|
||||
files = []
|
||||
cached_count = 0
|
||||
for result in availability:
|
||||
for torrent in result:
|
||||
if torrent["status"] != "cached":
|
||||
continue
|
||||
|
||||
for file in magnet["files"]:
|
||||
filename = file["name"]
|
||||
cached_count += 1
|
||||
hash = torrent["hash"]
|
||||
seeders = seeders_map[hash]
|
||||
tracker = tracker_map[hash]
|
||||
sources = sources_map[hash]
|
||||
|
||||
if not is_video(filename) or "sample" in filename:
|
||||
if is_offcloud:
|
||||
file_info = {
|
||||
"info_hash": hash,
|
||||
"index": None,
|
||||
"title": None,
|
||||
"size": None,
|
||||
"season": None,
|
||||
"episode": None,
|
||||
"parsed": None,
|
||||
}
|
||||
|
||||
files.append(file_info)
|
||||
else:
|
||||
for file in torrent["files"]:
|
||||
filename = file["name"].split("/")[-1]
|
||||
|
||||
if not is_video(filename) or "sample" in filename.lower():
|
||||
continue
|
||||
|
||||
filename_parsed = parse(filename)
|
||||
|
||||
if episode not in filename_parsed.episodes:
|
||||
season = (
|
||||
filename_parsed.seasons[0]
|
||||
if filename_parsed.seasons
|
||||
else None
|
||||
)
|
||||
episode = (
|
||||
filename_parsed.episodes[0]
|
||||
if filename_parsed.episodes
|
||||
else None
|
||||
)
|
||||
if ":" in self.sid and (season is None or episode is None):
|
||||
continue
|
||||
|
||||
if kitsu:
|
||||
if filename_parsed.seasons:
|
||||
continue
|
||||
else:
|
||||
if season not in filename_parsed.seasons:
|
||||
continue
|
||||
index = file["index"] if file["index"] != -1 else None
|
||||
size = file["size"] if file["size"] != -1 else None
|
||||
|
||||
files[magnet["hash"]] = {
|
||||
"index": file["index"],
|
||||
file_info = {
|
||||
"info_hash": hash,
|
||||
"index": index,
|
||||
"title": filename,
|
||||
"size": file["size"],
|
||||
"size": size,
|
||||
"season": season,
|
||||
"episode": episode,
|
||||
"parsed": filename_parsed,
|
||||
"seeders": seeders,
|
||||
"tracker": tracker,
|
||||
"sources": sources,
|
||||
}
|
||||
|
||||
break
|
||||
else:
|
||||
for magnets in availability:
|
||||
for magnet in magnets:
|
||||
if magnet["status"] != "cached":
|
||||
continue
|
||||
|
||||
for file in magnet["files"]:
|
||||
filename = file["name"]
|
||||
|
||||
if not is_video(filename) or "sample" in filename:
|
||||
continue
|
||||
|
||||
files[magnet["hash"]] = {
|
||||
"index": file["index"],
|
||||
"title": filename,
|
||||
"size": file["size"],
|
||||
}
|
||||
|
||||
break
|
||||
files.append(file_info)
|
||||
await torrent_update_queue.add_torrent_info(file_info, self.media_only_id)
|
||||
|
||||
logger.log(
|
||||
"SCRAPER",
|
||||
f"{self.name}: Found {cached_count} cached torrents with {len(files)} valid files",
|
||||
)
|
||||
return files
|
||||
|
||||
async def generate_download_link(self, hash: str, index: str):
|
||||
async def generate_download_link(
|
||||
self, hash: str, index: str, name: str, season: int, episode: int
|
||||
):
|
||||
try:
|
||||
magnet = await self.session.post(
|
||||
f"{self.base_url}/magnets?client_ip={self.client_ip}",
|
||||
@@ -175,26 +179,57 @@ class StremThru:
|
||||
if magnet["data"]["status"] != "downloaded":
|
||||
return
|
||||
|
||||
file = next(
|
||||
(
|
||||
file
|
||||
for file in magnet["data"]["files"]
|
||||
if str(file["index"]) == index or file["name"] == index
|
||||
),
|
||||
None,
|
||||
)
|
||||
name_parsed = parse(name)
|
||||
target_file = None
|
||||
|
||||
if not file:
|
||||
files = []
|
||||
for file in magnet["data"]["files"]:
|
||||
filename = file["name"]
|
||||
filename_parsed = parse(filename)
|
||||
|
||||
if not is_video(filename) or not title_match(
|
||||
name_parsed.parsed_title, filename_parsed.parsed_title
|
||||
):
|
||||
continue
|
||||
|
||||
file_season = (
|
||||
filename_parsed.seasons[0] if filename_parsed.seasons else None
|
||||
)
|
||||
file_episode = (
|
||||
filename_parsed.episodes[0] if filename_parsed.episodes else None
|
||||
)
|
||||
file_index = file["index"] if file["index"] != -1 else None
|
||||
file_size = file["size"] if file["size"] != -1 else None
|
||||
|
||||
file_info = {
|
||||
"info_hash": hash,
|
||||
"index": file_index,
|
||||
"title": filename,
|
||||
"size": file_size,
|
||||
"season": file_season,
|
||||
"episode": file_episode,
|
||||
"parsed": filename_parsed,
|
||||
}
|
||||
files.append(file_info)
|
||||
|
||||
if str(file["index"]) == index:
|
||||
target_file = file
|
||||
|
||||
if season == file_season and episode == file_episode:
|
||||
target_file = file
|
||||
|
||||
if len(files) > 0:
|
||||
asyncio.create_task(cache_availability(self.real_debrid_name, files))
|
||||
|
||||
if not target_file:
|
||||
return
|
||||
|
||||
link = await self.session.post(
|
||||
f"{self.base_url}/link/generate?client_ip={self.client_ip}",
|
||||
json={"link": file["link"]},
|
||||
json={"link": target_file["link"]},
|
||||
)
|
||||
link = await link.json()
|
||||
|
||||
return link["data"]["link"]
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while getting download link from {self.name} for {hash}|{index}: {e}"
|
||||
)
|
||||
logger.warning(f"Exception while getting download link for {hash}: {e}")
|
||||
|
||||
+3
-156
@@ -1,161 +1,8 @@
|
||||
import aiohttp
|
||||
import asyncio
|
||||
|
||||
from RTN import parse
|
||||
|
||||
from comet.utils.general import is_video
|
||||
from comet.utils.logger import logger
|
||||
|
||||
|
||||
class TorBox:
|
||||
def __init__(self, session: aiohttp.ClientSession, debrid_api_key: str):
|
||||
session.headers["Authorization"] = f"Bearer {debrid_api_key}"
|
||||
self.session = session
|
||||
self.proxy = None
|
||||
|
||||
self.api_url = "https://api.torbox.app/v1/api"
|
||||
self.debrid_api_key = debrid_api_key
|
||||
|
||||
async def check_premium(self):
|
||||
try:
|
||||
check_premium = await self.session.get(
|
||||
f"{self.api_url}/user/me?settings=false"
|
||||
)
|
||||
check_premium = await check_premium.text()
|
||||
if '"success":true' in check_premium:
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Exception while checking premium status on TorBox: {e}")
|
||||
|
||||
return False
|
||||
|
||||
async def get_instant(self, chunk: list):
|
||||
try:
|
||||
response = await self.session.get(
|
||||
f"{self.api_url}/torrents/checkcached?hash={','.join(chunk)}&format=list&list_files=true"
|
||||
)
|
||||
return await response.json()
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while checking hash instant availability on TorBox: {e}"
|
||||
)
|
||||
|
||||
async def get_files(
|
||||
self, torrent_hashes: list, type: str, season: str, episode: str, kitsu: bool, **kwargs
|
||||
def __init__(
|
||||
self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
|
||||
):
|
||||
chunk_size = 50
|
||||
chunks = [
|
||||
torrent_hashes[i : i + chunk_size]
|
||||
for i in range(0, len(torrent_hashes), chunk_size)
|
||||
]
|
||||
|
||||
tasks = []
|
||||
for chunk in chunks:
|
||||
tasks.append(self.get_instant(chunk))
|
||||
|
||||
responses = await asyncio.gather(*tasks)
|
||||
|
||||
availability = [response for response in responses if response is not None]
|
||||
|
||||
files = {}
|
||||
|
||||
if type == "series":
|
||||
for result in availability:
|
||||
if not result["success"] or not result["data"]:
|
||||
continue
|
||||
|
||||
for torrent in result["data"]:
|
||||
torrent_files = torrent["files"]
|
||||
for file in torrent_files:
|
||||
filename = file["name"].split("/")[1]
|
||||
|
||||
if not is_video(filename):
|
||||
continue
|
||||
|
||||
if "sample" in filename.lower():
|
||||
continue
|
||||
|
||||
filename_parsed = parse(filename)
|
||||
if episode not in filename_parsed.episodes:
|
||||
continue
|
||||
|
||||
if kitsu:
|
||||
if filename_parsed.seasons:
|
||||
continue
|
||||
else:
|
||||
if season not in filename_parsed.seasons:
|
||||
continue
|
||||
|
||||
files[torrent["hash"]] = {
|
||||
"index": torrent_files.index(file),
|
||||
"title": filename,
|
||||
"size": file["size"],
|
||||
}
|
||||
|
||||
break
|
||||
else:
|
||||
for result in availability:
|
||||
if not result["success"] or not result["data"]:
|
||||
continue
|
||||
|
||||
for torrent in result["data"]:
|
||||
torrent_files = torrent["files"]
|
||||
for file in torrent_files:
|
||||
filename = file["name"].split("/")[1]
|
||||
|
||||
if not is_video(filename):
|
||||
continue
|
||||
|
||||
if "sample" in filename.lower():
|
||||
continue
|
||||
|
||||
files[torrent["hash"]] = {
|
||||
"index": torrent_files.index(file),
|
||||
"title": filename,
|
||||
"size": file["size"],
|
||||
}
|
||||
|
||||
break
|
||||
|
||||
return files
|
||||
|
||||
async def generate_download_link(self, hash: str, index: str):
|
||||
try:
|
||||
get_torrents = await self.session.get(
|
||||
f"{self.api_url}/torrents/mylist?bypass_cache=true"
|
||||
)
|
||||
get_torrents = await get_torrents.json()
|
||||
exists = False
|
||||
for torrent in get_torrents["data"]:
|
||||
if torrent["hash"] == hash:
|
||||
torrent_id = torrent["id"]
|
||||
exists = True
|
||||
break
|
||||
if not exists:
|
||||
create_torrent = await self.session.post(
|
||||
f"{self.api_url}/torrents/createtorrent",
|
||||
data={"magnet": f"magnet:?xt=urn:btih:{hash}"},
|
||||
)
|
||||
create_torrent = await create_torrent.json()
|
||||
torrent_id = create_torrent["data"]["torrent_id"]
|
||||
|
||||
# get_torrents = await self.session.get(
|
||||
# f"{self.api_url}/torrents/mylist?bypass_cache=true"
|
||||
# )
|
||||
# get_torrents = await get_torrents.json()
|
||||
|
||||
# for torrent in get_torrents["data"]:
|
||||
# if torrent["id"] == torrent_id:
|
||||
# file_id = torrent["files"][int(index)]["id"]
|
||||
# Useless, we already have file index
|
||||
|
||||
get_download_link = await self.session.get(
|
||||
f"{self.api_url}/torrents/requestdl?token={self.debrid_api_key}&torrent_id={torrent_id}&file_id={index}&zip=false",
|
||||
)
|
||||
get_download_link = await get_download_link.json()
|
||||
|
||||
return get_download_link["data"]
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while getting download link from TorBox for {hash}|{index}: {e}"
|
||||
)
|
||||
pass
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
class Torrent:
|
||||
def __init__(self):
|
||||
pass
|
||||
+84
-25
@@ -4,9 +4,11 @@ import sys
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import uvicorn
|
||||
import os
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
@@ -15,7 +17,8 @@ from starlette.requests import Request
|
||||
|
||||
from comet.api.core import main
|
||||
from comet.api.stream import streams
|
||||
from comet.utils.db import setup_database, teardown_database
|
||||
from comet.utils.database import setup_database, teardown_database
|
||||
from comet.utils.trackers import download_best_trackers
|
||||
from comet.utils.logger import logger
|
||||
from comet.utils.models import settings
|
||||
|
||||
@@ -40,6 +43,7 @@ class LoguruMiddleware(BaseHTTPMiddleware):
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
await setup_database()
|
||||
await download_best_trackers()
|
||||
yield
|
||||
await teardown_database()
|
||||
|
||||
@@ -47,7 +51,6 @@ async def lifespan(app: FastAPI):
|
||||
app = FastAPI(
|
||||
title="Comet",
|
||||
summary="Stremio's fastest torrent/debrid search add-on.",
|
||||
version="1.0.0",
|
||||
lifespan=lifespan,
|
||||
redoc_url=None,
|
||||
)
|
||||
@@ -99,17 +102,6 @@ def signal_handler(sig, frame):
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
config = uvicorn.Config(
|
||||
app,
|
||||
host=settings.FASTAPI_HOST,
|
||||
port=settings.FASTAPI_PORT,
|
||||
proxy_headers=True,
|
||||
forwarded_allow_ips="*",
|
||||
workers=settings.FASTAPI_WORKERS,
|
||||
log_config=None,
|
||||
)
|
||||
server = Server(config=config)
|
||||
|
||||
|
||||
def start_log():
|
||||
logger.log(
|
||||
@@ -118,11 +110,11 @@ def start_log():
|
||||
)
|
||||
logger.log(
|
||||
"COMET",
|
||||
f"Dashboard Admin Password: {settings.DASHBOARD_ADMIN_PASSWORD} - http://{settings.FASTAPI_HOST}:{settings.FASTAPI_PORT}/active-connections?password={settings.DASHBOARD_ADMIN_PASSWORD}",
|
||||
f"Dashboard Admin Password: {settings.DASHBOARD_ADMIN_PASSWORD} - http://{settings.FASTAPI_HOST}:{settings.FASTAPI_PORT}/dashboard",
|
||||
)
|
||||
logger.log(
|
||||
"COMET",
|
||||
f"Database ({settings.DATABASE_TYPE}): {settings.DATABASE_PATH if settings.DATABASE_TYPE == 'sqlite' else settings.DATABASE_URL} - TTL: {settings.CACHE_TTL}s",
|
||||
f"Database ({settings.DATABASE_TYPE}): {settings.DATABASE_PATH if settings.DATABASE_TYPE == 'sqlite' else settings.DATABASE_URL} - TTL: metadata={settings.METADATA_CACHE_TTL}s, torrents={settings.TORRENT_CACHE_TTL}s, debrid={settings.DEBRID_CACHE_TTL}s",
|
||||
)
|
||||
logger.log("COMET", f"Debrid Proxy: {settings.DEBRID_PROXY_URL}")
|
||||
|
||||
@@ -133,18 +125,23 @@ def start_log():
|
||||
)
|
||||
logger.log("COMET", f"Indexers: {', '.join(settings.INDEXER_MANAGER_INDEXERS)}")
|
||||
logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s")
|
||||
logger.log(
|
||||
"COMET", f"Download Torrent Files: {bool(settings.DOWNLOAD_TORRENT_FILES)}"
|
||||
)
|
||||
else:
|
||||
logger.log("COMET", "Indexer Manager: False")
|
||||
|
||||
if settings.ZILEAN_URL:
|
||||
zilean_url = f" - {settings.ZILEAN_URL}"
|
||||
logger.log(
|
||||
"COMET",
|
||||
f"Zilean: {settings.ZILEAN_URL} - Take first: {settings.ZILEAN_TAKE_FIRST}",
|
||||
f"Zilean Scraper: {bool(settings.SCRAPE_ZILEAN)}{zilean_url if settings.SCRAPE_ZILEAN else ''}",
|
||||
)
|
||||
else:
|
||||
logger.log("COMET", "Zilean: False")
|
||||
|
||||
logger.log("COMET", f"Torrentio Scraper: {bool(settings.SCRAPE_TORRENTIO)}")
|
||||
torrentio_url = f" - {settings.TORRENTIO_URL}"
|
||||
logger.log(
|
||||
"COMET",
|
||||
f"Torrentio Scraper: {bool(settings.SCRAPE_TORRENTIO)}{torrentio_url if settings.SCRAPE_TORRENTIO else ''}",
|
||||
)
|
||||
|
||||
mediafusion_url = f" - {settings.MEDIAFUSION_URL}"
|
||||
logger.log(
|
||||
@@ -156,14 +153,27 @@ def start_log():
|
||||
"COMET",
|
||||
f"Debrid Stream Proxy: {bool(settings.PROXY_DEBRID_STREAM)} - Password: {settings.PROXY_DEBRID_STREAM_PASSWORD} - Max Connections: {settings.PROXY_DEBRID_STREAM_MAX_CONNECTIONS} - Default Debrid Service: {settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE} - Default Debrid API Key: {settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY}",
|
||||
)
|
||||
if settings.STREMTHRU_DEFAULT_URL:
|
||||
logger.log("COMET", f"Default StremThru URL: {settings.STREMTHRU_DEFAULT_URL}")
|
||||
logger.log("COMET", f"Title Match Check: {bool(settings.TITLE_MATCH_CHECK)}")
|
||||
|
||||
logger.log("COMET", f"StremThru URL: {settings.STREMTHRU_URL}")
|
||||
|
||||
logger.log("COMET", f"Remove Adult Content: {bool(settings.REMOVE_ADULT_CONTENT)}")
|
||||
logger.log("COMET", f"Custom Header HTML: {bool(settings.CUSTOM_HEADER_HTML)}")
|
||||
|
||||
|
||||
with server.run_in_thread():
|
||||
def run_with_uvicorn():
|
||||
"""Run the server with uvicorn only"""
|
||||
config = uvicorn.Config(
|
||||
app,
|
||||
host=settings.FASTAPI_HOST,
|
||||
port=settings.FASTAPI_PORT,
|
||||
proxy_headers=True,
|
||||
forwarded_allow_ips="*",
|
||||
workers=settings.FASTAPI_WORKERS,
|
||||
log_config=None,
|
||||
)
|
||||
server = Server(config=config)
|
||||
|
||||
with server.run_in_thread():
|
||||
start_log()
|
||||
try:
|
||||
while True:
|
||||
@@ -175,3 +185,52 @@ with server.run_in_thread():
|
||||
logger.exception(traceback.format_exc())
|
||||
finally:
|
||||
logger.log("COMET", "Server Shutdown")
|
||||
|
||||
|
||||
def run_with_gunicorn():
|
||||
"""Run the server with gunicorn and uvicorn workers"""
|
||||
import gunicorn.app.base
|
||||
|
||||
class StandaloneApplication(gunicorn.app.base.BaseApplication):
|
||||
def __init__(self, app, options=None):
|
||||
self.options = options or {}
|
||||
self.application = app
|
||||
super().__init__()
|
||||
|
||||
def load_config(self):
|
||||
config = {
|
||||
key: value for key, value in self.options.items()
|
||||
if key in self.cfg.settings and value is not None
|
||||
}
|
||||
for key, value in config.items():
|
||||
self.cfg.set(key.lower(), value)
|
||||
|
||||
def load(self):
|
||||
return self.application
|
||||
|
||||
workers = settings.FASTAPI_WORKERS
|
||||
if workers <= 1:
|
||||
workers = (os.cpu_count() or 1) * 2 + 1
|
||||
|
||||
options = {
|
||||
"bind": f"{settings.FASTAPI_HOST}:{settings.FASTAPI_PORT}",
|
||||
"workers": workers,
|
||||
"worker_class": "uvicorn.workers.UvicornWorker",
|
||||
"timeout": 120,
|
||||
"keepalive": 5,
|
||||
"preload_app": True,
|
||||
"proxy_protocol": True,
|
||||
"forwarded_allow_ips": "*",
|
||||
}
|
||||
|
||||
start_log()
|
||||
logger.log("COMET", f"Starting with gunicorn using {workers} workers")
|
||||
|
||||
StandaloneApplication(app, options).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if os.name == "nt" or not settings.USE_GUNICORN:
|
||||
run_with_uvicorn()
|
||||
else:
|
||||
run_with_gunicorn()
|
||||
@@ -0,0 +1,20 @@
|
||||
import aiohttp
|
||||
|
||||
from comet.utils.logger import logger
|
||||
|
||||
|
||||
async def get_imdb_metadata(session: aiohttp.ClientSession, id: str):
|
||||
try:
|
||||
response = await session.get(
|
||||
f"https://v3.sg.media-imdb.com/suggestion/a/{id}.json"
|
||||
)
|
||||
metadata = await response.json()
|
||||
for element in metadata["d"]:
|
||||
if "/" not in element["id"]:
|
||||
title = element["l"]
|
||||
year = element.get("y")
|
||||
year_end = int(element["yr"].split("-")[1]) if "yr" in element else None
|
||||
return title, year, year_end
|
||||
except Exception as e:
|
||||
logger.warning(f"Exception while getting IMDB metadata for {id}: {e}")
|
||||
return None, None, None, None, None
|
||||
@@ -0,0 +1,44 @@
|
||||
import aiohttp
|
||||
|
||||
from comet.utils.logger import logger
|
||||
|
||||
|
||||
async def get_kitsu_metadata(session: aiohttp.ClientSession, id: str):
|
||||
try:
|
||||
response = await session.get(f"https://kitsu.io/api/edge/anime/{id}")
|
||||
metadata = await response.json()
|
||||
|
||||
attributes = metadata["data"]["attributes"]
|
||||
year = int(attributes["createdAt"].split("-")[0])
|
||||
year_end = int(attributes["updatedAt"].split("-")[0])
|
||||
|
||||
return attributes["canonicalTitle"], year, year_end
|
||||
except Exception as e:
|
||||
logger.warning(f"Exception while getting Kitsu metadata for {id}: {e}")
|
||||
return None, None, None
|
||||
|
||||
|
||||
async def get_kitsu_aliases(session: aiohttp.ClientSession, id: str):
|
||||
aliases = {}
|
||||
try:
|
||||
response = await session.get(f"https://find-my-anime.dtimur.de/api?id={id}&provider=Kitsu")
|
||||
data = await response.json()
|
||||
|
||||
aliases["ez"] = []
|
||||
aliases["ez"].append(data[0]["title"])
|
||||
for synonym in data[0]["synonyms"]:
|
||||
aliases["ez"].append(synonym)
|
||||
|
||||
total_aliases = len(aliases["ez"])
|
||||
if total_aliases > 0:
|
||||
logger.log(
|
||||
"SCRAPER",
|
||||
f"📜 Found {total_aliases} Kitsu aliases for {id}",
|
||||
)
|
||||
return aliases
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.log("SCRAPER", f"📜 No Kitsu aliases found for {id}")
|
||||
|
||||
return {}
|
||||
@@ -0,0 +1,107 @@
|
||||
import aiohttp
|
||||
import asyncio
|
||||
import time
|
||||
import orjson
|
||||
|
||||
from RTN.patterns import normalize_title
|
||||
|
||||
from comet.utils.models import database, settings
|
||||
from comet.utils.general import parse_media_id
|
||||
|
||||
from .kitsu import get_kitsu_metadata, get_kitsu_aliases
|
||||
from .imdb import get_imdb_metadata
|
||||
from .trakt import get_trakt_aliases
|
||||
|
||||
|
||||
class MetadataScraper:
|
||||
def __init__(self, session: aiohttp.ClientSession):
|
||||
self.session = session
|
||||
|
||||
async def fetch_metadata_and_aliases(self, media_type: str, media_id: str):
|
||||
id, season, episode = parse_media_id(media_type, media_id)
|
||||
|
||||
get_cached = await self.get_cached(
|
||||
id, season if not "kitsu" in media_id else 1, episode
|
||||
)
|
||||
if get_cached is not None:
|
||||
return get_cached[0], get_cached[1]
|
||||
|
||||
is_kitsu = "kitsu" in media_id
|
||||
metadata_task = asyncio.create_task(self.get_metadata(id, season, episode, is_kitsu))
|
||||
aliases_task = asyncio.create_task(self.get_aliases(media_type, id, is_kitsu))
|
||||
metadata, aliases = await asyncio.gather(metadata_task, aliases_task)
|
||||
await self.cache_metadata(id, metadata, aliases)
|
||||
|
||||
return metadata, aliases
|
||||
|
||||
async def get_cached(self, media_id: str, season: int, episode: int):
|
||||
row = await database.fetch_one(
|
||||
"""
|
||||
SELECT title, year, year_end, aliases
|
||||
FROM metadata_cache
|
||||
WHERE media_id = :media_id
|
||||
AND timestamp + :cache_ttl >= :current_time
|
||||
""",
|
||||
{
|
||||
"media_id": media_id,
|
||||
"cache_ttl": settings.METADATA_CACHE_TTL,
|
||||
"current_time": time.time(),
|
||||
},
|
||||
)
|
||||
if row is not None:
|
||||
metadata = {
|
||||
"title": row["title"],
|
||||
"year": row["year"],
|
||||
"year_end": row["year_end"],
|
||||
"season": season,
|
||||
"episode": episode,
|
||||
}
|
||||
return metadata, orjson.loads(row["aliases"])
|
||||
|
||||
return None
|
||||
|
||||
async def cache_metadata(self, media_id: str, metadata: dict, aliases: dict):
|
||||
await database.execute(
|
||||
f"""
|
||||
INSERT {"OR IGNORE " if settings.DATABASE_TYPE == "sqlite" else ""}
|
||||
INTO metadata_cache
|
||||
VALUES (:media_id, :title, :year, :year_end, :aliases, :timestamp)
|
||||
{" ON CONFLICT DO NOTHING" if settings.DATABASE_TYPE == "postgresql" else ""}
|
||||
""",
|
||||
{
|
||||
"media_id": media_id,
|
||||
"title": metadata["title"],
|
||||
"year": metadata["year"],
|
||||
"year_end": metadata["year_end"],
|
||||
"aliases": orjson.dumps(aliases).decode("utf-8"),
|
||||
"timestamp": time.time(),
|
||||
},
|
||||
)
|
||||
|
||||
def normalize_metadata(self, metadata: dict, season: int, episode: int):
|
||||
title, year, year_end = metadata
|
||||
|
||||
if title is None: # metadata retrieving failed
|
||||
return None
|
||||
|
||||
return {
|
||||
"title": normalize_title(title),
|
||||
"year": year,
|
||||
"year_end": year_end,
|
||||
"season": season,
|
||||
"episode": episode,
|
||||
}
|
||||
|
||||
async def get_metadata(self, id: str, season: int, episode: int, is_kitsu: bool):
|
||||
if is_kitsu:
|
||||
raw_metadata = await get_kitsu_metadata(self.session, id)
|
||||
return self.normalize_metadata(raw_metadata, 1, episode)
|
||||
else:
|
||||
raw_metadata = await get_imdb_metadata(self.session, id)
|
||||
return self.normalize_metadata(raw_metadata, season, episode)
|
||||
|
||||
async def get_aliases(self, media_type: str, media_id: str, is_kitsu: bool):
|
||||
if is_kitsu:
|
||||
return await get_kitsu_aliases(self.session, media_id)
|
||||
|
||||
return await get_trakt_aliases(self.session, media_type, media_id)
|
||||
@@ -0,0 +1,31 @@
|
||||
import aiohttp
|
||||
|
||||
from comet.utils.logger import logger
|
||||
|
||||
|
||||
async def get_trakt_aliases(
|
||||
session: aiohttp.ClientSession, media_type: str, media_id: str
|
||||
):
|
||||
aliases = set()
|
||||
try:
|
||||
response = await session.get(
|
||||
f"https://api.trakt.tv/{'movies' if media_type == 'movie' else 'shows'}/{media_id}/aliases"
|
||||
)
|
||||
data = await response.json()
|
||||
|
||||
for aliase in data:
|
||||
aliases.add(aliase["title"])
|
||||
|
||||
total_aliases = len(aliases)
|
||||
if total_aliases > 0:
|
||||
logger.log(
|
||||
"SCRAPER",
|
||||
f"📜 Found {total_aliases} Trakt aliases for {media_id}",
|
||||
)
|
||||
return {"ez": list(aliases)}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.log("SCRAPER", f"📜 No Trakt aliases found for {media_id}")
|
||||
|
||||
return {}
|
||||
@@ -0,0 +1,128 @@
|
||||
import aiohttp
|
||||
import asyncio
|
||||
|
||||
from comet.utils.models import settings
|
||||
from comet.utils.logger import logger
|
||||
from comet.utils.torrent import (
|
||||
download_torrent,
|
||||
extract_torrent_metadata,
|
||||
extract_trackers_from_magnet,
|
||||
add_torrent_queue,
|
||||
)
|
||||
|
||||
|
||||
async def process_torrent(
|
||||
session: aiohttp.ClientSession, result: dict, media_id: str, season: int
|
||||
):
|
||||
base_torrent = {
|
||||
"title": result["Title"],
|
||||
"infoHash": None,
|
||||
"fileIndex": None,
|
||||
"seeders": result["Seeders"],
|
||||
"size": result["Size"],
|
||||
"tracker": result["Tracker"],
|
||||
"sources": [],
|
||||
}
|
||||
|
||||
torrents = []
|
||||
|
||||
if result["Link"] is not None:
|
||||
content, magnet_hash, magnet_url = await download_torrent(
|
||||
session, result["Link"]
|
||||
)
|
||||
|
||||
if content:
|
||||
metadata = extract_torrent_metadata(content)
|
||||
if metadata:
|
||||
for file in metadata["files"]:
|
||||
torrent = base_torrent.copy()
|
||||
torrent["title"] = file["name"]
|
||||
torrent["infoHash"] = metadata["info_hash"].lower()
|
||||
torrent["fileIndex"] = file["index"]
|
||||
torrent["size"] = file["size"]
|
||||
torrent["sources"] = metadata["announce_list"]
|
||||
torrents.append(torrent)
|
||||
return torrents
|
||||
|
||||
if magnet_hash:
|
||||
base_torrent["infoHash"] = magnet_hash.lower()
|
||||
base_torrent["sources"] = extract_trackers_from_magnet(magnet_url)
|
||||
|
||||
await add_torrent_queue.add_torrent(
|
||||
magnet_url,
|
||||
base_torrent["seeders"],
|
||||
base_torrent["tracker"],
|
||||
media_id,
|
||||
season,
|
||||
)
|
||||
|
||||
torrents.append(base_torrent)
|
||||
return torrents
|
||||
|
||||
if "InfoHash" in result and result["InfoHash"]:
|
||||
base_torrent["infoHash"] = result["InfoHash"].lower()
|
||||
if result["MagnetUri"] is not None:
|
||||
base_torrent["sources"] = extract_trackers_from_magnet(result["MagnetUri"])
|
||||
|
||||
await add_torrent_queue.add_torrent(
|
||||
result["MagnetUri"],
|
||||
base_torrent["seeders"],
|
||||
base_torrent["tracker"],
|
||||
media_id,
|
||||
season,
|
||||
)
|
||||
|
||||
torrents.append(base_torrent)
|
||||
|
||||
return torrents
|
||||
|
||||
|
||||
async def fetch_jackett_results(
|
||||
session: aiohttp.ClientSession, indexer: str, query: str
|
||||
):
|
||||
try:
|
||||
response = await session.get(
|
||||
f"{settings.INDEXER_MANAGER_URL}/api/v2.0/indexers/all/results?apikey={settings.INDEXER_MANAGER_API_KEY}&Query={query}&Tracker[]={indexer}",
|
||||
timeout=aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT),
|
||||
)
|
||||
response = await response.json()
|
||||
return response.get("Results", [])
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while fetching Jackett results for indexer {indexer}: {e}"
|
||||
)
|
||||
return []
|
||||
|
||||
|
||||
async def get_jackett(manager, session: aiohttp.ClientSession, title: str, seen: set):
|
||||
torrents = []
|
||||
try:
|
||||
tasks = [
|
||||
fetch_jackett_results(session, indexer, title)
|
||||
for indexer in settings.INDEXER_MANAGER_INDEXERS
|
||||
]
|
||||
all_results = await asyncio.gather(*tasks)
|
||||
|
||||
torrent_tasks = []
|
||||
for result_set in all_results:
|
||||
for result in result_set:
|
||||
if result["Details"] in seen:
|
||||
continue
|
||||
|
||||
seen.add(result["Details"])
|
||||
torrent_tasks.append(
|
||||
process_torrent(
|
||||
session, result, manager.media_only_id, manager.season
|
||||
)
|
||||
)
|
||||
|
||||
processed_torrents = await asyncio.gather(*torrent_tasks)
|
||||
torrents = [
|
||||
t for sublist in processed_torrents for t in sublist if t["infoHash"]
|
||||
]
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while getting torrents for {title} with Jackett: {e}"
|
||||
)
|
||||
|
||||
await manager.filter_manager(torrents)
|
||||
@@ -0,0 +1,349 @@
|
||||
import aiohttp
|
||||
import asyncio
|
||||
import orjson
|
||||
import time
|
||||
|
||||
from RTN import (
|
||||
parse,
|
||||
title_match,
|
||||
get_rank,
|
||||
check_fetch,
|
||||
sort_torrents,
|
||||
ParsedData,
|
||||
BestRanking,
|
||||
Torrent,
|
||||
)
|
||||
|
||||
from comet.utils.models import settings, database, CometSettingsModel
|
||||
from comet.utils.general import default_dump
|
||||
from comet.utils.debrid import cache_availability, get_cached_availability
|
||||
from comet.debrid.manager import retrieve_debrid_availability
|
||||
from .zilean import get_zilean
|
||||
from .torrentio import get_torrentio
|
||||
from .mediafusion import get_mediafusion
|
||||
from .jackett import get_jackett
|
||||
from .prowlarr import get_prowlarr
|
||||
|
||||
|
||||
class TorrentManager:
|
||||
def __init__(
|
||||
self,
|
||||
debrid_service: str,
|
||||
debrid_api_key: str,
|
||||
ip: str,
|
||||
media_type: str,
|
||||
media_full_id: str,
|
||||
media_only_id: str,
|
||||
title: str,
|
||||
year: int,
|
||||
year_end: int,
|
||||
season: int,
|
||||
episode: int,
|
||||
aliases: dict,
|
||||
remove_adult_content: bool,
|
||||
):
|
||||
self.debrid_service = debrid_service
|
||||
self.debrid_api_key = debrid_api_key
|
||||
self.ip = ip
|
||||
self.media_type = media_type
|
||||
self.media_id = media_full_id
|
||||
self.media_only_id = media_only_id
|
||||
self.title = title
|
||||
self.year = year
|
||||
self.year_end = year_end
|
||||
self.season = season
|
||||
self.episode = episode
|
||||
self.aliases = aliases
|
||||
self.remove_adult_content = remove_adult_content
|
||||
|
||||
self.seen_hashes = set()
|
||||
self.torrents = {}
|
||||
self.ready_to_cache = []
|
||||
self.ranked_torrents = {}
|
||||
|
||||
async def scrape_torrents(
|
||||
self,
|
||||
session: aiohttp.ClientSession,
|
||||
):
|
||||
tasks = []
|
||||
if settings.SCRAPE_TORRENTIO:
|
||||
tasks.append(get_torrentio(self, self.media_type, self.media_id))
|
||||
if settings.SCRAPE_MEDIAFUSION:
|
||||
tasks.append(get_mediafusion(self, self.media_type, self.media_id))
|
||||
if settings.SCRAPE_ZILEAN:
|
||||
tasks.append(
|
||||
get_zilean(self, session, self.title, self.season, self.episode)
|
||||
)
|
||||
if settings.INDEXER_MANAGER_API_KEY:
|
||||
queries = [self.title]
|
||||
|
||||
if self.media_type == "series":
|
||||
queries.append(f"{self.title} S{self.season:02d}")
|
||||
queries.append(f"{self.title} S{self.season:02d}E{self.episode:02d}")
|
||||
|
||||
seen_already = set()
|
||||
for query in queries:
|
||||
if settings.INDEXER_MANAGER_TYPE == "jackett":
|
||||
tasks.append(get_jackett(self, session, query, seen_already))
|
||||
elif settings.INDEXER_MANAGER_TYPE == "prowlarr":
|
||||
tasks.append(get_prowlarr(self, session, query, seen_already))
|
||||
|
||||
await asyncio.gather(*tasks)
|
||||
asyncio.create_task(self.cache_torrents())
|
||||
|
||||
for torrent in self.ready_to_cache:
|
||||
season = torrent["parsed"].seasons[0] if torrent["parsed"].seasons else None
|
||||
episode = (
|
||||
torrent["parsed"].episodes[0] if torrent["parsed"].episodes else None
|
||||
)
|
||||
|
||||
if (season is not None and season != self.season) or (
|
||||
episode is not None and episode != self.episode
|
||||
):
|
||||
continue
|
||||
|
||||
info_hash = torrent["infoHash"]
|
||||
self.torrents[info_hash] = {
|
||||
"fileIndex": torrent["fileIndex"],
|
||||
"title": torrent["title"],
|
||||
"seeders": torrent["seeders"],
|
||||
"size": torrent["size"],
|
||||
"tracker": torrent["tracker"],
|
||||
"sources": torrent["sources"],
|
||||
"parsed": torrent["parsed"],
|
||||
}
|
||||
|
||||
async def get_cached_torrents(self):
|
||||
rows = await database.fetch_all(
|
||||
"""
|
||||
SELECT info_hash, file_index, title, seeders, size, tracker, sources, parsed
|
||||
FROM torrents
|
||||
WHERE media_id = :media_id
|
||||
AND ((season IS NOT NULL AND season = cast(:season as INTEGER)) OR (season IS NULL AND cast(:season as INTEGER) IS NULL))
|
||||
AND (episode IS NULL OR episode = cast(:episode as INTEGER))
|
||||
AND timestamp + :cache_ttl >= :current_time
|
||||
""",
|
||||
{
|
||||
"media_id": self.media_only_id,
|
||||
"season": self.season,
|
||||
"episode": self.episode,
|
||||
"cache_ttl": settings.TORRENT_CACHE_TTL,
|
||||
"current_time": time.time(),
|
||||
},
|
||||
)
|
||||
|
||||
for row in rows:
|
||||
info_hash = row["info_hash"]
|
||||
self.torrents[info_hash] = {
|
||||
"fileIndex": row["file_index"],
|
||||
"title": row["title"],
|
||||
"seeders": row["seeders"],
|
||||
"size": row["size"],
|
||||
"tracker": row["tracker"],
|
||||
"sources": orjson.loads(row["sources"]),
|
||||
"parsed": ParsedData(**orjson.loads(row["parsed"])),
|
||||
}
|
||||
|
||||
async def cache_torrents(self):
|
||||
current_time = time.time()
|
||||
values = [
|
||||
{
|
||||
"media_id": self.media_only_id,
|
||||
"info_hash": torrent["infoHash"],
|
||||
"file_index": torrent["fileIndex"],
|
||||
"season": torrent["parsed"].seasons[0]
|
||||
if torrent["parsed"].seasons
|
||||
else self.season,
|
||||
"episode": torrent["parsed"].episodes[0]
|
||||
if torrent["parsed"].episodes
|
||||
else None,
|
||||
"title": torrent["title"],
|
||||
"seeders": torrent["seeders"],
|
||||
"size": torrent["size"],
|
||||
"tracker": torrent["tracker"],
|
||||
"sources": orjson.dumps(torrent["sources"]).decode("utf-8"),
|
||||
"parsed": orjson.dumps(torrent["parsed"], default_dump).decode("utf-8"),
|
||||
"timestamp": current_time,
|
||||
}
|
||||
for torrent in self.ready_to_cache
|
||||
]
|
||||
|
||||
query = f"""
|
||||
INSERT {"OR IGNORE " if settings.DATABASE_TYPE == "sqlite" else ""}
|
||||
INTO torrents
|
||||
VALUES (:media_id, :info_hash, :file_index, :season, :episode, :title, :seeders, :size, :tracker, :sources, :parsed, :timestamp)
|
||||
{" ON CONFLICT DO NOTHING" if settings.DATABASE_TYPE == "postgresql" else ""}
|
||||
"""
|
||||
|
||||
await database.execute_many(query, values)
|
||||
|
||||
async def filter(self, torrents: list):
|
||||
title = self.title
|
||||
year = self.year
|
||||
year_end = self.year_end
|
||||
aliases = self.aliases
|
||||
remove_adult_content = self.remove_adult_content
|
||||
|
||||
for torrent in torrents:
|
||||
parsed = parse(torrent["title"])
|
||||
|
||||
if remove_adult_content and parsed.adult:
|
||||
continue
|
||||
|
||||
if parsed.parsed_title and not title_match(
|
||||
title, parsed.parsed_title, aliases=aliases
|
||||
):
|
||||
continue
|
||||
|
||||
if year and parsed.year:
|
||||
if year_end is not None:
|
||||
if not (year <= parsed.year <= year_end):
|
||||
continue
|
||||
else:
|
||||
if year < (parsed.year - 1) or year > (parsed.year + 1):
|
||||
continue
|
||||
|
||||
torrent["parsed"] = parsed
|
||||
self.ready_to_cache.append(torrent)
|
||||
|
||||
async def filter_manager(self, torrents: list):
|
||||
new_torrents = [
|
||||
torrent
|
||||
for torrent in torrents
|
||||
if (torrent["infoHash"], torrent["title"]) not in self.seen_hashes
|
||||
]
|
||||
self.seen_hashes.update(
|
||||
(torrent["infoHash"], torrent["title"]) for torrent in new_torrents
|
||||
)
|
||||
|
||||
chunk_size = 50
|
||||
tasks = [
|
||||
self.filter(new_torrents[i : i + chunk_size])
|
||||
for i in range(0, len(new_torrents), chunk_size)
|
||||
]
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
def rank_torrents(
|
||||
self,
|
||||
rtn_settings: CometSettingsModel,
|
||||
rtn_ranking: BestRanking,
|
||||
max_results_per_resolution: int,
|
||||
max_size: int,
|
||||
cached_only: int,
|
||||
remove_trash: int,
|
||||
):
|
||||
ranked_torrents = set()
|
||||
for info_hash, torrent in self.torrents.items():
|
||||
if (
|
||||
cached_only
|
||||
and self.debrid_service != "torrent"
|
||||
and not torrent["cached"]
|
||||
):
|
||||
continue
|
||||
|
||||
if max_size != 0 and torrent["size"] > max_size:
|
||||
continue
|
||||
|
||||
parsed = torrent["parsed"]
|
||||
|
||||
raw_title = torrent["title"]
|
||||
|
||||
is_fetchable, failed_keys = check_fetch(parsed, rtn_settings)
|
||||
rank = get_rank(parsed, rtn_settings, rtn_ranking)
|
||||
|
||||
if remove_trash:
|
||||
if (
|
||||
not is_fetchable
|
||||
or rank < rtn_settings.options["remove_ranks_under"]
|
||||
):
|
||||
continue
|
||||
|
||||
try:
|
||||
ranked_torrents.add(
|
||||
Torrent(
|
||||
infohash=info_hash,
|
||||
raw_title=raw_title,
|
||||
data=parsed,
|
||||
fetch=is_fetchable,
|
||||
rank=rank,
|
||||
lev_ratio=0.0,
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self.ranked_torrents = sort_torrents(
|
||||
ranked_torrents, max_results_per_resolution
|
||||
)
|
||||
|
||||
async def get_and_cache_debrid_availability(self, session: aiohttp.ClientSession):
|
||||
info_hashes = list(self.torrents.keys())
|
||||
|
||||
seeders_map = {hash: self.torrents[hash]["seeders"] for hash in info_hashes}
|
||||
tracker_map = {hash: self.torrents[hash]["tracker"] for hash in info_hashes}
|
||||
sources_map = {hash: self.torrents[hash]["sources"] for hash in info_hashes}
|
||||
|
||||
availability = await retrieve_debrid_availability(
|
||||
session,
|
||||
self.media_id,
|
||||
self.media_only_id,
|
||||
self.debrid_service,
|
||||
self.debrid_api_key,
|
||||
self.ip,
|
||||
info_hashes,
|
||||
seeders_map,
|
||||
tracker_map,
|
||||
sources_map,
|
||||
)
|
||||
|
||||
if len(availability) == 0:
|
||||
return
|
||||
|
||||
for file in availability:
|
||||
season = file["season"]
|
||||
episode = file["episode"]
|
||||
if (season is not None and season != self.season) or (
|
||||
episode is not None and episode != self.episode
|
||||
):
|
||||
continue
|
||||
|
||||
info_hash = file["info_hash"]
|
||||
self.torrents[info_hash]["cached"] = True
|
||||
|
||||
if file["parsed"] is not None:
|
||||
self.torrents[info_hash]["parsed"] = file["parsed"]
|
||||
if file["index"] is not None:
|
||||
self.torrents[info_hash]["fileIndex"] = file["index"]
|
||||
if file["title"] is not None:
|
||||
self.torrents[info_hash]["title"] = file["title"]
|
||||
if file["size"] is not None:
|
||||
self.torrents[info_hash]["size"] = file["size"]
|
||||
|
||||
asyncio.create_task(cache_availability(self.debrid_service, availability))
|
||||
|
||||
async def get_cached_availability(self):
|
||||
info_hashes = list(self.torrents.keys())
|
||||
for hash in info_hashes:
|
||||
self.torrents[hash]["cached"] = False
|
||||
|
||||
if self.debrid_service == "torrent" or len(self.torrents) == 0:
|
||||
return
|
||||
|
||||
rows = await get_cached_availability(
|
||||
self.debrid_service, info_hashes, self.season, self.episode
|
||||
)
|
||||
|
||||
for row in rows:
|
||||
info_hash = row["info_hash"]
|
||||
self.torrents[info_hash]["cached"] = True
|
||||
|
||||
if row["parsed"] is not None:
|
||||
self.torrents[info_hash]["parsed"] = ParsedData(
|
||||
**orjson.loads(row["parsed"])
|
||||
)
|
||||
if row["file_index"] is not None:
|
||||
self.torrents[info_hash]["fileIndex"] = row["file_index"]
|
||||
if row["title"] is not None:
|
||||
self.torrents[info_hash]["title"] = row["title"]
|
||||
if row["size"] is not None:
|
||||
self.torrents[info_hash]["size"] = row["size"]
|
||||
@@ -0,0 +1,58 @@
|
||||
from curl_cffi import requests
|
||||
|
||||
from comet.utils.models import settings
|
||||
from comet.utils.logger import logger
|
||||
|
||||
|
||||
async def get_mediafusion(manager, media_type: str, media_id: str):
|
||||
torrents = []
|
||||
try:
|
||||
try:
|
||||
get_mediafusion = requests.get(
|
||||
f"{settings.MEDIAFUSION_URL}/D-zn4qJLK4wUZVWscY9ESCnoZBEiNJCZ9uwfCvmxuliDjY7vkc-fu0OdxUPxwsP3_A/stream/{media_type}/{media_id}.json"
|
||||
).json()
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed to get MediaFusion results without proxy for {media_id}: {e}"
|
||||
)
|
||||
|
||||
get_mediafusion = requests.get(
|
||||
f"{settings.MEDIAFUSION_URL}/stream/{media_type}/{media_id}.json",
|
||||
proxies={
|
||||
"http": settings.DEBRID_PROXY_URL,
|
||||
"https": settings.DEBRID_PROXY_URL,
|
||||
},
|
||||
).json()
|
||||
|
||||
for torrent in get_mediafusion["streams"]:
|
||||
title_full = torrent["description"]
|
||||
lines = title_full.split("\n")
|
||||
|
||||
title = lines[0].replace("📂 ", "").replace("/", "")
|
||||
|
||||
seeders = None
|
||||
if "👤" in lines[1]:
|
||||
seeders = int(lines[1].split("👤 ")[1].split("\n")[0])
|
||||
|
||||
tracker = lines[-1].split("🔗 ")[1]
|
||||
|
||||
torrents.append(
|
||||
{
|
||||
"title": title,
|
||||
"infoHash": torrent["infoHash"].lower(),
|
||||
"fileIndex": torrent["fileIdx"] if "fileIdx" in torrent else None,
|
||||
"seeders": seeders,
|
||||
"size": torrent["behaviorHints"][
|
||||
"videoSize"
|
||||
], # not the pack size but still useful for prowlarr userss
|
||||
"tracker": f"MediaFusion|{tracker}",
|
||||
"sources": torrent["sources"] if "sources" in torrent else [],
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while getting torrents for {media_id} with MediaFusion, your IP is most likely blacklisted (you should try proxying Comet): {e}"
|
||||
)
|
||||
pass
|
||||
|
||||
await manager.filter_manager(torrents)
|
||||
@@ -0,0 +1,124 @@
|
||||
import aiohttp
|
||||
import asyncio
|
||||
|
||||
from comet.utils.models import settings
|
||||
from comet.utils.logger import logger
|
||||
from comet.utils.torrent import (
|
||||
download_torrent,
|
||||
extract_torrent_metadata,
|
||||
extract_trackers_from_magnet,
|
||||
add_torrent_queue,
|
||||
)
|
||||
|
||||
|
||||
async def process_torrent(
|
||||
session: aiohttp.ClientSession, result: dict, media_id: str, season: int
|
||||
):
|
||||
base_torrent = {
|
||||
"title": result["title"],
|
||||
"infoHash": None,
|
||||
"fileIndex": None,
|
||||
"seeders": result["seeders"],
|
||||
"size": result["size"],
|
||||
"tracker": result["indexer"],
|
||||
"sources": [],
|
||||
}
|
||||
|
||||
torrents = []
|
||||
|
||||
if "downloadUrl" in result:
|
||||
content, magnet_hash, magnet_url = await download_torrent(
|
||||
session, result["downloadUrl"]
|
||||
)
|
||||
|
||||
if content:
|
||||
metadata = extract_torrent_metadata(content)
|
||||
if metadata:
|
||||
for file in metadata["files"]:
|
||||
torrent = base_torrent.copy()
|
||||
torrent["title"] = file["name"]
|
||||
torrent["infoHash"] = metadata["info_hash"].lower()
|
||||
torrent["fileIndex"] = file["index"]
|
||||
torrent["size"] = file["size"]
|
||||
torrent["sources"] = metadata["announce_list"]
|
||||
torrents.append(torrent)
|
||||
return torrents
|
||||
|
||||
if magnet_hash:
|
||||
base_torrent["infoHash"] = magnet_hash.lower()
|
||||
base_torrent["sources"] = extract_trackers_from_magnet(magnet_url)
|
||||
|
||||
await add_torrent_queue.add_torrent(
|
||||
magnet_url,
|
||||
base_torrent["seeders"],
|
||||
base_torrent["tracker"],
|
||||
media_id,
|
||||
season,
|
||||
)
|
||||
|
||||
torrents.append(base_torrent)
|
||||
return torrents
|
||||
|
||||
if "infoHash" in result and result["infoHash"]:
|
||||
base_torrent["infoHash"] = result["infoHash"].lower()
|
||||
if "guid" in result and result["guid"].startswith("magnet:"):
|
||||
base_torrent["sources"] = extract_trackers_from_magnet(result["guid"])
|
||||
|
||||
await add_torrent_queue.add_torrent(
|
||||
result["guid"],
|
||||
base_torrent["seeders"],
|
||||
base_torrent["tracker"],
|
||||
media_id,
|
||||
season,
|
||||
)
|
||||
|
||||
torrents.append(base_torrent)
|
||||
|
||||
return torrents
|
||||
|
||||
|
||||
async def get_prowlarr(manager, session: aiohttp.ClientSession, title: str, seen: set):
|
||||
torrents = []
|
||||
try:
|
||||
indexers = [indexer.lower() for indexer in settings.INDEXER_MANAGER_INDEXERS]
|
||||
|
||||
get_indexers = await session.get(
|
||||
f"{settings.INDEXER_MANAGER_URL}/api/v1/indexer",
|
||||
headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY},
|
||||
)
|
||||
get_indexers = await get_indexers.json()
|
||||
|
||||
indexers_id = []
|
||||
for indexer in get_indexers:
|
||||
if (
|
||||
indexer["name"].lower() in indexers
|
||||
or indexer["definitionName"].lower() in indexers
|
||||
):
|
||||
indexers_id.append(indexer["id"])
|
||||
|
||||
response = await session.get(
|
||||
f"{settings.INDEXER_MANAGER_URL}/api/v1/search?query={title}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search",
|
||||
headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY},
|
||||
)
|
||||
response = await response.json()
|
||||
|
||||
torrent_tasks = []
|
||||
for result in response:
|
||||
if result["infoUrl"] in seen:
|
||||
continue
|
||||
|
||||
seen.add(result["infoUrl"])
|
||||
torrent_tasks.append(
|
||||
process_torrent(session, result, manager.media_only_id, manager.season)
|
||||
)
|
||||
|
||||
processed_torrents = await asyncio.gather(*torrent_tasks)
|
||||
torrents = [
|
||||
t for sublist in processed_torrents for t in sublist if t["infoHash"]
|
||||
]
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while getting torrents for {title} with Prowlarr: {e}"
|
||||
)
|
||||
|
||||
await manager.filter_manager(torrents)
|
||||
@@ -0,0 +1,65 @@
|
||||
import re
|
||||
|
||||
from curl_cffi import requests
|
||||
|
||||
from comet.utils.models import settings
|
||||
from comet.utils.logger import logger
|
||||
from comet.utils.general import size_to_bytes
|
||||
|
||||
|
||||
data_pattern = re.compile(
|
||||
r"(?:👤 (\d+) )?💾 ([\d.]+ [KMGT]B)(?: ⚙️ (\w+))?", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
async def get_torrentio(manager, media_type: str, media_id: str):
|
||||
torrents = []
|
||||
try:
|
||||
try:
|
||||
get_torrentio = requests.get(
|
||||
f"{settings.TORRENTIO_URL}/stream/{media_type}/{media_id}.json"
|
||||
).json()
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed to get Torrentio results without proxy for {media_id}: {e}"
|
||||
)
|
||||
|
||||
get_torrentio = requests.get(
|
||||
f"{settings.TORRENTIO_URL}/stream/{media_type}/{media_id}.json",
|
||||
proxies={
|
||||
"http": settings.DEBRID_PROXY_URL,
|
||||
"https": settings.DEBRID_PROXY_URL,
|
||||
},
|
||||
).json()
|
||||
|
||||
for torrent in get_torrentio["streams"]:
|
||||
title_full = torrent["title"]
|
||||
title = (
|
||||
title_full.split("\n")[0]
|
||||
if settings.TORRENTIO_URL == "https://torrentio.strem.fun"
|
||||
else title_full.split("\n💾")[0].split("\n")[-1]
|
||||
)
|
||||
|
||||
match = data_pattern.search(title_full)
|
||||
|
||||
seeders = int(match.group(1)) if match.group(1) else None
|
||||
size = size_to_bytes(match.group(2))
|
||||
tracker = match.group(3) if match.group(3) else "KnightCrawler"
|
||||
|
||||
torrents.append(
|
||||
{
|
||||
"title": title,
|
||||
"infoHash": torrent["infoHash"].lower(),
|
||||
"fileIndex": torrent["fileIdx"] if "fileIdx" in torrent else None,
|
||||
"seeders": seeders,
|
||||
"size": size,
|
||||
"tracker": f"Torrentio|{tracker}",
|
||||
"sources": torrent["sources"] if "sources" in torrent else [],
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while getting torrents for {media_id} with Torrentio, your IP is most likely blacklisted (you should try proxying Comet): {e}"
|
||||
)
|
||||
|
||||
await manager.filter_manager(torrents)
|
||||
@@ -0,0 +1,33 @@
|
||||
import aiohttp
|
||||
|
||||
from comet.utils.models import settings
|
||||
from comet.utils.logger import logger
|
||||
|
||||
|
||||
async def get_zilean(
|
||||
manager, session: aiohttp.ClientSession, title: str, season: int, episode: int
|
||||
):
|
||||
torrents = []
|
||||
try:
|
||||
show = f"&season={season}&episode={episode}"
|
||||
get_dmm = await session.get(
|
||||
f"{settings.ZILEAN_URL}/dmm/filtered?query={title}{show if season else ''}"
|
||||
)
|
||||
get_dmm = await get_dmm.json()
|
||||
|
||||
for result in get_dmm:
|
||||
object = {
|
||||
"title": result["raw_title"],
|
||||
"infoHash": result["info_hash"].lower(),
|
||||
"fileIndex": None,
|
||||
"seeders": None,
|
||||
"size": int(result["size"]),
|
||||
"tracker": "DMM",
|
||||
"sources": [],
|
||||
}
|
||||
|
||||
torrents.append(object)
|
||||
except Exception as e:
|
||||
logger.warning(f"Exception while getting torrents for {title} with Zilean: {e}")
|
||||
|
||||
await manager.filter_manager(torrents)
|
||||
+232
-119
@@ -12,8 +12,8 @@
|
||||
<title>Comet - Stremio's fastest torrent/debrid search add-on.</title>
|
||||
<link id="favicon" rel="icon" type="image/x-icon" href="https://i.imgur.com/jmVoVMu.jpeg">
|
||||
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@shoelace-style/shoelace@2.15.1/cdn/themes/dark.css" />
|
||||
<script type="module" src="https://cdn.jsdelivr.net/npm/@shoelace-style/shoelace@2.15.1/cdn/shoelace-autoloader.js"></script>
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@shoelace-style/shoelace@2.19.0/cdn/themes/dark.css" />
|
||||
<script type="module" src="https://cdn.jsdelivr.net/npm/@shoelace-style/shoelace@2.19.0/cdn/shoelace-autoloader.js"></script>
|
||||
|
||||
<style>
|
||||
:not(:defined) {
|
||||
@@ -512,19 +512,26 @@
|
||||
|
||||
<div class="form-container">
|
||||
<div class="form-item">
|
||||
<sl-select id="indexers" multiple clearable label="Indexers" placeholder="Select indexers" {% if not indexerManager %} disabled {% endif %}></sl-select>
|
||||
<sl-select id="languages_required" multiple clearable label="Required Languages" placeholder="Select required languages"></sl-select>
|
||||
</div>
|
||||
|
||||
<div class="form-item">
|
||||
<sl-select id="languages" multiple clearable label="Languages" placeholder="Select languages"></sl-select>
|
||||
<sl-select id="languages_preferred" multiple clearable label="Preferred Languages" placeholder="Select preferred languages"></sl-select>
|
||||
</div>
|
||||
|
||||
<div class="form-item">
|
||||
<sl-select id="resolutions" multiple clearable label="Resolutions" placeholder="Select resolutions"></sl-select>
|
||||
<sl-select id="languages_excluded" multiple clearable label="Excluded Languages" placeholder="Select excluded languages"></sl-select>
|
||||
</div>
|
||||
|
||||
<div class="form-item">
|
||||
<sl-input id="maxResults" type="number" min=0 value=0 label="Max Results" placeholder="Enter max results"></sl-input>
|
||||
<sl-select id="resolutions" multiple clearable label="Resolutions" placeholder="Select resolutions">
|
||||
<sl-option value="r2160p">4K - 2160p</sl-option>
|
||||
<sl-option value="r1080p">Full HD - 1080p</sl-option>
|
||||
<sl-option value="r720p">HD - 720p</sl-option>
|
||||
<sl-option value="r480p">SD - 480p</sl-option>
|
||||
<sl-option value="r360p">LD - 360p</sl-option>
|
||||
<sl-option value="unknown">Unknown</sl-option>
|
||||
</sl-select>
|
||||
</div>
|
||||
|
||||
<div class="form-item">
|
||||
@@ -535,37 +542,39 @@
|
||||
<sl-input id="maxSize" type="number" min=0 value=0 label="Max Size (GB)" placeholder="Enter max size in gigabytes"></sl-input>
|
||||
</div>
|
||||
|
||||
<div class="form-item">
|
||||
<sl-input id="debridStreamProxyPassword" label="Debrid Stream Proxy Password" placeholder="{% if not proxyDebridStream %}Configuration required, see docs{% else %}Enter password{% endif %}" help-text="Debrid Stream Proxying allows you to use your Debrid Service from multiple IPs at same time!" {% if not proxyDebridStream %} disabled {% endif %}></sl-input>
|
||||
<div class="form-item" {% if not proxyDebridStream %}style="display: none"{% endif %}>
|
||||
<sl-input id="debridStreamProxyPassword" label="Debrid Stream Proxy Password" placeholder="Enter password" help-text="Debrid Stream Proxying allows you to use your Debrid Service from multiple IPs at same time!"></sl-input>
|
||||
</div>
|
||||
|
||||
<div class="form-item">
|
||||
<sl-select id="debridService" value="torbox" label="Debrid Service" placeholder="Select debrid service">
|
||||
<sl-select id="debridService" value="torrent" label="Debrid Service" placeholder="Select debrid service">
|
||||
<sl-option value="torrent">Torrent</sl-option>
|
||||
<sl-option value="torbox">TorBox</sl-option>
|
||||
<sl-option value="easydebrid">EasyDebrid</sl-option>
|
||||
<sl-option value="realdebrid">Real-Debrid</sl-option>
|
||||
<sl-option value="debridlink">Debrid-Link</sl-option>
|
||||
<sl-option value="alldebrid">All-Debrid</sl-option>
|
||||
<sl-option value="premiumize">Premiumize</sl-option>
|
||||
<sl-option value="realdebrid">Real-Debrid</sl-option>
|
||||
<sl-option value="stremthru">StremThru</sl-option>
|
||||
<sl-option value="offcloud">Offcloud</sl-option>
|
||||
<sl-option value="offcloud">PikPak</sl-option>
|
||||
</sl-select>
|
||||
</div>
|
||||
|
||||
<div class="form-item">
|
||||
<label for="debridService">
|
||||
Debrid API Key -
|
||||
<a id="apiKeyLink" href="https://torbox.app/settings" target="_blank">Get It Here</a>
|
||||
Debrid API Key
|
||||
<a id="apiKeyLink" href="" target="_blank"></a>
|
||||
</label>
|
||||
<sl-input id="debridApiKey" placeholder="Enter API key" help-text="If no key is specified, you'll get direct torrents!"></sl-input>
|
||||
</div>
|
||||
|
||||
<div class="form-item">
|
||||
<sl-input id="stremthruUrl" label="StremThru URL" placeholder="{{stremthruDefaultUrl}}" help-text="URL for StremThru."></sl-input>
|
||||
<sl-input id="debridApiKey" placeholder="Enter API key"></sl-input>
|
||||
</div>
|
||||
|
||||
<sl-details summary="Advanced Settings">
|
||||
<div class="form-item">
|
||||
<sl-checkbox checked id="removeTrash" help-text="Remove all trash from results (Adult Content, CAM, Clean Audio, PDTV, R5, Screener, Size, Telecine and Telesync)">Remove Trash</sl-checkbox>
|
||||
<sl-checkbox id="reverseResultOrder" help-text="Reverse the order of the results for each resolution (useful for those who need small file sizes)">Reverse Result Order</sl-checkbox>
|
||||
<sl-input id="rankThreshold" type="number" value="-10000000000" label="Minimum Rank Threshold" placeholder="Enter minimum rank threshold" help-text="Torrents ranked below this value will be excluded from results."></sl-input>
|
||||
<sl-checkbox id="cachedOnly" help-text="Show only content that has already been cached.">Show Cached Only</sl-checkbox>
|
||||
<sl-checkbox id="allowEnglishInLanguages" help-text="Allows English language torrents to be included even if you excluded English from your language preferences.">Allow English in Languages</sl-checkbox>
|
||||
<sl-checkbox id="removeUnknownLanguages" help-text="Exclude torrents that don't have a language in the title.">Remove Unknown Languages</sl-checkbox>
|
||||
<sl-checkbox checked id="removeTrash" help-text="Remove all trash from results (if disabled, your filters will not be applied - Adult Content, CAM, Clean Audio, PDTV, R5, Screener, Size, Telecine and Telesync)">Remove Trash</sl-checkbox>
|
||||
<sl-select id="resultFormat" multiple label="Result Format" placeholder="Select what to show in result title" hoist max-options-visible=10>
|
||||
</sl-select>
|
||||
</div>
|
||||
@@ -576,24 +585,51 @@
|
||||
const selectedService = event.target.value;
|
||||
const apiKeyLink = document.getElementById("apiKeyLink");
|
||||
const apiKeyInput = document.getElementById("debridApiKey");
|
||||
debridStreamProxyPasswordInput = document.getElementById("debridStreamProxyPassword");
|
||||
|
||||
if (selectedService === "realdebrid") {
|
||||
apiKeyLink.href = "https://real-debrid.com/apitoken";
|
||||
apiKeyLink.textContent = "Get It Here"
|
||||
} else if (selectedService === "alldebrid") {
|
||||
apiKeyLink.href = "https://alldebrid.com/apikeys";
|
||||
apiKeyLink.textContent = "Get It Here"
|
||||
} else if (selectedService === "premiumize") {
|
||||
apiKeyLink.href = "https://premiumize.me/account";
|
||||
apiKeyLink.textContent = "Get It Here"
|
||||
} else if (selectedService === "torbox") {
|
||||
apiKeyLink.href = "https://torbox.app/settings";
|
||||
apiKeyLink.textContent = "Get It Here"
|
||||
} else if (selectedService === "easydebrid") {
|
||||
apiKeyLink.href = "https://paradise-cloud.com/products/easydebrid";
|
||||
apiKeyLink.textContent = "Get It Here"
|
||||
} else if (selectedService === "debridlink") {
|
||||
apiKeyLink.href = "https://debrid-link.com/webapp/apikey";
|
||||
} else if (selectedService === "stremthru") {
|
||||
apiKeyLink.href = "https://github.com/MunifTanjim/stremthru?tab=readme-ov-file#configuration";
|
||||
apiKeyLink.textContent = "Get It Here"
|
||||
} else if (selectedService === "offcloud") {
|
||||
apiKeyLink.href = "https://offcloud.com";
|
||||
apiKeyLink.textContent = "Get It Here"
|
||||
} else if (selectedService === "pikpak") {
|
||||
apiKeyLink.href = "https://mypikpak.com";
|
||||
apiKeyLink.textContent = "Get It Here"
|
||||
} else if (selectedService === "torrent") {
|
||||
apiKeyLink.href = "";
|
||||
apiKeyLink.textContent = ""
|
||||
}
|
||||
if (selectedService === "stremthru") {
|
||||
apiKeyInput.helpText = "Enter Credential ('store_name:store_token' or base64 encoded basic token from 'STREMTHRU_PROXY_AUTH' config)"
|
||||
|
||||
if (selectedService === "offcloud" || selectedService === "pikpak") {
|
||||
apiKeyInput.helpText = "Format: `email:password`"
|
||||
} else if (selectedService != "torrent") {
|
||||
apiKeyInput.helpText = "Format: `api-key`"
|
||||
} else {
|
||||
apiKeyInput.helpText = "If no key is specified, you'll get direct torrents!"
|
||||
apiKeyInput.helpText = ""
|
||||
}
|
||||
|
||||
if (selectedService === "torrent") {
|
||||
debridStreamProxyPasswordInput.disabled = true;
|
||||
apiKeyInput.disabled = true;
|
||||
} else {
|
||||
debridStreamProxyPasswordInput.disabled = false;
|
||||
apiKeyInput.disabled = false;
|
||||
}
|
||||
});
|
||||
</script>
|
||||
@@ -613,59 +649,55 @@
|
||||
|
||||
<script type="module">
|
||||
const languagesEmojis = {
|
||||
"Unknown": "❓",
|
||||
"Multi": "🌎",
|
||||
"English": "🇬🇧",
|
||||
"Japanese": "🇯🇵",
|
||||
"Chinese": "🇨🇳",
|
||||
"Russian": "🇷🇺",
|
||||
"Arabic": "🇸🇦",
|
||||
"Portuguese": "🇵🇹",
|
||||
"Spanish": "🇪🇸",
|
||||
"French": "🇫🇷",
|
||||
"German": "🇩🇪",
|
||||
"Italian": "🇮🇹",
|
||||
"Korean": "🇰🇷",
|
||||
"Hindi": "🇮🇳",
|
||||
"Bengali": "🇧🇩",
|
||||
"Punjabi": "🇵🇰",
|
||||
"Marathi": "🇮🇳",
|
||||
"Gujarati": "🇮🇳",
|
||||
"Tamil": "🇮🇳",
|
||||
"Telugu": "🇮🇳",
|
||||
"Kannada": "🇮🇳",
|
||||
"Malayalam": "🇮🇳",
|
||||
"Thai": "🇹🇭",
|
||||
"Vietnamese": "🇻🇳",
|
||||
"Indonesian": "🇮🇩",
|
||||
"Turkish": "🇹🇷",
|
||||
"Hebrew": "🇮🇱",
|
||||
"Persian": "🇮🇷",
|
||||
"Ukrainian": "🇺🇦",
|
||||
"Greek": "🇬🇷",
|
||||
"Lithuanian": "🇱🇹",
|
||||
"Latvian": "🇱🇻",
|
||||
"Estonian": "🇪🇪",
|
||||
"Polish": "🇵🇱",
|
||||
"Czech": "🇨🇿",
|
||||
"Slovak": "🇸🇰",
|
||||
"Hungarian": "🇭🇺",
|
||||
"Romanian": "🇷🇴",
|
||||
"Bulgarian": "🇧🇬",
|
||||
"Serbian": "🇷🇸",
|
||||
"Croatian": "🇭🇷",
|
||||
"Slovenian": "🇸🇮",
|
||||
"Dutch": "🇳🇱",
|
||||
"Danish": "🇩🇰",
|
||||
"Finnish": "🇫🇮",
|
||||
"Swedish": "🇸🇪",
|
||||
"Norwegian": "🇳🇴",
|
||||
"Malay": "🇲🇾",
|
||||
"Latino": "💃🏻",
|
||||
"multi": "🌎",
|
||||
"en": "🇬🇧", // English
|
||||
"ja": "🇯🇵", // Japanese
|
||||
"zh": "🇨🇳", // Chinese
|
||||
"ru": "🇷🇺", // Russian
|
||||
"ar": "🇸🇦", // Arabic
|
||||
"pt": "🇵🇹", // Portuguese
|
||||
"es": "🇪🇸", // Spanish
|
||||
"fr": "🇫🇷", // French
|
||||
"de": "🇩🇪", // German
|
||||
"it": "🇮🇹", // Italian
|
||||
"ko": "🇰🇷", // Korean
|
||||
"hi": "🇮🇳", // Hindi
|
||||
"bn": "🇧🇩", // Bengali
|
||||
"pa": "🇵🇰", // Punjabi
|
||||
"mr": "🇮🇳", // Marathi
|
||||
"gu": "🇮🇳", // Gujarati
|
||||
"ta": "🇮🇳", // Tamil
|
||||
"te": "🇮🇳", // Telugu
|
||||
"kn": "🇮🇳", // Kannada
|
||||
"ml": "🇮🇳", // Malayalam
|
||||
"th": "🇹🇭", // Thai
|
||||
"vi": "🇻🇳", // Vietnamese
|
||||
"id": "🇮🇩", // Indonesian
|
||||
"tr": "🇹🇷", // Turkish
|
||||
"he": "🇮🇱", // Hebrew
|
||||
"fa": "🇮🇷", // Persian
|
||||
"uk": "🇺🇦", // Ukrainian
|
||||
"el": "🇬🇷", // Greek
|
||||
"lt": "🇱🇹", // Lithuanian
|
||||
"lv": "🇱🇻", // Latvian
|
||||
"et": "🇪🇪", // Estonian
|
||||
"pl": "🇵🇱", // Polish
|
||||
"cs": "🇨🇿", // Czech
|
||||
"sk": "🇸🇰", // Slovak
|
||||
"hu": "🇭🇺", // Hungarian
|
||||
"ro": "🇷🇴", // Romanian
|
||||
"bg": "🇧🇬", // Bulgarian
|
||||
"sr": "🇷🇸", // Serbian
|
||||
"hr": "🇭🇷", // Croatian
|
||||
"sl": "🇸🇮", // Slovenian
|
||||
"nl": "🇳🇱", // Dutch
|
||||
"da": "🇩🇰", // Danish
|
||||
"fi": "🇫🇮", // Finnish
|
||||
"sv": "🇸🇪", // Swedish
|
||||
"no": "🇳🇴", // Norwegian
|
||||
"ms": "🇲🇾", // Malay
|
||||
"la": "💃🏻", // Latino
|
||||
};
|
||||
|
||||
let defaultLanguages = [];
|
||||
let defaultResolutions = [];
|
||||
let defaultResultFormat = [];
|
||||
|
||||
document.addEventListener("DOMContentLoaded", async () => {
|
||||
@@ -680,14 +712,14 @@
|
||||
]);
|
||||
|
||||
const webConfig = {{webConfig|tojson}};
|
||||
populateSelect("indexers", webConfig.indexers);
|
||||
populateSelect("languages", webConfig.languages);
|
||||
populateSelect("resolutions", webConfig.resolutions);
|
||||
|
||||
const resolutionsSelect = document.getElementById("resolutions");
|
||||
resolutionsSelect.value = ["r2160p", "r1080p", "r720p", "r480p", "r360p", "unknown"];
|
||||
|
||||
populateSelect("resultFormat", webConfig.resultFormat);
|
||||
document.body.classList.add("ready");
|
||||
defaultLanguages = webConfig.languages;
|
||||
defaultResolutions = webConfig.resolutions;
|
||||
defaultResultFormat = webConfig.resultFormat;
|
||||
const resultFormatSelect = document.getElementById("resultFormat");
|
||||
resultFormatSelect.value = defaultResultFormat;
|
||||
|
||||
// try populate the form from previous settings
|
||||
try {
|
||||
@@ -697,28 +729,86 @@
|
||||
console.log("Failed to retrieve or parse settings from URL:", error);
|
||||
}
|
||||
|
||||
// trigger sl-change event on debridService to set initial state
|
||||
const debridServiceSelect = document.getElementById("debridService");
|
||||
debridServiceSelect.dispatchEvent(new Event("sl-change"));
|
||||
|
||||
// Populate language selects
|
||||
populateSelect("languages_required", Object.keys(languagesEmojis));
|
||||
populateSelect("languages_excluded", Object.keys(languagesEmojis));
|
||||
populateSelect("languages_preferred", Object.keys(languagesEmojis));
|
||||
|
||||
document.body.classList.add("ready");
|
||||
});
|
||||
|
||||
function populateSelect(selectId, options) {
|
||||
const selectElement = document.getElementById(selectId);
|
||||
|
||||
const languageNames = {
|
||||
"multi": "Multi",
|
||||
"en": "English",
|
||||
"ja": "Japanese",
|
||||
"zh": "Chinese",
|
||||
"ru": "Russian",
|
||||
"ar": "Arabic",
|
||||
"pt": "Portuguese",
|
||||
"es": "Spanish",
|
||||
"fr": "French",
|
||||
"de": "German",
|
||||
"it": "Italian",
|
||||
"ko": "Korean",
|
||||
"hi": "Hindi",
|
||||
"bn": "Bengali",
|
||||
"pa": "Punjabi",
|
||||
"mr": "Marathi",
|
||||
"gu": "Gujarati",
|
||||
"ta": "Tamil",
|
||||
"te": "Telugu",
|
||||
"kn": "Kannada",
|
||||
"ml": "Malayalam",
|
||||
"th": "Thai",
|
||||
"vi": "Vietnamese",
|
||||
"id": "Indonesian",
|
||||
"tr": "Turkish",
|
||||
"he": "Hebrew",
|
||||
"fa": "Persian",
|
||||
"uk": "Ukrainian",
|
||||
"el": "Greek",
|
||||
"lt": "Lithuanian",
|
||||
"lv": "Latvian",
|
||||
"et": "Estonian",
|
||||
"pl": "Polish",
|
||||
"cs": "Czech",
|
||||
"sk": "Slovak",
|
||||
"hu": "Hungarian",
|
||||
"ro": "Romanian",
|
||||
"bg": "Bulgarian",
|
||||
"sr": "Serbian",
|
||||
"hr": "Croatian",
|
||||
"sl": "Slovenian",
|
||||
"nl": "Dutch",
|
||||
"da": "Danish",
|
||||
"fi": "Finnish",
|
||||
"sv": "Swedish",
|
||||
"no": "Norwegian",
|
||||
"ms": "Malay",
|
||||
"la": "Latino"
|
||||
};
|
||||
|
||||
options.forEach(option => {
|
||||
const optionElement = document.createElement("sl-option");
|
||||
optionElement.value = option;
|
||||
|
||||
if (selectId === "languages") {
|
||||
// For languages, prepend the flag emoji if it exists
|
||||
if (selectId.startsWith('languages')) {
|
||||
const flag = languagesEmojis[option] || '';
|
||||
optionElement.textContent = `${flag} ${option}`;
|
||||
const fullName = languageNames[option] || option;
|
||||
optionElement.textContent = `${flag} ${fullName}`;
|
||||
} else {
|
||||
// For other selects, just use the option text as is
|
||||
optionElement.textContent = option;
|
||||
}
|
||||
|
||||
selectElement.appendChild(optionElement);
|
||||
});
|
||||
|
||||
// Set the default value
|
||||
selectElement.value = options; // Assuming first option as default
|
||||
}
|
||||
|
||||
const installButton = document.querySelector("#install");
|
||||
@@ -727,37 +817,52 @@
|
||||
const copyAlert = document.querySelector('#copyAlert');
|
||||
|
||||
function getSettings() {
|
||||
const indexers = Array.from(document.getElementById("indexers").selectedOptions).map(option => option.value);
|
||||
const languages = Array.from(document.getElementById("languages").selectedOptions).map(option => option.value);
|
||||
const resolutions = Array.from(document.getElementById("resolutions").selectedOptions).map(option => option.value);
|
||||
const maxResults = document.getElementById("maxResults").value;
|
||||
const languages_required = Array.from(document.getElementById("languages_required").selectedOptions).map(option => option.value);
|
||||
const languages_excluded = Array.from(document.getElementById("languages_excluded").selectedOptions).map(option => option.value);
|
||||
const languages_preferred = Array.from(document.getElementById("languages_preferred").selectedOptions).map(option => option.value);
|
||||
const selectedResolutions = Array.from(document.getElementById("resolutions").selectedOptions).map(option => option.value);
|
||||
const allResolutions = ["r2160p", "r1080p", "r720p", "r480p", "r360p", "unknown"];
|
||||
const resolutions = allResolutions.reduce((obj, res) => {
|
||||
// Only include resolutions that are NOT selected (disabled) as false
|
||||
if (!selectedResolutions.includes(res)) {
|
||||
obj[res] = false;
|
||||
}
|
||||
return obj;
|
||||
}, {});
|
||||
const rankThreshold = document.getElementById("rankThreshold").value;
|
||||
const maxResultsPerResolution = document.getElementById("maxResultsPerResolution").value;
|
||||
const maxSize = document.getElementById("maxSize").value;
|
||||
const reverseResultOrder = document.getElementById("reverseResultOrder").checked;
|
||||
const cachedOnly = document.getElementById("cachedOnly").checked;
|
||||
const removeTrash = document.getElementById("removeTrash").checked;
|
||||
const allowEnglishInLanguages = document.getElementById("allowEnglishInLanguages").checked;
|
||||
const removeUnknownLanguages = document.getElementById("removeUnknownLanguages").checked;
|
||||
const resultFormat = Array.from(document.getElementById("resultFormat").selectedOptions).map(option => option.value);
|
||||
const debridService = document.getElementById("debridService").value;
|
||||
const debridApiKey = document.getElementById("debridApiKey").value;
|
||||
const stremthruUrl = document.getElementById("stremthruUrl").value;
|
||||
const debridStreamProxyPassword = document.getElementById("debridStreamProxyPassword").value;
|
||||
const selectedLanguages = languages.length === defaultLanguages.length && languages.every((val, index) => val === defaultLanguages[index]) ? ["All"] : languages;
|
||||
const selectedResolutions = resolutions.length === defaultResolutions.length && resolutions.every((val, index) => val === defaultResolutions[index]) ? ["All"] : resolutions;
|
||||
const selectedResultFormat = resultFormat.length === defaultResultFormat.length && resultFormat.every((val, index) => val === defaultResultFormat[index]) ? ["All"] : resultFormat;
|
||||
|
||||
const selectedResultFormat = resultFormat.length === defaultResultFormat.length && resultFormat.every((val, index) => val === defaultResultFormat[index]) ? ["all"] : resultFormat;
|
||||
|
||||
return {
|
||||
indexers: indexers,
|
||||
maxResults: parseInt(maxResults),
|
||||
maxResultsPerResolution: parseInt(maxResultsPerResolution),
|
||||
maxSize: parseFloat(maxSize * 1073741824),
|
||||
reverseResultOrder: reverseResultOrder,
|
||||
cachedOnly: cachedOnly,
|
||||
removeTrash: removeTrash,
|
||||
resultFormat: selectedResultFormat,
|
||||
resolutions: selectedResolutions,
|
||||
languages: selectedLanguages,
|
||||
debridService: debridService,
|
||||
debridApiKey: debridApiKey,
|
||||
stremthruUrl: stremthruUrl,
|
||||
debridStreamProxyPassword: debridStreamProxyPassword,
|
||||
languages: {
|
||||
required: languages_required,
|
||||
exclude: languages_excluded,
|
||||
preferred: languages_preferred
|
||||
},
|
||||
resolutions: resolutions,
|
||||
options: {
|
||||
remove_ranks_under: parseFloat(rankThreshold),
|
||||
allow_english_in_languages: allowEnglishInLanguages,
|
||||
remove_unknown_languages: removeUnknownLanguages
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -802,10 +907,8 @@
|
||||
}
|
||||
|
||||
function populateFormFromSettings(settings) {
|
||||
if (settings.maxResults !== null)
|
||||
document.getElementById("maxResults").value = settings.maxResults;
|
||||
if (settings.reverseResultOrder !== null)
|
||||
document.getElementById("reverseResultOrder").checked = settings.reverseResultOrder;
|
||||
if (settings.cachedOnly !== null)
|
||||
document.getElementById("cachedOnly").checked = settings.cachedOnly;
|
||||
if (settings.removeTrash !== null)
|
||||
document.getElementById("removeTrash").checked = settings.removeTrash;
|
||||
if (settings.maxResultsPerResolution !== null)
|
||||
@@ -818,18 +921,28 @@
|
||||
document.getElementById("debridApiKey").value = settings.debridApiKey;
|
||||
if (settings.debridStreamProxyPassword !== null)
|
||||
document.getElementById("debridStreamProxyPassword").value = settings.debridStreamProxyPassword;
|
||||
if (settings.indexers !== null)
|
||||
document.getElementById("indexers").value = settings.indexers;
|
||||
if (settings.languages !== null && settings.languages != "All")
|
||||
document.getElementById("languages").value = settings.languages;
|
||||
if (settings.stremthruUrl !== null)
|
||||
document.getElementById("stremthruUrl").value = settings.stremthruUrl;
|
||||
if (settings.resolutions !== null && settings.resolutions != "All")
|
||||
document.getElementById("resolutions").value = settings.resolutions;
|
||||
if (settings.resultFormat !== null && settings.resultFormat != "All")
|
||||
if (settings.resultFormat !== null && settings.resultFormat != "all")
|
||||
document.getElementById("resultFormat").value = settings.resultFormat;
|
||||
if (settings.languages) {
|
||||
if (settings.languages.required)
|
||||
document.getElementById("languages_required").value = settings.languages.required;
|
||||
if (settings.languages.exclude)
|
||||
document.getElementById("languages_excluded").value = settings.languages.exclude;
|
||||
if (settings.languages.preferred)
|
||||
document.getElementById("languages_preferred").value = settings.languages.preferred;
|
||||
}
|
||||
if (settings.resolutions) {
|
||||
const allResolutions = ["r2160p", "r1080p", "r720p", "r480p", "r360p", "unknown"];
|
||||
const enabledResolutions = allResolutions.filter(res => settings.resolutions[res] !== false);
|
||||
document.getElementById("resolutions").value = enabledResolutions;
|
||||
}
|
||||
if (settings.options?.remove_ranks_under !== undefined)
|
||||
document.getElementById("rankThreshold").value = settings.options.remove_ranks_under;
|
||||
if (settings.options?.allow_english_in_languages !== undefined)
|
||||
document.getElementById("allowEnglishInLanguages").checked = settings.options.allow_english_in_languages;
|
||||
if (settings.options?.remove_unknown_languages !== undefined)
|
||||
document.getElementById("removeUnknownLanguages").checked = settings.options.remove_unknown_languages;
|
||||
}
|
||||
|
||||
</script>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
|
||||
from comet.utils.logger import logger
|
||||
from comet.utils.models import database, settings
|
||||
|
||||
DATABASE_VERSION = "1.0"
|
||||
|
||||
async def setup_database():
|
||||
try:
|
||||
if settings.DATABASE_TYPE == "sqlite":
|
||||
os.makedirs(os.path.dirname(settings.DATABASE_PATH), exist_ok=True)
|
||||
|
||||
if not os.path.exists(settings.DATABASE_PATH):
|
||||
open(settings.DATABASE_PATH, "a").close()
|
||||
|
||||
await database.connect()
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS db_version (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
version TEXT
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
current_version = await database.fetch_val(
|
||||
"""
|
||||
SELECT version FROM db_version WHERE id = 1
|
||||
"""
|
||||
)
|
||||
|
||||
if current_version != DATABASE_VERSION:
|
||||
logger.log("COMET", f"Database: Migration from {current_version} to {DATABASE_VERSION} version")
|
||||
|
||||
if settings.DATABASE_TYPE == "sqlite":
|
||||
tables = await database.fetch_all(
|
||||
"""
|
||||
SELECT name FROM sqlite_master
|
||||
WHERE type='table' AND name != 'db_version' AND name != 'sqlite_sequence'
|
||||
"""
|
||||
)
|
||||
|
||||
for table in tables:
|
||||
await database.execute(f"DROP TABLE IF EXISTS {table['name']}")
|
||||
else:
|
||||
await database.execute(
|
||||
"""
|
||||
DO $$ DECLARE
|
||||
r RECORD;
|
||||
BEGIN
|
||||
FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema() AND tablename != 'db_version') LOOP
|
||||
EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE';
|
||||
END LOOP;
|
||||
END $$;
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
INSERT INTO db_version VALUES (1, :version)
|
||||
ON CONFLICT (id) DO UPDATE SET version = :version
|
||||
""",
|
||||
{"version": DATABASE_VERSION},
|
||||
)
|
||||
|
||||
logger.log("COMET", f"Database: Migration to version {DATABASE_VERSION} completed")
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS ongoing_searches (
|
||||
media_id TEXT PRIMARY KEY,
|
||||
timestamp INTEGER
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS first_searches (
|
||||
media_id TEXT PRIMARY KEY,
|
||||
timestamp INTEGER
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS metadata_cache (
|
||||
media_id TEXT PRIMARY KEY,
|
||||
title TEXT,
|
||||
year INTEGER,
|
||||
year_end INTEGER,
|
||||
aliases TEXT,
|
||||
timestamp INTEGER
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS torrents (
|
||||
media_id TEXT,
|
||||
info_hash TEXT,
|
||||
file_index INTEGER,
|
||||
season INTEGER,
|
||||
episode INTEGER,
|
||||
title TEXT,
|
||||
seeders INTEGER,
|
||||
size BIGINT,
|
||||
tracker TEXT,
|
||||
sources TEXT,
|
||||
parsed TEXT,
|
||||
timestamp INTEGER
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS torrents_series_both_idx
|
||||
ON torrents (media_id, info_hash, season, episode)
|
||||
WHERE season IS NOT NULL AND episode IS NOT NULL
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS torrents_season_only_idx
|
||||
ON torrents (media_id, info_hash, season)
|
||||
WHERE season IS NOT NULL AND episode IS NULL
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS torrents_episode_only_idx
|
||||
ON torrents (media_id, info_hash, episode)
|
||||
WHERE season IS NULL AND episode IS NOT NULL
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS torrents_no_season_episode_idx
|
||||
ON torrents (media_id, info_hash)
|
||||
WHERE season IS NULL AND episode IS NULL
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS debrid_availability (
|
||||
debrid_service TEXT,
|
||||
info_hash TEXT,
|
||||
file_index TEXT,
|
||||
title TEXT,
|
||||
season INTEGER,
|
||||
episode INTEGER,
|
||||
size BIGINT,
|
||||
parsed TEXT,
|
||||
timestamp INTEGER
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS debrid_series_both_idx
|
||||
ON debrid_availability (debrid_service, info_hash, season, episode)
|
||||
WHERE season IS NOT NULL AND episode IS NOT NULL
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS debrid_season_only_idx
|
||||
ON debrid_availability (debrid_service, info_hash, season)
|
||||
WHERE season IS NOT NULL AND episode IS NULL
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS debrid_episode_only_idx
|
||||
ON debrid_availability (debrid_service, info_hash, episode)
|
||||
WHERE season IS NULL AND episode IS NOT NULL
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS debrid_no_season_episode_idx
|
||||
ON debrid_availability (debrid_service, info_hash)
|
||||
WHERE season IS NULL AND episode IS NULL
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS download_links_cache (
|
||||
debrid_key TEXT,
|
||||
info_hash TEXT,
|
||||
season INTEGER,
|
||||
episode INTEGER,
|
||||
download_url TEXT,
|
||||
timestamp INTEGER
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS download_links_series_both_idx
|
||||
ON download_links_cache (debrid_key, info_hash, season, episode)
|
||||
WHERE season IS NOT NULL AND episode IS NOT NULL
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS download_links_season_only_idx
|
||||
ON download_links_cache (debrid_key, info_hash, season)
|
||||
WHERE season IS NOT NULL AND episode IS NULL
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS download_links_episode_only_idx
|
||||
ON download_links_cache (debrid_key, info_hash, episode)
|
||||
WHERE season IS NULL AND episode IS NOT NULL
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS download_links_no_season_episode_idx
|
||||
ON download_links_cache (debrid_key, info_hash)
|
||||
WHERE season IS NULL AND episode IS NULL
|
||||
"""
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS active_connections (
|
||||
id TEXT PRIMARY KEY,
|
||||
ip TEXT,
|
||||
content TEXT,
|
||||
timestamp INTEGER
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
if settings.DATABASE_TYPE == "sqlite":
|
||||
await database.execute("PRAGMA busy_timeout=30000") # 30 seconds timeout
|
||||
await database.execute("PRAGMA journal_mode=OFF")
|
||||
await database.execute("PRAGMA synchronous=OFF")
|
||||
await database.execute("PRAGMA temp_store=MEMORY")
|
||||
await database.execute("PRAGMA mmap_size=30000000000")
|
||||
await database.execute("PRAGMA page_size=4096")
|
||||
await database.execute("PRAGMA cache_size=-2000")
|
||||
await database.execute("PRAGMA foreign_keys=OFF")
|
||||
await database.execute("PRAGMA count_changes=OFF")
|
||||
await database.execute("PRAGMA secure_delete=OFF")
|
||||
await database.execute("PRAGMA auto_vacuum=OFF")
|
||||
|
||||
await database.execute("DELETE FROM ongoing_searches")
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
DELETE FROM first_searches
|
||||
WHERE timestamp + :cache_ttl < :current_time;
|
||||
""",
|
||||
{"cache_ttl": settings.TORRENT_CACHE_TTL, "current_time": time.time()},
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
DELETE FROM metadata_cache
|
||||
WHERE timestamp + :cache_ttl < :current_time;
|
||||
""",
|
||||
{"cache_ttl": settings.METADATA_CACHE_TTL, "current_time": time.time()},
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
DELETE FROM torrents
|
||||
WHERE timestamp + :cache_ttl < :current_time;
|
||||
""",
|
||||
{"cache_ttl": settings.TORRENT_CACHE_TTL, "current_time": time.time()},
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
"""
|
||||
DELETE FROM debrid_availability
|
||||
WHERE timestamp + :cache_ttl < :current_time;
|
||||
""",
|
||||
{"cache_ttl": settings.DEBRID_CACHE_TTL, "current_time": time.time()},
|
||||
)
|
||||
|
||||
await database.execute("DELETE FROM download_links_cache")
|
||||
|
||||
await database.execute("DELETE FROM active_connections")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error setting up the database: {e}")
|
||||
logger.exception(traceback.format_exc())
|
||||
|
||||
|
||||
async def teardown_database():
|
||||
try:
|
||||
await database.disconnect()
|
||||
except Exception as e:
|
||||
logger.error(f"Error tearing down the database: {e}")
|
||||
logger.exception(traceback.format_exc())
|
||||
@@ -0,0 +1,173 @@
|
||||
import time
|
||||
import orjson
|
||||
|
||||
from comet.utils.models import settings, database
|
||||
from comet.utils.general import default_dump
|
||||
|
||||
|
||||
async def cache_availability(debrid_service: str, availability: list):
|
||||
current_time = time.time()
|
||||
|
||||
values = [
|
||||
{
|
||||
"debrid_service": debrid_service,
|
||||
"info_hash": file["info_hash"],
|
||||
"file_index": str(file["index"]) if file["index"] is not None else None,
|
||||
"title": file["title"],
|
||||
"season": file["season"],
|
||||
"episode": file["episode"],
|
||||
"size": file["size"] if file["index"] is not None else None,
|
||||
"parsed": orjson.dumps(file["parsed"], default_dump).decode("utf-8")
|
||||
if file["parsed"] is not None
|
||||
else None,
|
||||
"timestamp": current_time,
|
||||
}
|
||||
for file in availability
|
||||
]
|
||||
|
||||
if settings.DATABASE_TYPE == "sqlite":
|
||||
query = """
|
||||
INSERT OR REPLACE
|
||||
INTO debrid_availability
|
||||
VALUES (:debrid_service, :info_hash, :file_index, :title, :season, :episode, :size, :parsed, :timestamp)
|
||||
"""
|
||||
await database.execute_many(query, values)
|
||||
elif settings.DATABASE_TYPE == "postgresql":
|
||||
both_values = []
|
||||
season_only_values = []
|
||||
episode_only_values = []
|
||||
no_season_episode_values = []
|
||||
|
||||
for val in values:
|
||||
if val["season"] is not None and val["episode"] is not None:
|
||||
both_values.append(val)
|
||||
elif val["season"] is not None and val["episode"] is None:
|
||||
season_only_values.append(val)
|
||||
elif val["season"] is None and val["episode"] is not None:
|
||||
episode_only_values.append(val)
|
||||
else:
|
||||
no_season_episode_values.append(val)
|
||||
|
||||
# handle each case separately with appropriate ON CONFLICT clauses
|
||||
if both_values:
|
||||
query = """
|
||||
INSERT INTO debrid_availability
|
||||
VALUES (:debrid_service, :info_hash, :file_index, :title, :season, :episode, :size, :parsed, :timestamp)
|
||||
ON CONFLICT (debrid_service, info_hash, season, episode)
|
||||
WHERE season IS NOT NULL AND episode IS NOT NULL
|
||||
DO UPDATE SET
|
||||
title = EXCLUDED.title,
|
||||
file_index = EXCLUDED.file_index,
|
||||
size = EXCLUDED.size,
|
||||
parsed = EXCLUDED.parsed,
|
||||
timestamp = EXCLUDED.timestamp
|
||||
"""
|
||||
await database.execute_many(query, both_values)
|
||||
|
||||
if season_only_values:
|
||||
query = """
|
||||
INSERT INTO debrid_availability
|
||||
VALUES (:debrid_service, :info_hash, :file_index, :title, :season, :episode, :size, :parsed, :timestamp)
|
||||
ON CONFLICT (debrid_service, info_hash, season)
|
||||
WHERE season IS NOT NULL AND episode IS NULL
|
||||
DO UPDATE SET
|
||||
title = EXCLUDED.title,
|
||||
file_index = EXCLUDED.file_index,
|
||||
size = EXCLUDED.size,
|
||||
parsed = EXCLUDED.parsed,
|
||||
timestamp = EXCLUDED.timestamp
|
||||
"""
|
||||
await database.execute_many(query, season_only_values)
|
||||
|
||||
if episode_only_values:
|
||||
query = """
|
||||
INSERT INTO debrid_availability
|
||||
VALUES (:debrid_service, :info_hash, :file_index, :title, :season, :episode, :size, :parsed, :timestamp)
|
||||
ON CONFLICT (debrid_service, info_hash, episode)
|
||||
WHERE season IS NULL AND episode IS NOT NULL
|
||||
DO UPDATE SET
|
||||
title = EXCLUDED.title,
|
||||
file_index = EXCLUDED.file_index,
|
||||
size = EXCLUDED.size,
|
||||
parsed = EXCLUDED.parsed,
|
||||
timestamp = EXCLUDED.timestamp
|
||||
"""
|
||||
await database.execute_many(query, episode_only_values)
|
||||
|
||||
if no_season_episode_values:
|
||||
query = """
|
||||
INSERT INTO debrid_availability
|
||||
VALUES (:debrid_service, :info_hash, :file_index, :title, :season, :episode, :size, :parsed, :timestamp)
|
||||
ON CONFLICT (debrid_service, info_hash)
|
||||
WHERE season IS NULL AND episode IS NULL
|
||||
DO UPDATE SET
|
||||
title = EXCLUDED.title,
|
||||
file_index = EXCLUDED.file_index,
|
||||
size = EXCLUDED.size,
|
||||
parsed = EXCLUDED.parsed,
|
||||
timestamp = EXCLUDED.timestamp
|
||||
"""
|
||||
await database.execute_many(query, no_season_episode_values)
|
||||
else:
|
||||
query = """
|
||||
INSERT
|
||||
INTO debrid_availability
|
||||
VALUES (:debrid_service, :info_hash, :file_index, :title, :season, :episode, :size, :parsed, :timestamp)
|
||||
"""
|
||||
await database.execute_many(query, values)
|
||||
|
||||
|
||||
async def get_cached_availability(
|
||||
debrid_service: str, info_hashes: list, season: int = None, episode: int = None
|
||||
):
|
||||
base_query = f"""
|
||||
SELECT info_hash, file_index, title, size, parsed
|
||||
FROM debrid_availability
|
||||
WHERE info_hash IN (SELECT cast(value as TEXT) FROM {"json_array_elements_text" if settings.DATABASE_TYPE == "postgresql" else "json_each"}(:info_hashes))
|
||||
AND debrid_service = :debrid_service
|
||||
AND timestamp + :cache_ttl >= :current_time
|
||||
"""
|
||||
|
||||
params = {
|
||||
"info_hashes": orjson.dumps(info_hashes).decode("utf-8"),
|
||||
"debrid_service": debrid_service,
|
||||
"cache_ttl": settings.DEBRID_CACHE_TTL,
|
||||
"current_time": time.time(),
|
||||
"season": season,
|
||||
"episode": episode,
|
||||
}
|
||||
|
||||
if debrid_service == "offcloud":
|
||||
query = (
|
||||
base_query
|
||||
+ """
|
||||
AND ((cast(:season as INTEGER) IS NULL AND season IS NULL) OR season = cast(:season as INTEGER))
|
||||
AND ((cast(:episode as INTEGER) IS NULL AND episode IS NULL) OR episode = cast(:episode as INTEGER))
|
||||
"""
|
||||
)
|
||||
results = await database.fetch_all(query, params)
|
||||
|
||||
found_hashes = {r["info_hash"] for r in results}
|
||||
remaining_hashes = [h for h in info_hashes if h not in found_hashes]
|
||||
|
||||
if remaining_hashes:
|
||||
null_title_params = {
|
||||
"info_hashes": orjson.dumps(remaining_hashes).decode("utf-8"),
|
||||
"debrid_service": debrid_service,
|
||||
"cache_ttl": settings.DEBRID_CACHE_TTL,
|
||||
"current_time": time.time(),
|
||||
}
|
||||
null_title_query = base_query + " AND title IS NULL"
|
||||
null_results = await database.fetch_all(null_title_query, null_title_params)
|
||||
results.extend(null_results)
|
||||
else:
|
||||
query = (
|
||||
base_query
|
||||
+ """
|
||||
AND ((cast(:season as INTEGER) IS NULL AND season IS NULL) OR season = cast(:season as INTEGER))
|
||||
AND ((cast(:episode as INTEGER) IS NULL AND episode IS NULL) OR episode = cast(:episode as INTEGER))
|
||||
"""
|
||||
)
|
||||
results = await database.fetch_all(query, params)
|
||||
|
||||
return results
|
||||
+175
-653
@@ -1,20 +1,101 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import re
|
||||
import aiohttp
|
||||
import bencodepy
|
||||
import PTT
|
||||
import asyncio
|
||||
import orjson
|
||||
import time
|
||||
import copy
|
||||
|
||||
from RTN import parse, title_match
|
||||
from curl_cffi import requests
|
||||
from RTN import ParsedData
|
||||
from fastapi import Request
|
||||
|
||||
from comet.utils.logger import logger
|
||||
from comet.utils.models import database, settings, ConfigModel
|
||||
from comet.utils.models import (
|
||||
ConfigModel,
|
||||
default_config,
|
||||
settings,
|
||||
rtn_settings_default,
|
||||
rtn_ranking_default,
|
||||
)
|
||||
|
||||
|
||||
def config_check(b64config: str):
|
||||
try:
|
||||
config = orjson.loads(base64.b64decode(b64config).decode())
|
||||
|
||||
if "indexers" in config:
|
||||
return False
|
||||
|
||||
validated_config = ConfigModel(**config)
|
||||
validated_config = validated_config.model_dump()
|
||||
|
||||
for key in list(validated_config["options"].keys()):
|
||||
if key not in [
|
||||
"remove_ranks_under",
|
||||
"allow_english_in_languages",
|
||||
"remove_unknown_languages",
|
||||
]:
|
||||
validated_config["options"].pop(key)
|
||||
|
||||
validated_config["options"]["remove_all_trash"] = validated_config[
|
||||
"removeTrash"
|
||||
]
|
||||
|
||||
rtn_settings = rtn_settings_default.model_copy(
|
||||
update={
|
||||
"resolutions": rtn_settings_default.resolutions.model_copy(
|
||||
update=validated_config["resolutions"]
|
||||
),
|
||||
"options": rtn_settings_default.options.model_copy(
|
||||
update=validated_config["options"]
|
||||
),
|
||||
"languages": rtn_settings_default.languages.model_copy(
|
||||
update=validated_config["languages"]
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
validated_config["rtnSettings"] = rtn_settings
|
||||
validated_config["rtnRanking"] = rtn_ranking_default
|
||||
|
||||
if (
|
||||
settings.PROXY_DEBRID_STREAM
|
||||
and settings.PROXY_DEBRID_STREAM_PASSWORD
|
||||
== validated_config["debridStreamProxyPassword"]
|
||||
and validated_config["debridApiKey"] == ""
|
||||
):
|
||||
validated_config["debridService"] = (
|
||||
settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE
|
||||
)
|
||||
validated_config["debridApiKey"] = (
|
||||
settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY
|
||||
)
|
||||
|
||||
return validated_config
|
||||
except Exception:
|
||||
return default_config # if it doesn't pass, return default config
|
||||
|
||||
|
||||
def bytes_to_size(bytes: int):
|
||||
sizes = ["Bytes", "KB", "MB", "GB", "TB"]
|
||||
if bytes == 0:
|
||||
return "0 Byte"
|
||||
|
||||
i = 0
|
||||
while bytes >= 1024 and i < len(sizes) - 1:
|
||||
bytes /= 1024
|
||||
i += 1
|
||||
|
||||
return f"{round(bytes, 2)} {sizes[i]}"
|
||||
|
||||
|
||||
def size_to_bytes(size_str: str):
|
||||
sizes = ["b", "kb", "mb", "gb", "tb"]
|
||||
|
||||
value, unit = size_str.split()
|
||||
value = float(value)
|
||||
unit = unit.lower()
|
||||
|
||||
if unit not in sizes:
|
||||
return None
|
||||
|
||||
multiplier = 1024 ** sizes.index(unit)
|
||||
return int(value * multiplier)
|
||||
|
||||
|
||||
languages_emojis = {
|
||||
"unknown": "❓", # Unknown
|
||||
@@ -78,104 +159,78 @@ def get_language_emoji(language: str):
|
||||
)
|
||||
|
||||
|
||||
translation_table = {
|
||||
"ā": "a",
|
||||
"ă": "a",
|
||||
"ą": "a",
|
||||
"ć": "c",
|
||||
"č": "c",
|
||||
"ç": "c",
|
||||
"ĉ": "c",
|
||||
"ċ": "c",
|
||||
"ď": "d",
|
||||
"đ": "d",
|
||||
"è": "e",
|
||||
"é": "e",
|
||||
"ê": "e",
|
||||
"ë": "e",
|
||||
"ē": "e",
|
||||
"ĕ": "e",
|
||||
"ę": "e",
|
||||
"ě": "e",
|
||||
"ĝ": "g",
|
||||
"ğ": "g",
|
||||
"ġ": "g",
|
||||
"ģ": "g",
|
||||
"ĥ": "h",
|
||||
"î": "i",
|
||||
"ï": "i",
|
||||
"ì": "i",
|
||||
"í": "i",
|
||||
"ī": "i",
|
||||
"ĩ": "i",
|
||||
"ĭ": "i",
|
||||
"ı": "i",
|
||||
"ĵ": "j",
|
||||
"ķ": "k",
|
||||
"ĺ": "l",
|
||||
"ļ": "l",
|
||||
"ł": "l",
|
||||
"ń": "n",
|
||||
"ň": "n",
|
||||
"ñ": "n",
|
||||
"ņ": "n",
|
||||
"ʼn": "n",
|
||||
"ó": "o",
|
||||
"ô": "o",
|
||||
"õ": "o",
|
||||
"ö": "o",
|
||||
"ø": "o",
|
||||
"ō": "o",
|
||||
"ő": "o",
|
||||
"œ": "oe",
|
||||
"ŕ": "r",
|
||||
"ř": "r",
|
||||
"ŗ": "r",
|
||||
"š": "s",
|
||||
"ş": "s",
|
||||
"ś": "s",
|
||||
"ș": "s",
|
||||
"ß": "ss",
|
||||
"ť": "t",
|
||||
"ţ": "t",
|
||||
"ū": "u",
|
||||
"ŭ": "u",
|
||||
"ũ": "u",
|
||||
"û": "u",
|
||||
"ü": "u",
|
||||
"ù": "u",
|
||||
"ú": "u",
|
||||
"ų": "u",
|
||||
"ű": "u",
|
||||
"ŵ": "w",
|
||||
"ý": "y",
|
||||
"ÿ": "y",
|
||||
"ŷ": "y",
|
||||
"ž": "z",
|
||||
"ż": "z",
|
||||
"ź": "z",
|
||||
"æ": "ae",
|
||||
"ǎ": "a",
|
||||
"ǧ": "g",
|
||||
"ə": "e",
|
||||
"ƒ": "f",
|
||||
"ǐ": "i",
|
||||
"ǒ": "o",
|
||||
"ǔ": "u",
|
||||
"ǚ": "u",
|
||||
"ǜ": "u",
|
||||
"ǹ": "n",
|
||||
"ǻ": "a",
|
||||
"ǽ": "ae",
|
||||
"ǿ": "o",
|
||||
}
|
||||
def format_metadata(data: ParsedData):
|
||||
extras = []
|
||||
if data.quality:
|
||||
extras.append(data.quality)
|
||||
if data.hdr:
|
||||
extras.extend(data.hdr)
|
||||
if data.codec:
|
||||
extras.append(data.codec)
|
||||
if data.audio:
|
||||
extras.extend(data.audio)
|
||||
if data.channels:
|
||||
extras.extend(data.channels)
|
||||
if data.bit_depth:
|
||||
extras.append(data.bit_depth)
|
||||
if data.network:
|
||||
extras.append(data.network)
|
||||
if data.group:
|
||||
extras.append(data.group)
|
||||
|
||||
translation_table = str.maketrans(translation_table)
|
||||
info_hash_pattern = re.compile(r"\b([a-fA-F0-9]{40})\b")
|
||||
return "|".join(extras)
|
||||
|
||||
|
||||
def translate(title: str):
|
||||
return title.translate(translation_table)
|
||||
def format_title(
|
||||
data: ParsedData,
|
||||
ttitle: str,
|
||||
seeders: int,
|
||||
size: int,
|
||||
tracker: str,
|
||||
result_format: list,
|
||||
):
|
||||
has_all = "all" in result_format
|
||||
|
||||
title = ""
|
||||
if has_all or "title" in result_format:
|
||||
title += f"{ttitle}\n"
|
||||
|
||||
if has_all or "metadata" in result_format:
|
||||
metadata = format_metadata(data)
|
||||
if metadata != "":
|
||||
title += f"💿 {metadata}\n"
|
||||
|
||||
if (has_all or "seeders" in result_format) and seeders is not None:
|
||||
title += f"👤 {seeders} "
|
||||
|
||||
if has_all or "size" in result_format:
|
||||
title += f"💾 {bytes_to_size(size)} "
|
||||
|
||||
if has_all or "tracker" in result_format:
|
||||
title += f"🔎 {tracker}"
|
||||
|
||||
if has_all or "languages" in result_format:
|
||||
languages = data.languages
|
||||
if languages:
|
||||
formatted_languages = "/".join(
|
||||
get_language_emoji(language) for language in languages
|
||||
)
|
||||
languages_str = "\n" + formatted_languages
|
||||
title += f"{languages_str}"
|
||||
|
||||
if title == "":
|
||||
# Without this, Streamio shows SD as the result, which is confusing
|
||||
title = "Empty result format configuration"
|
||||
|
||||
return title
|
||||
|
||||
|
||||
def get_client_ip(request: Request):
|
||||
return (
|
||||
request.headers["cf-connecting-ip"]
|
||||
if "cf-connecting-ip" in request.headers
|
||||
else request.client.host
|
||||
)
|
||||
|
||||
|
||||
def is_video(title: str):
|
||||
@@ -221,553 +276,20 @@ def is_video(title: str):
|
||||
return title.endswith(video_extensions)
|
||||
|
||||
|
||||
def bytes_to_size(bytes: int):
|
||||
sizes = ["Bytes", "KB", "MB", "GB", "TB"]
|
||||
if bytes == 0:
|
||||
return "0 Byte"
|
||||
def default_dump(obj):
|
||||
if isinstance(obj, ParsedData):
|
||||
return obj.model_dump()
|
||||
|
||||
i = 0
|
||||
while bytes >= 1024 and i < len(sizes) - 1:
|
||||
bytes /= 1024
|
||||
i += 1
|
||||
|
||||
return f"{round(bytes, 2)} {sizes[i]}"
|
||||
def parse_media_id(media_type: str, media_id: str):
|
||||
if media_type == "series":
|
||||
info = media_id.split(":")
|
||||
|
||||
if "kitsu" in media_id:
|
||||
return info[1], 1, int(info[2])
|
||||
|
||||
def size_to_bytes(size_str: str):
|
||||
sizes = ["bytes", "kb", "mb", "gb", "tb"]
|
||||
try:
|
||||
value, unit = size_str.split()
|
||||
value = float(value)
|
||||
unit = unit.lower()
|
||||
|
||||
if unit not in sizes:
|
||||
return None
|
||||
|
||||
multiplier = 1024 ** sizes.index(unit)
|
||||
return int(value * multiplier)
|
||||
except:
|
||||
return None
|
||||
|
||||
|
||||
def config_check(b64config: str):
|
||||
try:
|
||||
config = orjson.loads(base64.b64decode(b64config).decode())
|
||||
validated_config = ConfigModel(**config)
|
||||
return validated_config.model_dump()
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
def get_debrid_extension(debridService: str, debridApiKey: str = None):
|
||||
if debridApiKey == "":
|
||||
return "TORRENT"
|
||||
|
||||
debrid_extensions = {
|
||||
"realdebrid": "RD",
|
||||
"alldebrid": "AD",
|
||||
"premiumize": "PM",
|
||||
"torbox": "TB",
|
||||
"debridlink": "DL",
|
||||
"easydebrid": "ED",
|
||||
"stremthru": "ST",
|
||||
}
|
||||
|
||||
extension = debrid_extensions.get(debridService, None)
|
||||
if extension == "ST" and debridApiKey and ":" in debridApiKey:
|
||||
ext = debrid_extensions[debridApiKey.split(":")[0]]
|
||||
return f"{extension}({ext})" if extension != ext else extension
|
||||
return extension
|
||||
|
||||
|
||||
async def get_indexer_manager(
|
||||
session: aiohttp.ClientSession,
|
||||
indexer_manager_type: str,
|
||||
indexers: list,
|
||||
query: str,
|
||||
):
|
||||
results = []
|
||||
try:
|
||||
indexers = [indexer.replace("_", " ") for indexer in indexers]
|
||||
|
||||
if indexer_manager_type == "jackett":
|
||||
|
||||
async def fetch_jackett_results(
|
||||
session: aiohttp.ClientSession, indexer: str, query: str
|
||||
):
|
||||
try:
|
||||
async with session.get(
|
||||
f"{settings.INDEXER_MANAGER_URL}/api/v2.0/indexers/all/results?apikey={settings.INDEXER_MANAGER_API_KEY}&Query={query}&Tracker[]={indexer}",
|
||||
timeout=aiohttp.ClientTimeout(
|
||||
total=settings.INDEXER_MANAGER_TIMEOUT
|
||||
),
|
||||
) as response:
|
||||
response_json = await response.json()
|
||||
return response_json.get("Results", [])
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while fetching Jackett results for indexer {indexer}: {e}"
|
||||
)
|
||||
return []
|
||||
|
||||
tasks = [
|
||||
fetch_jackett_results(session, indexer, query) for indexer in indexers
|
||||
]
|
||||
all_results = await asyncio.gather(*tasks)
|
||||
|
||||
for result_set in all_results:
|
||||
results.extend(result_set)
|
||||
|
||||
elif indexer_manager_type == "prowlarr":
|
||||
get_indexers = await session.get(
|
||||
f"{settings.INDEXER_MANAGER_URL}/api/v1/indexer",
|
||||
headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY},
|
||||
)
|
||||
get_indexers = await get_indexers.json()
|
||||
|
||||
indexers_id = []
|
||||
for indexer in get_indexers:
|
||||
if (
|
||||
indexer["name"].lower() in indexers
|
||||
or indexer["definitionName"].lower() in indexers
|
||||
):
|
||||
indexers_id.append(indexer["id"])
|
||||
|
||||
response = await session.get(
|
||||
f"{settings.INDEXER_MANAGER_URL}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search",
|
||||
headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY},
|
||||
)
|
||||
response = await response.json()
|
||||
|
||||
for result in response:
|
||||
result["InfoHash"] = (
|
||||
result["infoHash"] if "infoHash" in result else None
|
||||
)
|
||||
result["Title"] = result["title"]
|
||||
result["Size"] = result["size"]
|
||||
result["Link"] = (
|
||||
result["downloadUrl"] if "downloadUrl" in result else None
|
||||
)
|
||||
result["Tracker"] = result["indexer"]
|
||||
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while getting {indexer_manager_type} results for {query} with {indexers}: {e}"
|
||||
)
|
||||
pass
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def get_zilean(
|
||||
session: aiohttp.ClientSession, name: str, log_name: str, season: int, episode: int
|
||||
):
|
||||
results = []
|
||||
try:
|
||||
show = f"&season={season}&episode={episode}"
|
||||
get_dmm = await session.get(
|
||||
f"{settings.ZILEAN_URL}/dmm/filtered?query={name}{show if season else ''}"
|
||||
)
|
||||
get_dmm = await get_dmm.json()
|
||||
|
||||
if isinstance(get_dmm, list):
|
||||
take_first = get_dmm[: settings.ZILEAN_TAKE_FIRST]
|
||||
for result in take_first:
|
||||
object = {
|
||||
"Title": result["raw_title"],
|
||||
"InfoHash": result["info_hash"],
|
||||
"Size": int(result["size"]),
|
||||
"Tracker": "DMM",
|
||||
}
|
||||
|
||||
results.append(object)
|
||||
|
||||
logger.info(f"{len(results)} torrents found for {log_name} with Zilean")
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while getting torrents for {log_name} with Zilean: {e}"
|
||||
)
|
||||
pass
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def get_torrentio(log_name: str, type: str, full_id: str):
|
||||
results = []
|
||||
try:
|
||||
try:
|
||||
get_torrentio = requests.get(
|
||||
f"https://torrentio.strem.fun/stream/{type}/{full_id}.json"
|
||||
).json()
|
||||
except:
|
||||
get_torrentio = requests.get(
|
||||
f"https://torrentio.strem.fun/stream/{type}/{full_id}.json",
|
||||
proxies={
|
||||
"http": settings.DEBRID_PROXY_URL,
|
||||
"https": settings.DEBRID_PROXY_URL,
|
||||
},
|
||||
).json()
|
||||
|
||||
for torrent in get_torrentio["streams"]:
|
||||
title_full = torrent["title"]
|
||||
title = title_full.split("\n")[0]
|
||||
tracker = title_full.split("⚙️ ")[1].split("\n")[0]
|
||||
size = size_to_bytes(title_full.split("💾 ")[1].split(" ⚙️")[0])
|
||||
|
||||
results.append(
|
||||
{
|
||||
"Title": title,
|
||||
"InfoHash": torrent["infoHash"],
|
||||
"Size": size,
|
||||
"Tracker": f"Torrentio|{tracker}",
|
||||
}
|
||||
)
|
||||
|
||||
logger.info(f"{len(results)} torrents found for {log_name} with Torrentio")
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while getting torrents for {log_name} with Torrentio, your IP is most likely blacklisted (you should try proxying Comet): {e}"
|
||||
)
|
||||
pass
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def get_mediafusion(log_name: str, type: str, full_id: str):
|
||||
results = []
|
||||
try:
|
||||
try:
|
||||
get_mediafusion = requests.get(
|
||||
f"{settings.MEDIAFUSION_URL}/stream/{type}/{full_id}.json"
|
||||
).json()
|
||||
except:
|
||||
get_mediafusion = requests.get(
|
||||
f"{settings.MEDIAFUSION_URL}/stream/{type}/{full_id}.json",
|
||||
proxies={
|
||||
"http": settings.DEBRID_PROXY_URL,
|
||||
"https": settings.DEBRID_PROXY_URL,
|
||||
},
|
||||
).json()
|
||||
|
||||
for torrent in get_mediafusion["streams"]:
|
||||
title_full = torrent["description"]
|
||||
title = title_full.split("\n")[0].replace("📂 ", "").replace("/", "")
|
||||
tracker = title_full.split("🔗 ")[1]
|
||||
|
||||
results.append(
|
||||
{
|
||||
"Title": title,
|
||||
"InfoHash": torrent["infoHash"],
|
||||
"Size": torrent["behaviorHints"][
|
||||
"videoSize"
|
||||
], # not the pack size but still useful for prowlarr userss
|
||||
"Tracker": f"MediaFusion|{tracker}",
|
||||
}
|
||||
)
|
||||
|
||||
logger.info(f"{len(results)} torrents found for {log_name} with MediaFusion")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while getting torrents for {log_name} with MediaFusion, your IP is most likely blacklisted (you should try proxying Comet): {e}"
|
||||
)
|
||||
pass
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def filter(
|
||||
torrents: list,
|
||||
name: str,
|
||||
year: int,
|
||||
year_end: int,
|
||||
aliases: dict,
|
||||
remove_adult_content: bool,
|
||||
):
|
||||
results = []
|
||||
for torrent in torrents:
|
||||
index = torrent[0]
|
||||
title = torrent[1]
|
||||
|
||||
if "\n" in title: # Torrentio title parsing
|
||||
title = title.split("\n")[1]
|
||||
|
||||
parsed = parse(title)
|
||||
|
||||
if remove_adult_content and parsed.adult:
|
||||
results.append((index, False))
|
||||
continue
|
||||
|
||||
if parsed.parsed_title and not title_match(
|
||||
name, parsed.parsed_title, aliases=aliases
|
||||
):
|
||||
results.append((index, False))
|
||||
continue
|
||||
|
||||
if year and parsed.year:
|
||||
if year_end is not None:
|
||||
if not (year <= parsed.year <= year_end):
|
||||
results.append((index, False))
|
||||
continue
|
||||
else:
|
||||
if year < (parsed.year - 1) or year > (parsed.year + 1):
|
||||
results.append((index, False))
|
||||
continue
|
||||
|
||||
results.append((index, True))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def get_torrent_hash(session: aiohttp.ClientSession, torrent: tuple):
|
||||
index = torrent[0]
|
||||
torrent = torrent[1]
|
||||
if "InfoHash" in torrent and torrent["InfoHash"] is not None:
|
||||
return (index, torrent["InfoHash"].lower())
|
||||
|
||||
url = torrent["Link"]
|
||||
|
||||
try:
|
||||
timeout = aiohttp.ClientTimeout(total=settings.GET_TORRENT_TIMEOUT)
|
||||
response = await session.get(url, allow_redirects=False, timeout=timeout)
|
||||
if response.status == 200:
|
||||
torrent_data = await response.read()
|
||||
torrent_dict = bencodepy.decode(torrent_data)
|
||||
info = bencodepy.encode(torrent_dict[b"info"])
|
||||
hash = hashlib.sha1(info).hexdigest()
|
||||
else:
|
||||
location = response.headers.get("Location", "")
|
||||
if not location:
|
||||
return (index, None)
|
||||
|
||||
match = info_hash_pattern.search(location)
|
||||
if not match:
|
||||
return (index, None)
|
||||
|
||||
hash = match.group(1).upper()
|
||||
|
||||
return (index, hash.lower())
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Exception while getting torrent info hash for {torrent['indexer'] if 'indexer' in torrent else (torrent['Tracker'] if 'Tracker' in torrent else '')}|{url}: {e}"
|
||||
)
|
||||
|
||||
return (index, None)
|
||||
|
||||
|
||||
def get_balanced_hashes(hashes: dict, config: dict):
|
||||
max_results = config["maxResults"]
|
||||
max_results_per_resolution = config["maxResultsPerResolution"]
|
||||
|
||||
max_size = config["maxSize"]
|
||||
config_resolutions = [resolution.lower() for resolution in config["resolutions"]]
|
||||
include_all_resolutions = "all" in config_resolutions
|
||||
remove_trash = config["removeTrash"]
|
||||
|
||||
languages = [language.lower() for language in config["languages"]]
|
||||
include_all_languages = "all" in languages
|
||||
if not include_all_languages:
|
||||
config_languages = [
|
||||
code
|
||||
for code, name in PTT.parse.LANGUAGES_TRANSLATION_TABLE.items()
|
||||
if name.lower() in languages
|
||||
]
|
||||
|
||||
hashes_by_resolution = {}
|
||||
for hash, hash_data in hashes.items():
|
||||
if remove_trash and not hash_data["fetch"]:
|
||||
continue
|
||||
|
||||
hash_info = hash_data["data"]
|
||||
|
||||
if max_size != 0 and hash_info["size"] > max_size:
|
||||
continue
|
||||
|
||||
if (
|
||||
not include_all_languages
|
||||
and not any(lang in hash_info["languages"] for lang in config_languages)
|
||||
and ("multi" not in languages if hash_info["dubbed"] else True)
|
||||
and not (len(hash_info["languages"]) == 0 and "unknown" in languages)
|
||||
):
|
||||
continue
|
||||
|
||||
resolution = hash_info["resolution"]
|
||||
if not include_all_resolutions and resolution not in config_resolutions:
|
||||
continue
|
||||
|
||||
if resolution not in hashes_by_resolution:
|
||||
hashes_by_resolution[resolution] = []
|
||||
hashes_by_resolution[resolution].append(hash)
|
||||
|
||||
if config["reverseResultOrder"]:
|
||||
hashes_by_resolution = {
|
||||
res: lst[::-1] for res, lst in hashes_by_resolution.items()
|
||||
}
|
||||
|
||||
total_resolutions = len(hashes_by_resolution)
|
||||
if max_results == 0 and max_results_per_resolution == 0 or total_resolutions == 0:
|
||||
return hashes_by_resolution
|
||||
|
||||
hashes_per_resolution = (
|
||||
max_results // total_resolutions
|
||||
if max_results > 0
|
||||
else max_results_per_resolution
|
||||
)
|
||||
extra_hashes = max_results % total_resolutions
|
||||
|
||||
balanced_hashes = {}
|
||||
for resolution, hash_list in hashes_by_resolution.items():
|
||||
selected_count = hashes_per_resolution + (1 if extra_hashes > 0 else 0)
|
||||
if max_results_per_resolution > 0:
|
||||
selected_count = min(selected_count, max_results_per_resolution)
|
||||
balanced_hashes[resolution] = hash_list[:selected_count]
|
||||
if extra_hashes > 0:
|
||||
extra_hashes -= 1
|
||||
|
||||
selected_total = sum(len(hashes) for hashes in balanced_hashes.values())
|
||||
if selected_total < max_results:
|
||||
missing_hashes = max_results - selected_total
|
||||
for resolution, hash_list in hashes_by_resolution.items():
|
||||
if missing_hashes <= 0:
|
||||
break
|
||||
current_count = len(balanced_hashes[resolution])
|
||||
available_hashes = hash_list[current_count : current_count + missing_hashes]
|
||||
balanced_hashes[resolution].extend(available_hashes)
|
||||
missing_hashes -= len(available_hashes)
|
||||
|
||||
return balanced_hashes
|
||||
|
||||
|
||||
def format_metadata(data: dict):
|
||||
extras = []
|
||||
if data["quality"]:
|
||||
extras.append(data["quality"])
|
||||
if data["hdr"]:
|
||||
extras.extend(data["hdr"])
|
||||
if data["codec"]:
|
||||
extras.append(data["codec"])
|
||||
if data["audio"]:
|
||||
extras.extend(data["audio"])
|
||||
if data["channels"]:
|
||||
extras.extend(data["channels"])
|
||||
if data["bit_depth"]:
|
||||
extras.append(data["bit_depth"])
|
||||
if data["network"]:
|
||||
extras.append(data["network"])
|
||||
if data["group"]:
|
||||
extras.append(data["group"])
|
||||
|
||||
return "|".join(extras)
|
||||
|
||||
|
||||
def format_title(data: dict, config: dict):
|
||||
result_format = config["resultFormat"]
|
||||
has_all = "All" in result_format
|
||||
|
||||
title = ""
|
||||
if has_all or "Title" in result_format:
|
||||
title += f"{data['title']}\n"
|
||||
|
||||
if has_all or "Metadata" in result_format:
|
||||
metadata = format_metadata(data)
|
||||
if metadata != "":
|
||||
title += f"💿 {metadata}\n"
|
||||
|
||||
if has_all or "Size" in result_format:
|
||||
title += f"💾 {bytes_to_size(data['size'])} "
|
||||
|
||||
if has_all or "Tracker" in result_format:
|
||||
title += f"🔎 {data['tracker'] if 'tracker' in data else '?'}"
|
||||
|
||||
if has_all or "Languages" in result_format:
|
||||
languages = data["languages"]
|
||||
if data["dubbed"]:
|
||||
languages.insert(0, "multi")
|
||||
if languages:
|
||||
formatted_languages = "/".join(
|
||||
get_language_emoji(language) for language in languages
|
||||
)
|
||||
languages_str = "\n" + formatted_languages
|
||||
title += f"{languages_str}"
|
||||
|
||||
if title == "":
|
||||
# Without this, Streamio shows SD as the result, which is confusing
|
||||
title = "Empty result format configuration"
|
||||
|
||||
return title
|
||||
|
||||
|
||||
def get_client_ip(request: Request):
|
||||
return (
|
||||
request.headers["cf-connecting-ip"]
|
||||
if "cf-connecting-ip" in request.headers
|
||||
else request.client.host
|
||||
)
|
||||
|
||||
|
||||
async def get_aliases(session: aiohttp.ClientSession, media_type: str, media_id: str):
|
||||
aliases = {}
|
||||
try:
|
||||
response = await session.get(
|
||||
f"https://api.trakt.tv/{media_type}/{media_id}/aliases"
|
||||
)
|
||||
|
||||
for aliase in await response.json():
|
||||
country = aliase["country"]
|
||||
if country not in aliases:
|
||||
aliases[country] = []
|
||||
|
||||
aliases[country].append(aliase["title"])
|
||||
except:
|
||||
pass
|
||||
|
||||
return aliases
|
||||
|
||||
|
||||
async def add_torrent_to_cache(
|
||||
config: dict, name: str, season: int, episode: int, sorted_ranked_files: dict
|
||||
):
|
||||
# trace of which indexers were used when cache was created - not optimal
|
||||
indexers = config["indexers"].copy()
|
||||
if settings.SCRAPE_TORRENTIO:
|
||||
indexers.append("torrentio")
|
||||
if settings.SCRAPE_MEDIAFUSION:
|
||||
indexers.append("mediafusion")
|
||||
if settings.ZILEAN_URL:
|
||||
indexers.append("dmm")
|
||||
for indexer in indexers:
|
||||
hash = f"searched-{indexer}-{name}-{season}-{episode}"
|
||||
|
||||
searched = copy.deepcopy(
|
||||
sorted_ranked_files[list(sorted_ranked_files.keys())[0]]
|
||||
)
|
||||
searched["infohash"] = hash
|
||||
searched["data"]["tracker"] = indexer
|
||||
|
||||
sorted_ranked_files[hash] = searched
|
||||
|
||||
values = [
|
||||
{
|
||||
"debridService": config["debridService"],
|
||||
"info_hash": sorted_ranked_files[torrent]["infohash"],
|
||||
"name": name,
|
||||
"season": season,
|
||||
"episode": episode,
|
||||
"tracker": sorted_ranked_files[torrent]["data"]["tracker"]
|
||||
.split("|")[0]
|
||||
.lower(),
|
||||
"data": orjson.dumps(sorted_ranked_files[torrent]).decode("utf-8"),
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
for torrent in sorted_ranked_files
|
||||
]
|
||||
|
||||
query = f"""
|
||||
INSERT {'OR IGNORE ' if settings.DATABASE_TYPE == 'sqlite' else ''}
|
||||
INTO cache (debridService, info_hash, name, season, episode, tracker, data, timestamp)
|
||||
VALUES (:debridService, :info_hash, :name, :season, :episode, :tracker, :data, :timestamp)
|
||||
{' ON CONFLICT DO NOTHING' if settings.DATABASE_TYPE == 'postgresql' else ''}
|
||||
"""
|
||||
|
||||
await database.execute_many(query, values)
|
||||
return info[0], int(info[1]), int(info[2])
|
||||
elif media_type == "movie" and "kitsu" in media_id:
|
||||
info = media_id.split(":")
|
||||
return info[0], int(info[1]), None
|
||||
return media_id, None, None
|
||||
|
||||
@@ -1,11 +1,18 @@
|
||||
import sys
|
||||
import logging
|
||||
|
||||
from loguru import logger
|
||||
|
||||
logging.getLogger("demagnetize").setLevel(
|
||||
logging.CRITICAL
|
||||
) # disable demagnetize logging
|
||||
|
||||
|
||||
def setupLogger(level: str):
|
||||
logger.level("COMET", no=50, icon="🌠", color="<fg #7871d6>")
|
||||
logger.level("API", no=40, icon="👾", color="<fg #7871d6>")
|
||||
logger.level("API", no=45, icon="👾", color="<fg #006989>")
|
||||
logger.level("SCRAPER", no=40, icon="👻", color="<fg #d6bb71>")
|
||||
logger.level("STREAM", no=35, icon="🎬", color="<fg #d171d6>")
|
||||
|
||||
logger.level("INFO", icon="📰", color="<fg #FC5F39>")
|
||||
logger.level("DEBUG", icon="🕸️", color="<fg #DC5F00>")
|
||||
|
||||
+541
-219
@@ -1,27 +1,46 @@
|
||||
import os
|
||||
import random
|
||||
import string
|
||||
import RTN
|
||||
|
||||
from typing import List, Optional
|
||||
from databases import Database
|
||||
from pydantic import BaseModel, field_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
from RTN import RTN, BestRanking, SettingsModel
|
||||
from RTN import BestRanking, SettingsModel
|
||||
from RTN.models import (
|
||||
ResolutionConfig,
|
||||
OptionsConfig,
|
||||
LanguagesConfig,
|
||||
CustomRanksConfig,
|
||||
CustomRank,
|
||||
QualityRankModel,
|
||||
RipsRankModel,
|
||||
HdrRankModel,
|
||||
AudioRankModel,
|
||||
ExtrasRankModel,
|
||||
)
|
||||
|
||||
|
||||
class AppSettings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8")
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env", env_file_encoding="utf-8", extra="ignore"
|
||||
)
|
||||
|
||||
ADDON_ID: Optional[str] = "stremio.comet.fast"
|
||||
ADDON_NAME: Optional[str] = "Comet"
|
||||
FASTAPI_HOST: Optional[str] = "0.0.0.0"
|
||||
FASTAPI_PORT: Optional[int] = 8000
|
||||
FASTAPI_WORKERS: Optional[int] = 2 * (os.cpu_count() or 1)
|
||||
DASHBOARD_ADMIN_PASSWORD: Optional[str] = None
|
||||
FASTAPI_WORKERS: Optional[int] = 1
|
||||
USE_GUNICORN: Optional[bool] = True
|
||||
DASHBOARD_ADMIN_PASSWORD: Optional[str] = "".join(
|
||||
random.choices(string.ascii_letters + string.digits, k=16)
|
||||
)
|
||||
DATABASE_TYPE: Optional[str] = "sqlite"
|
||||
DATABASE_URL: Optional[str] = "username:password@hostname:port"
|
||||
DATABASE_PATH: Optional[str] = "data/comet.db"
|
||||
CACHE_TTL: Optional[int] = 86400
|
||||
METADATA_CACHE_TTL: Optional[int] = 2592000 # 30 days
|
||||
TORRENT_CACHE_TTL: Optional[int] = 1296000 # 15 days
|
||||
DEBRID_CACHE_TTL: Optional[int] = 86400 # 1 day
|
||||
DEBRID_PROXY_URL: Optional[str] = None
|
||||
INDEXER_MANAGER_TYPE: Optional[str] = None
|
||||
INDEXER_MANAGER_URL: Optional[str] = "http://127.0.0.1:9117"
|
||||
@@ -29,86 +48,498 @@ class AppSettings(BaseSettings):
|
||||
INDEXER_MANAGER_TIMEOUT: Optional[int] = 30
|
||||
INDEXER_MANAGER_INDEXERS: List[str] = []
|
||||
GET_TORRENT_TIMEOUT: Optional[int] = 5
|
||||
ZILEAN_URL: Optional[str] = None
|
||||
ZILEAN_TAKE_FIRST: Optional[int] = 500
|
||||
DOWNLOAD_TORRENT_FILES: Optional[bool] = False
|
||||
SCRAPE_MEDIAFUSION: Optional[bool] = False
|
||||
SCRAPE_ZILEAN: Optional[bool] = False
|
||||
ZILEAN_URL: Optional[str] = "https://zilean.elfhosted.com"
|
||||
SCRAPE_TORRENTIO: Optional[bool] = False
|
||||
TORRENTIO_URL: Optional[str] = "https://torrentio.strem.fun"
|
||||
SCRAPE_MEDIAFUSION: Optional[bool] = False
|
||||
MEDIAFUSION_URL: Optional[str] = "https://mediafusion.elfhosted.com"
|
||||
CUSTOM_HEADER_HTML: Optional[str] = None
|
||||
PROXY_DEBRID_STREAM: Optional[bool] = False
|
||||
PROXY_DEBRID_STREAM_PASSWORD: Optional[str] = None
|
||||
PROXY_DEBRID_STREAM_PASSWORD: Optional[str] = "".join(
|
||||
random.choices(string.ascii_letters + string.digits, k=16)
|
||||
)
|
||||
PROXY_DEBRID_STREAM_MAX_CONNECTIONS: Optional[int] = -1
|
||||
PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE: Optional[str] = "realdebrid"
|
||||
PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY: Optional[str] = None
|
||||
TITLE_MATCH_CHECK: Optional[bool] = True
|
||||
STREMTHRU_URL: Optional[str] = "https://stremthru.13377001.xyz"
|
||||
REMOVE_ADULT_CONTENT: Optional[bool] = False
|
||||
STREMTHRU_DEFAULT_URL: Optional[str] = "https://stremthru.mooo.com"
|
||||
STREMTHRU_AUTO_ENABLED_DEBRID_SERVICES: List[str] = [
|
||||
"realdebrid",
|
||||
"alldebrid",
|
||||
"debridlink",
|
||||
]
|
||||
|
||||
@field_validator("DASHBOARD_ADMIN_PASSWORD")
|
||||
def set_dashboard_admin_password(cls, v, values):
|
||||
if v is None:
|
||||
return "".join(random.choices(string.ascii_letters + string.digits, k=16))
|
||||
@field_validator(
|
||||
"INDEXER_MANAGER_URL",
|
||||
"ZILEAN_URL",
|
||||
"TORRENTIO_URL",
|
||||
"MEDIAFUSION_URL",
|
||||
"STREMTHRU_URL",
|
||||
)
|
||||
def remove_trailing_slash(cls, v):
|
||||
if v and v.endswith("/"):
|
||||
return v[:-1]
|
||||
return v
|
||||
|
||||
@field_validator("INDEXER_MANAGER_TYPE")
|
||||
def set_indexer_manager_type(cls, v, values):
|
||||
if v == "None":
|
||||
if v is not None and v.lower() == "none":
|
||||
return None
|
||||
return v
|
||||
|
||||
@field_validator("PROXY_DEBRID_STREAM_PASSWORD")
|
||||
def set_debrid_stream_proxy_password(cls, v, values):
|
||||
if v is None:
|
||||
return "".join(random.choices(string.ascii_letters + string.digits, k=16))
|
||||
@field_validator("INDEXER_MANAGER_INDEXERS")
|
||||
def indexer_manager_indexers_normalization(cls, v, values):
|
||||
v = [indexer.replace(" ", "").lower() for indexer in v]
|
||||
return v
|
||||
|
||||
|
||||
settings = AppSettings()
|
||||
|
||||
|
||||
class CometSettingsModel(SettingsModel):
|
||||
model_config = SettingsConfigDict()
|
||||
|
||||
resolutions: ResolutionConfig = ResolutionConfig(
|
||||
r2160p=True, r480p=True, r360p=True
|
||||
)
|
||||
|
||||
options: OptionsConfig = OptionsConfig(remove_ranks_under=-10000000000)
|
||||
|
||||
languages: LanguagesConfig = LanguagesConfig(exclude=[])
|
||||
|
||||
custom_ranks: CustomRanksConfig = CustomRanksConfig(
|
||||
quality=QualityRankModel(
|
||||
av1=CustomRank(fetch=True),
|
||||
dvd=CustomRank(fetch=True),
|
||||
mpeg=CustomRank(fetch=True),
|
||||
remux=CustomRank(fetch=True),
|
||||
vhs=CustomRank(fetch=True),
|
||||
webmux=CustomRank(fetch=True),
|
||||
xvid=CustomRank(fetch=True),
|
||||
),
|
||||
rips=RipsRankModel(
|
||||
bdrip=CustomRank(fetch=True),
|
||||
dvdrip=CustomRank(fetch=True),
|
||||
ppvrip=CustomRank(fetch=True),
|
||||
satrip=CustomRank(fetch=True),
|
||||
tvrip=CustomRank(fetch=True),
|
||||
uhdrip=CustomRank(fetch=True),
|
||||
vhsrip=CustomRank(fetch=True),
|
||||
webdlrip=CustomRank(fetch=True),
|
||||
),
|
||||
hdr=HdrRankModel(
|
||||
dolby_vision=CustomRank(fetch=True),
|
||||
),
|
||||
audio=AudioRankModel(
|
||||
mono=CustomRank(fetch=True),
|
||||
mp3=CustomRank(fetch=True),
|
||||
),
|
||||
extras=ExtrasRankModel(
|
||||
three_d=CustomRank(fetch=True),
|
||||
converted=CustomRank(fetch=True),
|
||||
documentary=CustomRank(fetch=True),
|
||||
site=CustomRank(fetch=True),
|
||||
upscaled=CustomRank(fetch=True),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
rtn_settings_default = CometSettingsModel()
|
||||
rtn_settings_default_dumped = rtn_settings_default.model_dump()
|
||||
# {
|
||||
# "profile":"default",
|
||||
# "require":[
|
||||
|
||||
# ],
|
||||
# "exclude":[
|
||||
|
||||
# ],
|
||||
# "preferred":[
|
||||
|
||||
# ],
|
||||
# "resolutions":{
|
||||
# "r2160p":true,
|
||||
# "r1080p":true,
|
||||
# "r720p":true,
|
||||
# "r480p":true,
|
||||
# "r360p":true,
|
||||
# "unknown":true
|
||||
# },
|
||||
# "options":{
|
||||
# "title_similarity":0.85,
|
||||
# "remove_all_trash":true,
|
||||
# "remove_ranks_under":-10000000000,
|
||||
# "remove_unknown_languages":false,
|
||||
# "allow_english_in_languages":false,
|
||||
# "enable_fetch_speed_mode":true,
|
||||
# "remove_adult_content":true
|
||||
# },
|
||||
# "languages":{
|
||||
# "required":[
|
||||
|
||||
# ],
|
||||
# "exclude":[
|
||||
|
||||
# ],
|
||||
# "preferred":[
|
||||
|
||||
# ]
|
||||
# },
|
||||
# "custom_ranks":{
|
||||
# "quality":{
|
||||
# "av1":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "avc":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "bluray":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "dvd":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "hdtv":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "hevc":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "mpeg":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "remux":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "vhs":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "web":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "webdl":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "webmux":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "xvid":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# }
|
||||
# },
|
||||
# "rips":{
|
||||
# "bdrip":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "brrip":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "dvdrip":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "hdrip":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "ppvrip":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "satrip":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "tvrip":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "uhdrip":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "vhsrip":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "webdlrip":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "webrip":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# }
|
||||
# },
|
||||
# "hdr":{
|
||||
# "bit10":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "dolby_vision":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "hdr":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "hdr10plus":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "sdr":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# }
|
||||
# },
|
||||
# "audio":{
|
||||
# "aac":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "ac3":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "atmos":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "dolby_digital":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "dolby_digital_plus":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "dts_lossy":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "dts_lossless":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "eac3":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "flac":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "mono":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "mp3":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "stereo":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "surround":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "truehd":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# }
|
||||
# },
|
||||
# "extras":{
|
||||
# "three_d":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "converted":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "documentary":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "dubbed":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "edition":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "hardcoded":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "network":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "proper":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "repack":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "retail":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "site":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "subbed":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "upscaled":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "scene":{
|
||||
# "fetch":true,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# }
|
||||
# },
|
||||
# "trash":{
|
||||
# "cam":{
|
||||
# "fetch":false,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "clean_audio":{
|
||||
# "fetch":false,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "pdtv":{
|
||||
# "fetch":false,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "r5":{
|
||||
# "fetch":false,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "screener":{
|
||||
# "fetch":false,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "size":{
|
||||
# "fetch":false,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "telecine":{
|
||||
# "fetch":false,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# },
|
||||
# "telesync":{
|
||||
# "fetch":false,
|
||||
# "use_custom_rank":false,
|
||||
# "rank":0
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
rtn_ranking_default = BestRanking()
|
||||
|
||||
|
||||
class ConfigModel(BaseModel):
|
||||
indexers: List[str]
|
||||
languages: Optional[List[str]] = ["All"]
|
||||
resolutions: Optional[List[str]] = ["All"]
|
||||
reverseResultOrder: Optional[bool] = False
|
||||
cachedOnly: Optional[bool] = False
|
||||
removeTrash: Optional[bool] = True
|
||||
resultFormat: Optional[List[str]] = ["All"]
|
||||
maxResults: Optional[int] = 0
|
||||
resultFormat: Optional[List[str]] = ["all"]
|
||||
maxResultsPerResolution: Optional[int] = 0
|
||||
maxSize: Optional[float] = 0
|
||||
debridService: str
|
||||
debridApiKey: str
|
||||
debridService: Optional[str] = "torrent"
|
||||
debridApiKey: Optional[str] = ""
|
||||
debridStreamProxyPassword: Optional[str] = ""
|
||||
stremthruUrl: Optional[str] = None
|
||||
|
||||
@field_validator("indexers")
|
||||
def check_indexers(cls, v, values):
|
||||
settings.INDEXER_MANAGER_INDEXERS = [
|
||||
indexer.replace(" ", "_").lower()
|
||||
for indexer in settings.INDEXER_MANAGER_INDEXERS
|
||||
] # to equal webui
|
||||
valid_indexers = [
|
||||
indexer for indexer in v if indexer in settings.INDEXER_MANAGER_INDEXERS
|
||||
]
|
||||
# if not valid_indexers: # For only Zilean mode
|
||||
# raise ValueError(
|
||||
# f"At least one indexer must be from {settings.INDEXER_MANAGER_INDEXERS}"
|
||||
# )
|
||||
return valid_indexers
|
||||
|
||||
@field_validator("maxResults")
|
||||
def check_max_results(cls, v):
|
||||
if not isinstance(v, int):
|
||||
v = 0
|
||||
|
||||
if v < 0:
|
||||
v = 0
|
||||
return v
|
||||
languages: Optional[dict] = rtn_settings_default_dumped["languages"]
|
||||
resolutions: Optional[dict] = rtn_settings_default_dumped["resolutions"]
|
||||
options: Optional[dict] = rtn_settings_default_dumped["options"]
|
||||
rtnSettings: Optional[CometSettingsModel] = rtn_settings_default
|
||||
rtnRanking: Optional[BestRanking] = rtn_ranking_default
|
||||
|
||||
@field_validator("maxResultsPerResolution")
|
||||
def check_max_results_per_resolution(cls, v):
|
||||
@@ -130,132 +561,35 @@ class ConfigModel(BaseModel):
|
||||
|
||||
@field_validator("debridService")
|
||||
def check_debrid_service(cls, v):
|
||||
if v not in ["realdebrid", "alldebrid", "premiumize", "torbox", "debridlink", "stremthru"]:
|
||||
if v not in [
|
||||
"realdebrid",
|
||||
"alldebrid",
|
||||
"premiumize",
|
||||
"torbox",
|
||||
"easydebrid",
|
||||
"debridlink",
|
||||
"offcloud",
|
||||
"pikpak",
|
||||
"torrent",
|
||||
]:
|
||||
raise ValueError("Invalid debridService")
|
||||
return v
|
||||
|
||||
|
||||
default_settings = {
|
||||
"profile": "default",
|
||||
"require": [],
|
||||
"exclude": [],
|
||||
"preferred": [],
|
||||
"resolutions": {
|
||||
"r2160p": True,
|
||||
"r1080p": True,
|
||||
"r720p": True,
|
||||
"r480p": True,
|
||||
"r360p": True,
|
||||
"unknown": True,
|
||||
},
|
||||
"options": {
|
||||
"title_similarity": 0.85,
|
||||
"remove_all_trash": True,
|
||||
"remove_ranks_under": -1000000000000000,
|
||||
"remove_unknown_languages": False,
|
||||
"allow_english_in_languages": True,
|
||||
"enable_fetch_speed_mode": True,
|
||||
"remove_adult_content": settings.REMOVE_ADULT_CONTENT,
|
||||
},
|
||||
"languages": {
|
||||
"required": [],
|
||||
"exclude": [
|
||||
# "ar",
|
||||
# "hi",
|
||||
# "fr",
|
||||
# "es",
|
||||
# "de",
|
||||
# "ru",
|
||||
# "pt",
|
||||
# "it"
|
||||
],
|
||||
"preferred": [],
|
||||
},
|
||||
"custom_ranks": {
|
||||
"quality": {
|
||||
"av1": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"avc": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"bluray": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"dvd": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"hdtv": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"hevc": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"mpeg": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"remux": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"vhs": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"web": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"webdl": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"webmux": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"xvid": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
},
|
||||
"rips": {
|
||||
"bdrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"brrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"dvdrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"hdrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"ppvrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"satrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"tvrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"uhdrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"vhsrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"webdlrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"webrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
},
|
||||
"hdr": {
|
||||
"bit10": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"dolby_vision": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"hdr": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"hdr10plus": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"sdr": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
},
|
||||
"audio": {
|
||||
"aac": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"ac3": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"atmos": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"dolby_digital": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"dolby_digital_plus": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"dts_lossy": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"dts_lossless": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"eac3": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"flac": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"mono": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"mp3": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"stereo": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"surround": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"Truehd": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
},
|
||||
"extras": {
|
||||
"three_d": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"converted": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"documentary": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"dubbed": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"edition": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"hardcoded": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"network": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"proper": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"repack": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"retail": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"site": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"subbed": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"upscaled": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
"scene": {"fetch": True, "use_custom_rank": False, "rank": 0},
|
||||
},
|
||||
"trash": {
|
||||
"cam": {"fetch": False, "use_custom_rank": False, "rank": 0},
|
||||
"clean_audio": {"fetch": False, "use_custom_rank": False, "rank": 0},
|
||||
"pdtv": {"fetch": False, "use_custom_rank": False, "rank": 0},
|
||||
"r5": {"fetch": False, "use_custom_rank": False, "rank": 0},
|
||||
"screener": {"fetch": False, "use_custom_rank": False, "rank": 0},
|
||||
"size": {"fetch": False, "use_custom_rank": False, "rank": 0},
|
||||
"telecine": {"fetch": False, "use_custom_rank": False, "rank": 0},
|
||||
"telesync": {"fetch": False, "use_custom_rank": False, "rank": 0},
|
||||
},
|
||||
},
|
||||
}
|
||||
rtn_settings = SettingsModel(**default_settings)
|
||||
rtn_ranking = BestRanking()
|
||||
default_config = ConfigModel().model_dump()
|
||||
default_config["rtnSettings"] = rtn_settings_default
|
||||
default_config["rtnRanking"] = rtn_ranking_default
|
||||
|
||||
# For use anywhere
|
||||
rtn = RTN(settings=rtn_settings, ranking_model=rtn_ranking)
|
||||
|
||||
# Web Config Initialization
|
||||
# languages = [language for language in PTT.parse.LANGUAGES_TRANSLATION_TABLE.values()]
|
||||
# languages.insert(0, "Unknown")
|
||||
# languages.insert(1, "Multi")
|
||||
web_config = {
|
||||
# "languages": languages,
|
||||
"resolutions": [resolution.value for resolution in RTN.models.Resolution],
|
||||
"resultFormat": ["title", "metadata", "seeders", "size", "tracker", "languages"],
|
||||
}
|
||||
|
||||
database_url = (
|
||||
settings.DATABASE_PATH
|
||||
@@ -267,44 +601,32 @@ database = Database(
|
||||
)
|
||||
|
||||
trackers = [
|
||||
"tracker:https://tracker.gbitt.info:443/announce",
|
||||
"tracker:udp://discord.heihachi.pw:6969/announce",
|
||||
"tracker:http://tracker.corpscorp.online:80/announce",
|
||||
"tracker:udp://tracker.leechers-paradise.org:6969/announce",
|
||||
"tracker:https://tracker.renfei.net:443/announce",
|
||||
"tracker:udp://exodus.desync.com:6969/announce",
|
||||
"tracker:http://tracker.xiaoduola.xyz:6969/announce",
|
||||
"tracker:udp://ipv4.tracker.harry.lu:80/announce",
|
||||
"tracker:udp://tracker.torrent.eu.org:451/announce",
|
||||
"tracker:udp://tracker.coppersurfer.tk:6969/announce",
|
||||
"tracker:http://tracker.dmcomic.org:2710/announce",
|
||||
"tracker:http://www.genesis-sp.org:2710/announce",
|
||||
"tracker:http://t.jaekr.sh:6969/announce",
|
||||
"tracker:http://tracker.bt-hash.com:80/announce",
|
||||
"tracker:https://tracker.tamersunion.org:443/announce",
|
||||
"tracker:udp://open.stealth.si:80/announce",
|
||||
"tracker:udp://tracker.opentrackr.org:1337/announce",
|
||||
"tracker:udp://leet-tracker.moe:1337/announce",
|
||||
"tracker:udp://oh.fuuuuuck.com:6969/announce",
|
||||
"tracker:udp://tracker.bittor.pw:1337/announce",
|
||||
"tracker:udp://explodie.org:6969/announce",
|
||||
"tracker:http://finbytes.org:80/announce.php",
|
||||
"tracker:udp://tracker.dump.cl:6969/announce",
|
||||
"tracker:udp://open.free-tracker.ga:6969/announce",
|
||||
"tracker:http://tracker.gbitt.info:80/announce",
|
||||
"tracker:udp://isk.richardsw.club:6969/announce",
|
||||
"tracker:http://bt1.xxxxbt.cc:6969/announce",
|
||||
"tracker:udp://tracker.qu.ax:6969/announce",
|
||||
"tracker:udp://opentracker.io:6969/announce",
|
||||
"tracker:udp://tracker.internetwarriors.net:1337/announce",
|
||||
"tracker:udp://tracker.0x7c0.com:6969/announce",
|
||||
"tracker:udp://9.rarbg.me:2710/announce",
|
||||
"tracker:udp://tracker.pomf.se:80/announce",
|
||||
"tracker:udp://tracker.openbittorrent.com:80/announce",
|
||||
"tracker:udp://open.tracker.cl:1337/announce",
|
||||
"tracker:http://www.torrentsnipe.info:2701/announce",
|
||||
"tracker:udp://retracker01-msk-virt.corbina.net:80/announce",
|
||||
"tracker:udp://open.demonii.com:1337/announce",
|
||||
"tracker:udp://tracker-udp.gbitt.info:80/announce",
|
||||
"tracker:udp://tracker.tiny-vps.com:6969/announce",
|
||||
"udp://tracker-udp.gbitt.info:80/announce",
|
||||
"udp://tracker.0x7c0.com:6969/announce",
|
||||
"udp://opentracker.io:6969/announce",
|
||||
"udp://leet-tracker.moe:1337/announce",
|
||||
"udp://tracker.torrent.eu.org:451/announce",
|
||||
"udp://tracker.tiny-vps.com:6969/announce",
|
||||
"udp://tracker.leechers-paradise.org:6969/announce",
|
||||
"udp://tracker.pomf.se:80/announce",
|
||||
"udp://9.rarbg.me:2710/announce",
|
||||
"http://tracker.gbitt.info:80/announce",
|
||||
"udp://tracker.bittor.pw:1337/announce",
|
||||
"udp://open.free-tracker.ga:6969/announce",
|
||||
"udp://open.stealth.si:80/announce",
|
||||
"udp://retracker01-msk-virt.corbina.net:80/announce",
|
||||
"udp://tracker.openbittorrent.com:80/announce",
|
||||
"udp://tracker.opentrackr.org:1337/announce",
|
||||
"udp://isk.richardsw.club:6969/announce",
|
||||
"https://tracker.gbitt.info:443/announce",
|
||||
"udp://tracker.coppersurfer.tk:6969/announce",
|
||||
"udp://oh.fuuuuuck.com:6969/announce",
|
||||
"udp://ipv4.tracker.harry.lu:80/announce",
|
||||
"udp://open.demonii.com:1337/announce",
|
||||
"https://tracker.tamersunion.org:443/announce",
|
||||
"https://tracker.renfei.net:443/announce",
|
||||
"udp://open.tracker.cl:1337/announce",
|
||||
"udp://tracker.internetwarriors.net:1337/announce",
|
||||
"udp://exodus.desync.com:6969/announce",
|
||||
"udp://tracker.dump.cl:6969/announce",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import time
|
||||
import uuid
|
||||
from starlette.background import BackgroundTask
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from comet.utils.models import settings, database
|
||||
from comet.utils.logger import logger
|
||||
import mediaflow_proxy.handlers
|
||||
import mediaflow_proxy.utils.http_utils
|
||||
|
||||
|
||||
async def on_stream_end(connection_id: str, ip: str):
|
||||
try:
|
||||
await database.execute(
|
||||
"DELETE FROM active_connections WHERE id = :connection_id AND ip = :ip",
|
||||
{"connection_id": connection_id, "ip": ip},
|
||||
)
|
||||
logger.log(
|
||||
"STREAM", f"Stream ended - Connection: {connection_id} from IP: {ip}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Error handling stream end for connection {connection_id} from IP {ip}: {e}"
|
||||
)
|
||||
|
||||
|
||||
async def check_ip_connections(ip: str):
|
||||
if settings.PROXY_DEBRID_STREAM_MAX_CONNECTIONS <= -1:
|
||||
return True
|
||||
|
||||
try:
|
||||
count = await database.fetch_val(
|
||||
"SELECT COUNT(*) FROM active_connections WHERE ip = :ip",
|
||||
{"ip": ip},
|
||||
)
|
||||
if count >= settings.PROXY_DEBRID_STREAM_MAX_CONNECTIONS:
|
||||
logger.log(
|
||||
"STREAM",
|
||||
f"Connection limit reached for IP: {ip} ({count} active connections)",
|
||||
)
|
||||
return False
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Error checking IP connections for {ip}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def add_active_connection(media_id: str, ip: str):
|
||||
connection_id = str(uuid.uuid4())
|
||||
await database.execute(
|
||||
"INSERT INTO active_connections (id, ip, content, timestamp) VALUES (:connection_id, :ip, :content, :timestamp)",
|
||||
{
|
||||
"connection_id": connection_id,
|
||||
"ip": ip,
|
||||
"content": media_id,
|
||||
"timestamp": time.time(),
|
||||
},
|
||||
)
|
||||
logger.log(
|
||||
"STREAM",
|
||||
f"New stream connection - ID: {connection_id}, IP: {ip}, Content: {media_id}",
|
||||
)
|
||||
return connection_id
|
||||
|
||||
|
||||
async def combined_background_tasks(
|
||||
connection_id: str, ip: str, streamer_close_task: BackgroundTask
|
||||
):
|
||||
await streamer_close_task()
|
||||
await on_stream_end(connection_id, ip)
|
||||
|
||||
|
||||
async def custom_handle_stream_request(
|
||||
method: str,
|
||||
video_url: str,
|
||||
proxy_headers: mediaflow_proxy.utils.http_utils.ProxyRequestHeaders,
|
||||
media_id: str,
|
||||
ip: str,
|
||||
):
|
||||
if not await check_ip_connections(ip):
|
||||
return FileResponse("comet/assets/proxylimit.mp4")
|
||||
|
||||
connection_id = await add_active_connection(media_id, ip)
|
||||
|
||||
response = await mediaflow_proxy.handlers.handle_stream_request(
|
||||
method, video_url, proxy_headers
|
||||
)
|
||||
original_background_task = response.background
|
||||
response.background = BackgroundTask(
|
||||
combined_background_tasks,
|
||||
connection_id=connection_id,
|
||||
ip=ip,
|
||||
streamer_close_task=original_background_task,
|
||||
)
|
||||
return response
|
||||
@@ -0,0 +1,490 @@
|
||||
import hashlib
|
||||
import re
|
||||
import bencodepy
|
||||
import aiohttp
|
||||
import anyio
|
||||
import asyncio
|
||||
import orjson
|
||||
import time
|
||||
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
from demagnetize.core import Demagnetizer
|
||||
from torf import Magnet
|
||||
from RTN import ParsedData, parse
|
||||
|
||||
from comet.utils.logger import logger
|
||||
from comet.utils.models import settings, database
|
||||
from comet.utils.general import is_video, default_dump
|
||||
|
||||
info_hash_pattern = re.compile(r"btih:([a-fA-F0-9]{40})")
|
||||
|
||||
|
||||
def extract_trackers_from_magnet(magnet_uri: str):
|
||||
try:
|
||||
parsed = urlparse(magnet_uri)
|
||||
params = parse_qs(parsed.query)
|
||||
return params.get("tr", [])
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to extract trackers from magnet URI: {e}")
|
||||
return []
|
||||
|
||||
|
||||
async def download_torrent(session: aiohttp.ClientSession, url: str):
|
||||
try:
|
||||
timeout = aiohttp.ClientTimeout(total=settings.GET_TORRENT_TIMEOUT)
|
||||
async with session.get(url, allow_redirects=False, timeout=timeout) as response:
|
||||
if response.status == 200:
|
||||
return (await response.read(), None, None)
|
||||
|
||||
location = response.headers.get("Location", "")
|
||||
if location:
|
||||
match = info_hash_pattern.search(location)
|
||||
if match:
|
||||
return (None, match.group(1), location)
|
||||
return (None, None, None)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed to download torrent from {url}: {e} (in most cases, you can ignore this error)"
|
||||
)
|
||||
return (None, None, None)
|
||||
|
||||
|
||||
demagnetizer = Demagnetizer()
|
||||
|
||||
|
||||
async def get_torrent_from_magnet(magnet_uri: str):
|
||||
try:
|
||||
magnet = Magnet.from_string(magnet_uri)
|
||||
with anyio.fail_after(60):
|
||||
torrent_data = await demagnetizer.demagnetize(magnet)
|
||||
if torrent_data:
|
||||
return torrent_data.dump()
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get torrent from magnet: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def extract_torrent_metadata(content: bytes):
|
||||
try:
|
||||
torrent_data = bencodepy.decode(content)
|
||||
info = torrent_data[b"info"]
|
||||
info_encoded = bencodepy.encode(info)
|
||||
m = hashlib.sha1()
|
||||
m.update(info_encoded)
|
||||
info_hash = m.hexdigest()
|
||||
|
||||
announce_list = [
|
||||
tracker[0].decode() for tracker in torrent_data.get(b"announce-list", [])
|
||||
]
|
||||
|
||||
metadata = {"info_hash": info_hash, "announce_list": announce_list, "files": []}
|
||||
|
||||
files = info[b"files"] if b"files" in info else [info]
|
||||
for idx, file in enumerate(files):
|
||||
name = (
|
||||
file[b"path"][-1].decode()
|
||||
if b"path" in file
|
||||
else file[b"name"].decode()
|
||||
)
|
||||
|
||||
if not is_video(name) or "sample" in name.lower():
|
||||
continue
|
||||
|
||||
size = file[b"length"]
|
||||
|
||||
metadata["files"].append({"index": idx, "name": name, "size": size})
|
||||
|
||||
return metadata
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to extract torrent metadata: {e}")
|
||||
return {}
|
||||
|
||||
|
||||
async def add_torrent(
|
||||
info_hash: str,
|
||||
seeders: int,
|
||||
tracker: str,
|
||||
media_id: str,
|
||||
search_season: int,
|
||||
sources: list,
|
||||
file_index: int,
|
||||
title: str,
|
||||
size: int,
|
||||
parsed: ParsedData,
|
||||
):
|
||||
try:
|
||||
parsed_season = parsed.seasons[0] if parsed.seasons else search_season
|
||||
parsed_episode = parsed.episodes[0] if parsed.episodes else None
|
||||
|
||||
if parsed_episode is not None:
|
||||
await database.execute(
|
||||
"""
|
||||
DELETE FROM torrents
|
||||
WHERE info_hash = :info_hash
|
||||
AND season = :season
|
||||
AND episode IS NULL
|
||||
""",
|
||||
{
|
||||
"info_hash": info_hash,
|
||||
"season": parsed_season,
|
||||
},
|
||||
)
|
||||
logger.log(
|
||||
"SCRAPER",
|
||||
f"Deleted season-only entry for S{parsed_season:02d} of {info_hash}",
|
||||
)
|
||||
|
||||
await database.execute(
|
||||
f"""
|
||||
INSERT {"OR IGNORE " if settings.DATABASE_TYPE == "sqlite" else ""}
|
||||
INTO torrents
|
||||
VALUES (:media_id, :info_hash, :file_index, :season, :episode, :title, :seeders, :size, :tracker, :sources, :parsed, :timestamp)
|
||||
{" ON CONFLICT DO NOTHING" if settings.DATABASE_TYPE == "postgresql" else ""}
|
||||
""",
|
||||
{
|
||||
"media_id": media_id,
|
||||
"info_hash": info_hash,
|
||||
"file_index": file_index,
|
||||
"season": parsed_season,
|
||||
"episode": parsed_episode,
|
||||
"title": title,
|
||||
"seeders": seeders,
|
||||
"size": size,
|
||||
"tracker": tracker,
|
||||
"sources": orjson.dumps(sources).decode("utf-8"),
|
||||
"parsed": orjson.dumps(parsed, default_dump).decode("utf-8"),
|
||||
"timestamp": time.time(),
|
||||
},
|
||||
)
|
||||
|
||||
additional = ""
|
||||
if parsed_season:
|
||||
additional += f" - S{parsed_season:02d}"
|
||||
additional += f"E{parsed_episode:02d}" if parsed_episode else ""
|
||||
|
||||
logger.log("SCRAPER", f"Added torrent for {media_id} - {title}{additional}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to add torrent for {info_hash}: {e}")
|
||||
|
||||
|
||||
class AddTorrentQueue:
|
||||
def __init__(self, max_concurrent: int = 10):
|
||||
self.queue = asyncio.Queue()
|
||||
self.max_concurrent = max_concurrent
|
||||
self.is_running = False
|
||||
self.semaphore = asyncio.Semaphore(max_concurrent)
|
||||
|
||||
async def add_torrent(
|
||||
self,
|
||||
magnet_url: str,
|
||||
seeders: int,
|
||||
tracker: str,
|
||||
media_id: str,
|
||||
search_season: int,
|
||||
):
|
||||
if not settings.DOWNLOAD_TORRENT_FILES:
|
||||
return
|
||||
|
||||
await self.queue.put((magnet_url, seeders, tracker, media_id, search_season))
|
||||
if not self.is_running:
|
||||
self.is_running = True
|
||||
asyncio.create_task(self._process_queue())
|
||||
|
||||
async def _process_queue(self):
|
||||
while self.is_running:
|
||||
try:
|
||||
(
|
||||
magnet_url,
|
||||
seeders,
|
||||
tracker,
|
||||
media_id,
|
||||
search_season,
|
||||
) = await self.queue.get()
|
||||
|
||||
async with self.semaphore:
|
||||
try:
|
||||
content = await get_torrent_from_magnet(magnet_url)
|
||||
if content:
|
||||
metadata = extract_torrent_metadata(content)
|
||||
for file in metadata["files"]:
|
||||
parsed = parse(file["name"])
|
||||
|
||||
await add_torrent(
|
||||
metadata["info_hash"],
|
||||
seeders,
|
||||
tracker,
|
||||
media_id,
|
||||
search_season,
|
||||
metadata["announce_list"],
|
||||
file["index"],
|
||||
file["name"],
|
||||
file["size"],
|
||||
parsed,
|
||||
)
|
||||
finally:
|
||||
self.queue.task_done()
|
||||
|
||||
except Exception:
|
||||
await asyncio.sleep(1)
|
||||
|
||||
self.is_running = False
|
||||
|
||||
|
||||
add_torrent_queue = AddTorrentQueue()
|
||||
|
||||
|
||||
class TorrentUpdateQueue:
|
||||
def __init__(self, batch_size: int = 100, flush_interval: float = 5.0):
|
||||
self.queue = asyncio.Queue()
|
||||
self.batch_size = batch_size
|
||||
self.flush_interval = flush_interval
|
||||
self.is_running = False
|
||||
self.batches = {"to_check": [], "to_delete": [], "inserts": [], "updates": []}
|
||||
self.last_error_time = 0
|
||||
self.error_backoff = 1
|
||||
|
||||
async def add_torrent_info(self, file_info: dict, media_id: str = None):
|
||||
await self.queue.put((file_info, media_id))
|
||||
if not self.is_running:
|
||||
self.is_running = True
|
||||
asyncio.create_task(self._process_queue())
|
||||
|
||||
async def _process_queue(self):
|
||||
last_flush_time = time.time()
|
||||
|
||||
while self.is_running:
|
||||
try:
|
||||
while not self.queue.empty():
|
||||
try:
|
||||
file_info, media_id = self.queue.get_nowait()
|
||||
await self._process_file_info(file_info, media_id)
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
|
||||
current_time = time.time()
|
||||
|
||||
if (
|
||||
current_time - last_flush_time >= self.flush_interval
|
||||
or self.queue.empty()
|
||||
) and any(len(batch) > 0 for batch in self.batches.values()):
|
||||
await self._flush_batch()
|
||||
last_flush_time = current_time
|
||||
|
||||
if self.queue.empty() and not any(
|
||||
len(batch) > 0 for batch in self.batches.values()
|
||||
):
|
||||
self.is_running = False
|
||||
break
|
||||
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"Error in _process_queue: {e}")
|
||||
await self._handle_error(e)
|
||||
|
||||
if any(len(batch) > 0 for batch in self.batches.values()):
|
||||
await self._flush_batch()
|
||||
|
||||
self.is_running = False
|
||||
|
||||
async def _flush_batch(self):
|
||||
try:
|
||||
if self.batches["to_check"]:
|
||||
sub_batch_size = 100
|
||||
for i in range(0, len(self.batches["to_check"]), sub_batch_size):
|
||||
sub_batch = self.batches["to_check"][i : i + sub_batch_size]
|
||||
placeholders = []
|
||||
params = {}
|
||||
|
||||
for idx, item in enumerate(sub_batch):
|
||||
key = str(idx)
|
||||
placeholders.append(
|
||||
f"(CAST(:info_hash_{key} AS TEXT) = info_hash AND "
|
||||
f"(CAST(:season_{key} AS INTEGER) IS NULL AND season IS NULL OR season = CAST(:season_{key} AS INTEGER)) AND "
|
||||
f"(CAST(:episode_{key} AS INTEGER) IS NULL AND episode IS NULL OR episode = CAST(:episode_{key} AS INTEGER)))"
|
||||
)
|
||||
params[f"info_hash_{key}"] = item["info_hash"]
|
||||
params[f"season_{key}"] = item["season"]
|
||||
params[f"episode_{key}"] = item["episode"]
|
||||
|
||||
query = f"""
|
||||
SELECT info_hash, season, episode
|
||||
FROM torrents
|
||||
WHERE {" OR ".join(placeholders)}
|
||||
"""
|
||||
|
||||
async with database.transaction():
|
||||
existing_rows = await database.fetch_all(query, params)
|
||||
|
||||
existing_set = {
|
||||
(
|
||||
row["info_hash"],
|
||||
row["season"] if row["season"] is not None else None,
|
||||
row["episode"] if row["episode"] is not None else None,
|
||||
)
|
||||
for row in existing_rows
|
||||
}
|
||||
|
||||
for item in sub_batch:
|
||||
key = (item["info_hash"], item["season"], item["episode"])
|
||||
if key in existing_set:
|
||||
self.batches["updates"].append(item["params"])
|
||||
else:
|
||||
self.batches["inserts"].append(item["params"])
|
||||
|
||||
self.batches["to_check"] = []
|
||||
|
||||
if self.batches["to_delete"]:
|
||||
sub_batch_size = 100
|
||||
for i in range(0, len(self.batches["to_delete"]), sub_batch_size):
|
||||
sub_batch = self.batches["to_delete"][i : i + sub_batch_size]
|
||||
|
||||
placeholders = []
|
||||
params = {}
|
||||
for idx, item in enumerate(sub_batch):
|
||||
key_suffix = f"_{idx}"
|
||||
placeholders.append(
|
||||
f"(CAST(:info_hash{key_suffix} AS TEXT), CAST(:season{key_suffix} AS INTEGER))"
|
||||
)
|
||||
params[f"info_hash{key_suffix}"] = item["info_hash"]
|
||||
params[f"season{key_suffix}"] = item["season"]
|
||||
|
||||
async with database.transaction():
|
||||
delete_query = f"""
|
||||
DELETE FROM torrents
|
||||
WHERE (info_hash, season) IN (
|
||||
{",".join(placeholders)}
|
||||
)
|
||||
AND episode IS NULL
|
||||
"""
|
||||
await database.execute(delete_query, params)
|
||||
|
||||
self.batches["to_delete"] = []
|
||||
|
||||
if self.batches["inserts"]:
|
||||
sub_batch_size = 100
|
||||
for i in range(0, len(self.batches["inserts"]), sub_batch_size):
|
||||
sub_batch = self.batches["inserts"][i : i + sub_batch_size]
|
||||
async with database.transaction():
|
||||
insert_query = f"""
|
||||
INSERT {"OR IGNORE " if settings.DATABASE_TYPE == "sqlite" else ""}
|
||||
INTO torrents
|
||||
VALUES (
|
||||
:media_id,
|
||||
:info_hash,
|
||||
:file_index,
|
||||
:season,
|
||||
:episode,
|
||||
:title,
|
||||
:seeders,
|
||||
:size,
|
||||
:tracker,
|
||||
:sources,
|
||||
:parsed,
|
||||
:timestamp
|
||||
)
|
||||
{" ON CONFLICT DO NOTHING" if settings.DATABASE_TYPE == "postgresql" else ""}
|
||||
"""
|
||||
await database.execute_many(insert_query, sub_batch)
|
||||
|
||||
if len(self.batches["inserts"]) > 0:
|
||||
logger.log(
|
||||
"SCRAPER",
|
||||
f"Inserted {len(self.batches['inserts'])} new torrents in batch",
|
||||
)
|
||||
self.batches["inserts"] = []
|
||||
|
||||
if self.batches["updates"]:
|
||||
sub_batch_size = 100
|
||||
for i in range(0, len(self.batches["updates"]), sub_batch_size):
|
||||
sub_batch = self.batches["updates"][i : i + sub_batch_size]
|
||||
async with database.transaction():
|
||||
update_query = """
|
||||
UPDATE torrents
|
||||
SET title = CAST(:title AS TEXT),
|
||||
file_index = CAST(:file_index AS INTEGER),
|
||||
size = CAST(:size AS BIGINT),
|
||||
seeders = CAST(:seeders AS INTEGER),
|
||||
tracker = CAST(:tracker AS TEXT),
|
||||
sources = CAST(:sources AS TEXT),
|
||||
parsed = CAST(:parsed AS TEXT),
|
||||
timestamp = CAST(:timestamp AS FLOAT),
|
||||
media_id = CAST(:media_id AS TEXT)
|
||||
WHERE info_hash = CAST(:info_hash AS TEXT)
|
||||
AND (CAST(:season AS INTEGER) IS NULL AND season IS NULL OR season = CAST(:season AS INTEGER))
|
||||
AND (CAST(:episode AS INTEGER) IS NULL AND episode IS NULL OR episode = CAST(:episode AS INTEGER))
|
||||
"""
|
||||
await database.execute_many(update_query, sub_batch)
|
||||
|
||||
if len(self.batches["updates"]) > 0:
|
||||
logger.log(
|
||||
"SCRAPER",
|
||||
f"Updated {len(self.batches['updates'])} existing torrents in batch",
|
||||
)
|
||||
self.batches["updates"] = []
|
||||
|
||||
self.error_backoff = 1
|
||||
|
||||
except Exception as e:
|
||||
await self._handle_error(e)
|
||||
|
||||
async def _process_file_info(self, file_info: dict, media_id: str = None):
|
||||
try:
|
||||
params = {
|
||||
"info_hash": file_info["info_hash"],
|
||||
"file_index": file_info["index"],
|
||||
"season": file_info["season"],
|
||||
"episode": file_info["episode"],
|
||||
"title": file_info["title"],
|
||||
"seeders": file_info["seeders"],
|
||||
"size": file_info["size"],
|
||||
"tracker": file_info["tracker"],
|
||||
"sources": orjson.dumps(file_info["sources"]).decode("utf-8"),
|
||||
"parsed": orjson.dumps(
|
||||
file_info["parsed"], default=default_dump
|
||||
).decode("utf-8"),
|
||||
"timestamp": time.time(),
|
||||
"media_id": media_id,
|
||||
}
|
||||
|
||||
self.batches["to_check"].append(
|
||||
{
|
||||
"info_hash": file_info["info_hash"],
|
||||
"season": file_info["season"],
|
||||
"episode": file_info["episode"],
|
||||
"params": params,
|
||||
}
|
||||
)
|
||||
|
||||
if file_info["episode"] is not None:
|
||||
self.batches["to_delete"].append(
|
||||
{"info_hash": file_info["info_hash"], "season": file_info["season"]}
|
||||
)
|
||||
|
||||
await self._check_batch_size()
|
||||
|
||||
finally:
|
||||
self.queue.task_done()
|
||||
|
||||
async def _check_batch_size(self):
|
||||
if any(len(batch) >= self.batch_size for batch in self.batches.values()):
|
||||
await self._flush_batch()
|
||||
self.error_backoff = 1
|
||||
|
||||
async def _handle_error(self, e: Exception):
|
||||
current_time = time.time()
|
||||
if current_time - self.last_error_time < 5:
|
||||
self.error_backoff = min(self.error_backoff * 2, 30)
|
||||
else:
|
||||
self.error_backoff = 1
|
||||
|
||||
self.last_error_time = current_time
|
||||
logger.warning(f"Database error in torrent batch processing: {e}")
|
||||
logger.warning(f"Waiting {self.error_backoff} seconds before retry")
|
||||
await asyncio.sleep(self.error_backoff)
|
||||
|
||||
|
||||
torrent_update_queue = TorrentUpdateQueue()
|
||||
@@ -0,0 +1,18 @@
|
||||
import aiohttp
|
||||
|
||||
from comet.utils.models import trackers
|
||||
from comet.utils.logger import logger
|
||||
|
||||
|
||||
async def download_best_trackers():
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
response = await session.get(
|
||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best.txt"
|
||||
)
|
||||
response = await response.text()
|
||||
|
||||
other_trackers = [tracker for tracker in response.split("\n") if tracker]
|
||||
trackers.extend(other_trackers)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to download best trackers: {e}")
|
||||
@@ -0,0 +1,17 @@
|
||||
services:
|
||||
comet:
|
||||
container_name: comet
|
||||
image: g0ldyy/comet:rewrite
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8000:8000"
|
||||
env_file:
|
||||
- .env
|
||||
volumes:
|
||||
- ./data/comet:/app/data
|
||||
healthcheck:
|
||||
test: wget -qO- http://127.0.0.1:8000/health
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 20s
|
||||
@@ -0,0 +1,11 @@
|
||||
server {
|
||||
server_name example.com;
|
||||
|
||||
location / {
|
||||
proxy_pass http://localhost:8000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
}
|
||||
+24
-34
@@ -1,37 +1,27 @@
|
||||
[tool.poetry]
|
||||
[project]
|
||||
name = "comet"
|
||||
version = "0.1.0"
|
||||
description = "Stremio's fastest torrent/debrid search add-on."
|
||||
authors = ["Goldy"]
|
||||
license = "MIT"
|
||||
readme = "README.md"
|
||||
repository = "https://github.com/g0ldyy/comet"
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.11"
|
||||
uvicorn = "*"
|
||||
fastapi = "*"
|
||||
aiohttp = "*"
|
||||
asyncio = "*"
|
||||
loguru = "*"
|
||||
databases = "*"
|
||||
pydantic-settings = "*"
|
||||
bencode-py = "*"
|
||||
httpx = "*"
|
||||
curl-cffi = "*"
|
||||
orjson = "*"
|
||||
asyncpg = "*"
|
||||
aiosqlite = "*"
|
||||
jinja2 = "*"
|
||||
rank-torrent-name = "*"
|
||||
parsett = "*"
|
||||
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
isort = "*"
|
||||
pyright = "*"
|
||||
pytest = "*"
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
authors = [
|
||||
{ name = "Goldy", email = "153996346+g0ldyy@users.noreply.github.com" }
|
||||
]
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"aiohttp",
|
||||
"aiosqlite",
|
||||
"asyncio",
|
||||
"asyncpg",
|
||||
"bencode-py",
|
||||
"curl-cffi",
|
||||
"databases",
|
||||
"demagnetize",
|
||||
"fastapi",
|
||||
"gunicorn",
|
||||
"jinja2",
|
||||
"loguru",
|
||||
"mediaflow-proxy",
|
||||
"orjson",
|
||||
"pydantic-settings",
|
||||
"rank-torrent-name",
|
||||
"uvicorn",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user