g0ldyy/comet
1
0
Fork 0
mirror of https://github.com/g0ldyy/comet.git synced 2026-01-12 01:16:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge rewrite into main by overwriting files

Browse Source
This commit is contained in:
Goldy
2025-02-28 22:12:57 +01:00
parent 399d858e31
commit dacb7910aa
45 changed files with 4319 additions and 3211 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env-sample
+35 -31
View File
@@ -1,31 +1,35 @@
ADDON_ID=stremio.comet.fast # for Stremio ADDON_ID=stremio.comet.fast # for Stremio
ADDON_NAME=Comet # for Stremio ADDON_NAME=Comet # for Stremio
FASTAPI_HOST=0.0.0.0 FASTAPI_HOST=0.0.0.0
FASTAPI_PORT=8000 FASTAPI_PORT=8000
FASTAPI_WORKERS=1 # remove to destroy CPU -> max performances :) FASTAPI_WORKERS=1
DASHBOARD_ADMIN_PASSWORD=CHANGE_ME # The password to access the dashboard with active connections and soon more... USE_GUNICORN=True # will use uvicorn if False or if on Windows
DATABASE_TYPE=sqlite # or postgresql if you know what you're doing DASHBOARD_ADMIN_PASSWORD=CHANGE_ME # The password to access the dashboard with active connections and soon more...
DATABASE_URL=username:password@hostname:port # to connect to PostgreSQL DATABASE_TYPE=sqlite # or postgresql if you're making a Comet cluster
DATABASE_PATH=data/comet.db # only change it if you know what it is - folders in path must exist - ignored if PostgreSQL used DATABASE_URL=username:password@hostname:port # to connect to PostgreSQL
CACHE_TTL=86400 # cache duration in seconds DATABASE_PATH=data/comet.db # only change it if you know what it is - folders in path must exist - ignored if PostgreSQL used
DEBRID_PROXY_URL=http://127.0.0.1:1080 # https://github.com/cmj2002/warp-docker to bypass Debrid Services and Torrentio server IP blacklist METADATA_CACHE_TTL=2592000 # metadata cache duration in seconds (30 days by default)
INDEXER_MANAGER_TYPE=None # jackett or prowlarr or None if you want to disable it completely and use Zilean or Torrentio TORRENT_CACHE_TTL=1296000 # torrent cache duration in seconds (15 days by default)
INDEXER_MANAGER_URL=http://127.0.0.1:9117 DEBRID_CACHE_TTL=86400 # debrid availability cache duration in seconds (1 day by default)
INDEXER_MANAGER_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX DEBRID_PROXY_URL=http://127.0.0.1:1080 # https://github.com/cmj2002/warp-docker to bypass Debrid Services and Torrentio server IP blacklist
INDEXER_MANAGER_TIMEOUT=60 # maximum time to obtain search results from indexer manager in seconds INDEXER_MANAGER_TYPE=none # jackett or prowlarr or none if you want to disable it completely and use Zilean or Torrentio
INDEXER_MANAGER_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' # for jackett, get the names from https://github.com/Jackett/Jackett/tree/master/src/Jackett.Common/Definitions - for prowlarr you can write them like on the web dashboard INDEXER_MANAGER_URL=http://127.0.0.1:9117
GET_TORRENT_TIMEOUT=5 # maximum time to obtain the torrent info hash in seconds INDEXER_MANAGER_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
ZILEAN_URL=None # for DMM search - https://github.com/iPromKnight/zilean - ex: http://127.0.0.1:8181 INDEXER_MANAGER_TIMEOUT=60 # maximum time to obtain search results from indexer manager in seconds
ZILEAN_TAKE_FIRST=500 # only change it if you know what it is INDEXER_MANAGER_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' # for jackett, get the names from https://github.com/Jackett/Jackett/tree/master/src/Jackett.Common/Definitions - for prowlarr you can write them like on the web dashboard
SCRAPE_TORRENTIO=False # scrape Torrentio GET_TORRENT_TIMEOUT=5 # maximum time to obtain the torrent info hash in seconds
SCRAPE_MEDIAFUSION=False # scrape MediaFusion - has better results for Indian content DOWNLOAD_TORRENT_FILES=False # set to True to enable torrent file retrieval instead of using only magnet link info (infohash and sources, ensuring file index is included in results for Jackett and Prowlarr torrents)
MEDIAFUSION_URL=https://mediafusion.elfhosted.com # Allows you to scrape custom instances of MediaFusion SCRAPE_ZILEAN=False # scrape Zilean/DMM
PROXY_DEBRID_STREAM=False # Proxy Debrid Streams (very useful to use your debrid service on multiple IPs at same time) ZILEAN_URL=https://zilean.elfhosted.com # for DMM search - https://github.com/iPromKnight/zilean - ex: http://127.0.0.1:8181
PROXY_DEBRID_STREAM_PASSWORD=CHANGE_ME # Secret password to enter on configuration page to prevent people from abusing your debrid stream proxy SCRAPE_TORRENTIO=False # scrape Torrentio
PROXY_DEBRID_STREAM_MAX_CONNECTIONS=-1 # IP-Based connection limit for the Debrid Stream Proxy (-1 = disabled) TORRENTIO_URL=https://torrentio.strem.fun # or https://knightcrawler.elfhosted.com if you prefer to scrape the ElfHosted KnightCrawler instance
PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE=realdebrid # if you want your users who use the Debrid Stream Proxy not to have to specify Debrid information, but to use the default one instead SCRAPE_MEDIAFUSION=False # scrape MediaFusion - has better results for Indian content
PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY=CHANGE_ME # if you want your users who use the Debrid Stream Proxy not to have to specify Debrid information, but to use the default one instead MEDIAFUSION_URL=https://mediafusion.elfhosted.com # Allows you to scrape custom instances of MediaFusion
TITLE_MATCH_CHECK=True # disable if you only use Torrentio / MediaFusion and are sure you're only scraping good titles, for example (keep it True if Zilean is enabled) PROXY_DEBRID_STREAM=False # Proxy Debrid Streams (very useful to use your debrid service on multiple IPs at same time)
REMOVE_ADULT_CONTENT=False # detect and remove adult content PROXY_DEBRID_STREAM_PASSWORD=CHANGE_ME # Secret password to enter on configuration page to prevent people from abusing your debrid stream proxy
STREMTHRU_DEFAULT_URL=None # if you want your users to use StremThru without having to specify it PROXY_DEBRID_STREAM_MAX_CONNECTIONS=-1 # IP-Based connection limit for the Debrid Stream Proxy (-1 = disabled)
CUSTOM_HEADER_HTML=None # only set it if you know what it is PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE=realdebrid # if you want your users who use the Debrid Stream Proxy not to have to specify Debrid information, but to use the default one instead
PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY=CHANGE_ME # if you want your users who use the Debrid Stream Proxy not to have to specify Debrid information, but to use the default one instead
REMOVE_ADULT_CONTENT=False # detect and remove adult content
CUSTOM_HEADER_HTML=None # only set it if you know what it is
STREMTHRU_URL=https://stremthru.13377001.xyz # StremThru acts as a proxy between Comet and debrid services to support them all, so you must have it
.github/workflows/docker-build-push.yml
+15 -5
View File
@@ -4,8 +4,7 @@ on:
push: push:
branches: branches:
- main - main
# release: - rewrite
# types: [created]
workflow_dispatch: workflow_dispatch:
jobs: jobs:
@@ -42,6 +41,19 @@ jobs:
username: ${{ secrets.DOCKER_HUB_USERNAME }} username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
- name: Get branch name
id: branch-name
run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
- name: Set Docker tags
id: docker_tags
run: |
if [ "${{ steps.branch-name.outputs.branch }}" = "main" ]; then
echo "tags=ghcr.io/g0ldyy/comet:latest,docker.io/g0ldyy/comet:latest" >> $GITHUB_OUTPUT
else
echo "tags=ghcr.io/g0ldyy/comet:${{ steps.branch-name.outputs.branch }},docker.io/g0ldyy/comet:${{ steps.branch-name.outputs.branch }}" >> $GITHUB_OUTPUT
fi
- name: Build and push Docker image - name: Build and push Docker image
uses: docker/build-push-action@v5.3.0 uses: docker/build-push-action@v5.3.0
with: with:
@@ -51,6 +63,4 @@ jobs:
push: true push: true
cache-from: type=gha cache-from: type=gha
cache-to: type=gha,mode=max cache-to: type=gha,mode=max
tags: | tags: ${{ steps.docker_tags.outputs.tags }}
ghcr.io/g0ldyy/comet:latest
docker.io/g0ldyy/comet:latest
.gitignore
+177 -167
View File
@@ -1,167 +1,177 @@
# Project Based # Project Based
*.db *.db
# Byte-compiled / optimized / DLL files # Byte-compiled / optimized / DLL files
__pycache__/ __pycache__/
*.py[cod] *.py[cod]
*$py.class *$py.class
logs/
.vscode/ # C extensions
*.so
# C extensions
*.so # Distribution / packaging
.Python
# Distribution / packaging build/
.Python develop-eggs/
build/ dist/
develop-eggs/ downloads/
dist/ eggs/
downloads/ .eggs/
eggs/ lib/
.eggs/ lib64/
lib/ parts/
lib64/ sdist/
parts/ var/
sdist/ wheels/
var/ share/python-wheels/
wheels/ *.egg-info/
share/python-wheels/ .installed.cfg
*.egg-info/ *.egg
.installed.cfg MANIFEST
*.egg
MANIFEST # PyInstaller
# Usually these files are written by a python script from a template
# PyInstaller # before PyInstaller builds the exe, so as to inject date/other infos into it.
# Usually these files are written by a python script from a template *.manifest
# before PyInstaller builds the exe, so as to inject date/other infos into it. *.spec
*.manifest
*.spec # Installer logs
pip-log.txt
# Installer logs pip-delete-this-directory.txt
pip-log.txt
pip-delete-this-directory.txt # Unit test / coverage reports
htmlcov/
# Unit test / coverage reports .tox/
htmlcov/ .nox/
.tox/ .coverage
.nox/ .coverage.*
.coverage .cache
.coverage.* nosetests.xml
.cache coverage.xml
nosetests.xml *.cover
coverage.xml *.py,cover
*.cover .hypothesis/
*.py,cover .pytest_cache/
.hypothesis/ cover/
.pytest_cache/
cover/ # Translations
*.mo
# Translations *.pot
*.mo
*.pot # Django stuff:
*.log
# Django stuff: local_settings.py
*.log db.sqlite3
local_settings.py db.sqlite3-journal
db.sqlite3
db.sqlite3-journal # Flask stuff:
instance/
# Flask stuff: .webassets-cache
instance/
.webassets-cache # Scrapy stuff:
.scrapy
# Scrapy stuff:
.scrapy # Sphinx documentation
docs/_build/
# Sphinx documentation
docs/_build/ # PyBuilder
.pybuilder/
# PyBuilder target/
.pybuilder/
target/ # Jupyter Notebook
.ipynb_checkpoints
# Jupyter Notebook
.ipynb_checkpoints # IPython
profile_default/
# IPython ipython_config.py
profile_default/
ipython_config.py # pyenv
# For a library or package, you might want to ignore these files since the code is
# pyenv # intended to run in multiple environments; otherwise, check them in:
# For a library or package, you might want to ignore these files since the code is # .python-version
# intended to run in multiple environments; otherwise, check them in:
# .python-version # pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# pipenv # However, in case of collaboration, if having platform-specific dependencies or dependencies
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # having no cross-platform support, pipenv may install dependencies that don't work, or not
# However, in case of collaboration, if having platform-specific dependencies or dependencies # install all needed dependencies.
# having no cross-platform support, pipenv may install dependencies that don't work, or not #Pipfile.lock
# install all needed dependencies.
#Pipfile.lock # UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# poetry # This is especially recommended for binary packages to ensure reproducibility, and is more
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. # commonly ignored for libraries.
# This is especially recommended for binary packages to ensure reproducibility, and is more uv.lock
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control # poetry
poetry.lock # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# pdm # commonly ignored for libraries.
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#pdm.lock #poetry.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control. # pdm
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
.pdm.toml #pdm.lock
.pdm-python # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
.pdm-build/ # in version control.
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm .pdm.toml
__pypackages__/ .pdm-python
.pdm-build/
# Celery stuff
celerybeat-schedule # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
celerybeat.pid __pypackages__/
# SageMath parsed files # Celery stuff
*.sage.py celerybeat-schedule
celerybeat.pid
# Environments
.env # SageMath parsed files
.venv *.sage.py
env/
venv/ # Environments
ENV/ .env
env.bak/ .venv
venv.bak/ env/
venv/
# Spyder project settings ENV/
.spyderproject env.bak/
.spyproject venv.bak/
# Rope project settings # Spyder project settings
.ropeproject .spyderproject
.spyproject
# mkdocs documentation
/site # Rope project settings
.ropeproject
# mypy
.mypy_cache/ # mkdocs documentation
.dmypy.json /site
dmypy.json
# mypy
# Pyre type checker .mypy_cache/
.pyre/ .dmypy.json
dmypy.json
# pytype static type analyzer
.pytype/ # Pyre type checker
.pyre/
# Cython debug symbols
cython_debug/ # pytype static type analyzer
.pytype/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can # Cython debug symbols
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore cython_debug/
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder. # PyCharm
#.idea/ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
# Ruff stuff:
.ruff_cache/
# PyPI configuration file
.pypirc
CHANGELOG.md
-24
View File
@@ -1,29 +1,5 @@
# Changelog # Changelog
## [1.53.0](https://github.com/g0ldyy/comet/compare/v1.52.0...v1.53.0) (2025-02-07)
### Features
* **debrid/stremthru:** forward client ip ([982c398](https://github.com/g0ldyy/comet/commit/982c398774480badbb60d7214a91c3a62260fee6))
* **debrid/stremthru:** match behavior with mediafusion ([24611a2](https://github.com/g0ldyy/comet/commit/24611a2940dec82cd3cd09aab757cef06a9beb8a))
* **debrid/stremthru:** pass stremio video id for magnet cache check ([786a298](https://github.com/g0ldyy/comet/commit/786a2985ca3ef2463b65c684bc8cef6a86ce772f))
* **debrid/stremthru:** use default url only for specific services ([f0b029d](https://github.com/g0ldyy/comet/commit/f0b029d7933bc85a7ca794c3edea1afccc8d4f28))
* **debrid:** add support for stremthru ([64ba2b3](https://github.com/g0ldyy/comet/commit/64ba2b39fb0faab69e16e8755f024eb82c14f888))
### Bug Fixes
* **debrid/stremthru:** handle missing file index ([32ae49e](https://github.com/g0ldyy/comet/commit/32ae49ed1dd7ef0ab52ec8f6247250b3a8e888c7))
* direct torrent results ([dea8a90](https://github.com/g0ldyy/comet/commit/dea8a9041cb54245e2fdadd9d914c0285efebc56))
## [1.52.0](https://github.com/g0ldyy/comet/compare/v1.51.0...v1.52.0) (2025-01-03)
### Features
* revert random manifest id ([f792529](https://github.com/g0ldyy/comet/commit/f7925298993e904a635041248079cf578d602818))
## [1.51.0](https://github.com/g0ldyy/comet/compare/v1.50.1...v1.51.0) (2024-11-28) ## [1.51.0](https://github.com/g0ldyy/comet/compare/v1.50.1...v1.51.0) (2024-11-28)
Dockerfile
+16 -29
View File
@@ -1,29 +1,16 @@
FROM python:3.11-alpine FROM ghcr.io/astral-sh/uv:python3.11-alpine
LABEL name="Comet" \ LABEL name="Comet" \
description="Stremio's fastest torrent/debrid search add-on." \ description="Stremio's fastest torrent/debrid search add-on." \
url="https://github.com/g0ldyy/comet" url="https://github.com/g0ldyy/comet"
WORKDIR /app WORKDIR /app
ARG DATABASE_PATH ARG DATABASE_PATH
ENV PYTHONUNBUFFERED=1 \ COPY pyproject.toml .
PYTHONDONTWRITEBYTECODE=1 \
POETRY_NO_INTERACTION=1 \ RUN uv sync
POETRY_HOME="/usr/local" \
FORCE_COLOR=1 \ COPY . .
TERM=xterm-256color
ENTRYPOINT ["uv", "run", "python", "-m", "comet.main"]
# Fix python-alpine gcc
RUN apk add --no-cache \
gcc \
musl-dev \
libffi-dev \
make
RUN pip install poetry
COPY pyproject.toml .
RUN poetry install --no-cache --no-root --without dev
COPY . .
ENTRYPOINT ["poetry", "run", "python", "-m", "comet.main"]
LICENSE
+21 -21
View File
@@ -1,21 +1,21 @@
MIT License MIT License
Copyright (c) 2024 Goldy Copyright (c) 2024 Goldy
Permission is hereby granted, free of charge, to any person obtaining a copy Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions: furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software. copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE. SOFTWARE.
README.md
+85 -112
View File
@@ -1,112 +1,85 @@
<h1 align="center" id="title">☄️ Comet - <a href="https://discord.gg/rivenmedia">Discord</a></h1> <h1 align="center" id="title">☄️ Comet - <a href="https://discord.gg/rivenmedia">Discord</a></h1>
<p align="center"><img src="https://socialify.git.ci/g0ldyy/comet/image?description=1&font=Inter&forks=1&language=1&name=1&owner=1&pattern=Solid&stargazers=1&theme=Dark" /></p> <p align="center"><img src="https://socialify.git.ci/g0ldyy/comet/image?description=1&font=Inter&forks=1&language=1&name=1&owner=1&pattern=Solid&stargazers=1&theme=Dark" /></p>
<p align="center"> <p align="center">
<a href="https://ko-fi.com/E1E7ZVMAD"> <a href="https://ko-fi.com/E1E7ZVMAD">
<img src="https://ko-fi.com/img/githubbutton_sm.svg"> <img src="https://ko-fi.com/img/githubbutton_sm.svg">
</a> </a>
</p> </p>
# Features # Features
- The only Stremio addon that can Proxy Debrid Streams to allow use of the Debrid Service on multiple IPs at the same time on the same account! - The only Stremio addon that can Proxy Debrid Streams to allow use of the Debrid Service on multiple IPs at the same time on the same account!
- IP-Based Max Connection Limit and Dashboard for Debrid Stream Proxier - IP-Based Max Connection Limit and Dashboard for Debrid Stream Proxier
- Jackett and Prowlarr support (change the `INDEXER_MANAGER_TYPE` environment variable to `jackett` or `prowlarr`) - Jackett and Prowlarr support (change the `INDEXER_MANAGER_TYPE` environment variable to `jackett` or `prowlarr`)
- [Zilean](https://github.com/iPromKnight/zilean) ([DMM](https://hashlists.debridmediamanager.com/) Scraper) support for even more results - [Zilean](https://github.com/iPromKnight/zilean) ([DMM](https://hashlists.debridmediamanager.com/) Scraper) support for even more results
- [Torrentio](https://torrentio.strem.fun/) Scraper - [Torrentio](https://torrentio.strem.fun/) Scraper
- Caching system ft. SQLite / PostgreSQL - Caching system ft. SQLite / PostgreSQL
- Smart Torrent Ranking powered by [RTN](https://github.com/dreulavelle/rank-torrent-name) - Smart Torrent Ranking powered by [RTN](https://github.com/dreulavelle/rank-torrent-name)
- Proxy support to bypass debrid restrictions - Proxy support to bypass debrid restrictions
- Real-Debrid, All-Debrid, Premiumize, TorBox and Debrid-Link supported - Real-Debrid, All-Debrid, Premiumize, TorBox and Debrid-Link supported
- Direct Torrent supported (do not specify a Debrid API Key on the configuration page (webui) to activate it - it will use the cached results of other users using debrid service) - Direct Torrent supported
- [Kitsu](https://kitsu.io/) support (anime) - [Kitsu](https://kitsu.io/) support (anime)
- Adult Content Filter - Adult Content Filter
- [StremThru](https://github.com/MunifTanjim/stremthru) support - [StremThru](https://github.com/MunifTanjim/stremthru) support
# Installation # Installation
To customize your Comet experience to suit your needs, please first take a look at all the [environment variables](https://github.com/g0ldyy/comet/blob/main/.env-sample)! To customize your Comet experience to suit your needs, please first take a look at all the [environment variables](https://github.com/g0ldyy/comet/blob/main/.env-sample)!
## ElfHosted
A free, public Comet instance is available at https://comet.elfhosted.com ## ElfHosted
A free, public Comet instance is available at https://comet.elfhosted.com
[ElfHosted](https://elfhosted.com) is a geeky [open-source](https://elfhosted.com/open/) PaaS which provides all the "plumbing" (*hosting, security, updates, etc*) for your self-hosted apps.
[ElfHosted](https://elfhosted.com) is a geeky [open-source](https://elfhosted.com/open/) PaaS which provides all the "plumbing" (*hosting, security, updates, etc*) for your self-hosted apps.
ElfHosted offer "one-click" [private Comet instances](https://elfhosted.com/app/comet/), allowing you to customize your indexers, and enabling "Proxy Stream" mode, to permit streaming from multiple source IPs with the same RD token!
ElfHosted offer "one-click" [private Comet instances](https://elfhosted.com/app/comet/), allowing you to customize your indexers, and enabling "Proxy Stream" mode, to permit streaming from multiple source IPs with the same RD token!
> [!IMPORTANT]
> Comet is a top-tier app in the [ElfHosted app catalogue](https://elfhosted.com/apps/). 30% of your subscription goes to the app developer :heart: > [!IMPORTANT]
> Comet is a top-tier app in the [ElfHosted app catalogue](https://elfhosted.com/apps/). 30% of your subscription goes to the app developer :heart:
(*[ElfHosted Discord](https://discord.elfhosted.com)*)
(*[ElfHosted Discord](https://discord.elfhosted.com)*)
## Self Hosted
### From source ## Self Hosted
- Clone the repository and enter the folder ### From source
```sh - Clone the repository and enter the folder
git clone https://github.com/g0ldyy/comet ```sh
cd comet git clone https://github.com/g0ldyy/comet
``` cd comet
- Install dependencies ```
```sh - Install dependencies
pip install poetry ```sh
poetry install pip install uv
```` uv sync
- Start Comet ````
```sh - Start Comet
poetry run python -m comet.main ```sh
```` uv run python -m comet.main
````
### With Docker
- Simply run the Docker image after modifying the environment variables ### With Docker Compose
```sh - Copy *deployment/docker-compose.yml* in a directory
docker run --name comet -p 8000:8000 -d \ - Copy *.env-sample* to *.env* in the same directory and keep only the variables you wish to modify, also remove all comments
-e FASTAPI_HOST=0.0.0.0 \ - Pull the latest version from docker hub
-e FASTAPI_PORT=8000 \ ```sh
-e FASTAPI_WORKERS=1 \ docker compose pull
-e CACHE_TTL=86400 \ ```
-e DEBRID_PROXY_URL=http://127.0.0.1:1080 \ - Run
-e INDEXER_MANAGER_TYPE=jackett \ ```sh
-e INDEXER_MANAGER_URL=http://127.0.0.1:9117 \ docker compose up -d
-e INDEXER_MANAGER_API_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \ ```
-e INDEXER_MANAGER_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' \
-e INDEXER_MANAGER_TIMEOUT=30 \ ### Nginx Reverse Proxy
-e GET_TORRENT_TIMEOUT=5 \ If you want to serve Comet via a Nginx Reverse Proxy, here's the configuration you should use.
g0ldyy/comet ```
``` server {
- To update your container server_name example.com;
- Find your existing container name location / {
```sh proxy_pass http://localhost:8000;
docker ps proxy_set_header Host $host;
``` proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- Stop your existing container proxy_set_header X-Forwarded-Proto $scheme;
```sh }
docker stop <CONTAINER_ID> }
``` ```
- Remove your existing container ## Web UI Showcase
```sh <img src="https://i.imgur.com/SaD365F.png" />
docker rm <CONTAINER_ID>
```
- Pull the latest version from docker hub
```sh
docker pull g0ldyy/comet
```
- Finally, re-run the docker run command
### With Docker Compose
- Copy *compose.yaml* in a directory
- Copy *env-sample* to *.env* in the same directory
- Pull the latest version from docker hub
```sh
docker compose pull
```
- Run
```sh
docker compose up -d
```
## Debrid IP Blacklist
To bypass Real-Debrid's (or AllDebrid) IP blacklist, start a cloudflare-warp container: https://github.com/cmj2002/warp-docker
## Web UI Showcase
<img src="https://i.imgur.com/SaD365F.png" />
comet/api/core.py
+56 -32
View File
@@ -1,39 +1,32 @@
import PTT import random
import RTN import string
import secrets
import orjson
from fastapi import APIRouter, Request from fastapi import APIRouter, Request, Depends, HTTPException
from fastapi.responses import RedirectResponse from fastapi.responses import RedirectResponse, Response
from fastapi.templating import Jinja2Templates from fastapi.templating import Jinja2Templates
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from comet.utils.models import settings from comet.utils.models import settings, web_config, database
from comet.utils.general import config_check, get_debrid_extension from comet.utils.general import config_check
from comet.debrid.manager import get_debrid_extension
templates = Jinja2Templates("comet/templates") templates = Jinja2Templates("comet/templates")
main = APIRouter() main = APIRouter()
security = HTTPBasic()
@main.get("/", status_code=200) @main.get("/")
async def root(): async def root():
return RedirectResponse("/configure") return RedirectResponse("/configure")
@main.get("/health", status_code=200) @main.get("/health")
async def health(): async def health():
return {"status": "ok"} return {"status": "ok"}
indexers = settings.INDEXER_MANAGER_INDEXERS
languages = [language for language in PTT.parse.LANGUAGES_TRANSLATION_TABLE.values()]
languages.insert(0, "Unknown")
languages.insert(1, "Multi")
web_config = {
"indexers": [indexer.replace(" ", "_").lower() for indexer in indexers],
"languages": languages,
"resolutions": [resolution.value for resolution in RTN.models.Resolution],
"resultFormat": ["Title", "Metadata", "Size", "Tracker", "Languages"],
}
@main.get("/configure") @main.get("/configure")
@main.get("/{b64config}/configure") @main.get("/{b64config}/configure")
async def configure(request: Request): async def configure(request: Request):
@@ -45,25 +38,16 @@ async def configure(request: Request):
if settings.CUSTOM_HEADER_HTML if settings.CUSTOM_HEADER_HTML
else "", else "",
"webConfig": web_config, "webConfig": web_config,
"indexerManager": settings.INDEXER_MANAGER_TYPE,
"proxyDebridStream": settings.PROXY_DEBRID_STREAM, "proxyDebridStream": settings.PROXY_DEBRID_STREAM,
"stremthruDefaultUrl": settings.STREMTHRU_DEFAULT_URL or "",
}, },
) )
@main.get("/manifest.json") @main.get("/manifest.json")
@main.get("/{b64config}/manifest.json") @main.get("/{b64config}/manifest.json")
async def manifest(b64config: str = None): async def manifest(request: Request, b64config: str = None):
config = config_check(b64config) base_manifest = {
if not config: "id": f"{settings.ADDON_ID}.{''.join(random.choice(string.ascii_letters) for _ in range(4))}",
config = {"debridService": None}
debrid_extension = get_debrid_extension(config["debridService"], config["debridApiKey"])
return {
"id": settings.ADDON_ID,
"name": f"{settings.ADDON_NAME}{(' | ' + debrid_extension) if debrid_extension is not None else ''}",
"description": "Stremio's fastest torrent/debrid search add-on.", "description": "Stremio's fastest torrent/debrid search add-on.",
"version": "1.0.0", "version": "1.0.0",
"catalogs": [], "catalogs": [],
@@ -79,3 +63,43 @@ async def manifest(b64config: str = None):
"background": "https://i.imgur.com/WwnXB3k.jpeg", "background": "https://i.imgur.com/WwnXB3k.jpeg",
"behaviorHints": {"configurable": True, "configurationRequired": False}, "behaviorHints": {"configurable": True, "configurationRequired": False},
} }
config = config_check(b64config)
if not config:
base_manifest["name"] = "❌ | Comet"
base_manifest["description"] = f"⚠️ OBSOLETE CONFIGURATION, PLEASE RE-CONFIGURE ON {request.url.scheme}://{request.url.netloc} ⚠️"
return base_manifest
debrid_extension = get_debrid_extension(config["debridService"])
base_manifest["name"] = f"{settings.ADDON_NAME}{(' | ' + debrid_extension) if debrid_extension is not None else ''}"
return base_manifest
class CustomORJSONResponse(Response):
media_type = "application/json"
def render(self, content) -> bytes:
assert orjson is not None, "orjson must be installed"
return orjson.dumps(content, option=orjson.OPT_INDENT_2)
def verify_dashboard_auth(credentials: HTTPBasicCredentials = Depends(security)):
is_correct = secrets.compare_digest(
credentials.password, settings.DASHBOARD_ADMIN_PASSWORD
)
if not is_correct:
raise HTTPException(
status_code=401,
detail="Incorrect password",
headers={"WWW-Authenticate": "Basic"},
)
return True
@main.get("/dashboard", response_class=CustomORJSONResponse)
async def dashboard(authenticated: bool = Depends(verify_dashboard_auth)):
rows = await database.fetch_all("SELECT * FROM active_connections")
return rows
comet/api/stream.py
+304 -608
View File
@@ -1,480 +1,248 @@
import asyncio
import time
import aiohttp import aiohttp
import httpx import time
import uuid import mediaflow_proxy.utils.http_utils
import orjson
from fastapi import APIRouter, Request, BackgroundTasks from fastapi import APIRouter, Request, BackgroundTasks
from fastapi.responses import ( from fastapi.responses import (
RedirectResponse,
StreamingResponse,
FileResponse, FileResponse,
Response, RedirectResponse,
) )
from starlette.background import BackgroundTask
from RTN import Torrent, sort_torrents
from comet.debrid.manager import getDebrid from comet.utils.models import settings, database, trackers
from comet.utils.general import ( from comet.utils.general import parse_media_id
config_check, from comet.metadata.manager import MetadataScraper
get_debrid_extension, from comet.scrapers.manager import TorrentManager
get_indexer_manager, from comet.utils.general import config_check, format_title, get_client_ip
get_zilean, from comet.debrid.manager import get_debrid_extension, get_debrid
get_torrentio, from comet.utils.streaming import custom_handle_stream_request
get_mediafusion,
filter,
get_torrent_hash,
translate,
get_balanced_hashes,
format_title,
get_client_ip,
get_aliases,
add_torrent_to_cache,
)
from comet.utils.config import is_proxy_stream_authed, is_proxy_stream_enabled, prepare_debrid_config, should_skip_proxy_stream
from comet.utils.logger import logger from comet.utils.logger import logger
from comet.utils.models import database, rtn, settings, trackers
streams = APIRouter() streams = APIRouter()
@streams.get("/stream/{type}/{id}.json") async def remove_ongoing_search_from_database(media_id: str):
async def stream_noconfig(request: Request, type: str, id: str): await database.execute(
return { "DELETE FROM ongoing_searches WHERE media_id = :media_id",
"streams": [ {"media_id": media_id},
{ )
"name": "[⚠️] Comet",
"description": f"{request.url.scheme}://{request.url.netloc}/configure",
"url": "https://comet.fast",
}
]
}
@streams.get("/{b64config}/stream/{type}/{id}.json") async def is_first_search(media_id: str) -> bool:
try:
await database.execute(
"INSERT INTO first_searches VALUES (:media_id, :timestamp)",
{"media_id": media_id, "timestamp": time.time()},
)
return True
except Exception:
return False
async def background_scrape(
torrent_manager: TorrentManager, media_id: str, debrid_service: str
):
try:
async with aiohttp.ClientSession() as new_session:
await torrent_manager.scrape_torrents(new_session)
if debrid_service != "torrent" and len(torrent_manager.torrents) > 0:
await torrent_manager.get_and_cache_debrid_availability(new_session)
logger.log(
"SCRAPER",
"📥 Background scrape + availability check complete!",
)
except Exception as e:
logger.log("SCRAPER", f"❌ Background scrape + availability check failed: {e}")
finally:
await remove_ongoing_search_from_database(media_id)
@streams.get("/stream/{media_type}/{media_id}.json")
@streams.get("/{b64config}/stream/{media_type}/{media_id}.json")
async def stream( async def stream(
request: Request, request: Request,
b64config: str, media_type: str,
type: str, media_id: str,
id: str,
background_tasks: BackgroundTasks, background_tasks: BackgroundTasks,
b64config: str = None,
): ):
config = config_check(b64config) config = config_check(b64config)
if not config: if not config:
return { return {
"streams": [ "streams": [
{ {
"name": "[⚠️] Comet", "name": "[❌] Comet",
"description": "Invalid Comet config.", "description": f"⚠️ OBSOLETE CONFIGURATION, PLEASE RE-CONFIGURE ON {request.url.scheme}://{request.url.netloc} ⚠️",
"url": "https://comet.fast",
}
]
}
ongoing_search = await database.fetch_one(
"SELECT timestamp FROM ongoing_searches WHERE media_id = :media_id",
{"media_id": media_id},
)
if ongoing_search:
return {
"streams": [
{
"name": "[🔄] Comet",
"description": "Search in progress, please try again in a few seconds...",
"url": "https://comet.fast", "url": "https://comet.fast",
} }
] ]
} }
connector = aiohttp.TCPConnector(limit=0) connector = aiohttp.TCPConnector(limit=0)
async with aiohttp.ClientSession( async with aiohttp.ClientSession(connector=connector) as session:
connector=connector, raise_for_status=True metadata, aliases = await MetadataScraper(session).fetch_metadata_and_aliases(
) as session: media_type, media_id
full_id = id
season = None
episode = None
if type == "series":
info = id.split(":")
id = info[0]
season = int(info[1])
episode = int(info[2])
year = None
year_end = None
try:
kitsu = False
if id == "kitsu":
kitsu = True
get_metadata = await session.get(
f"https://kitsu.io/api/edge/anime/{season}"
)
metadata = await get_metadata.json()
name = metadata["data"]["attributes"]["canonicalTitle"]
season = 1
else:
get_metadata = await session.get(
f"https://v3.sg.media-imdb.com/suggestion/a/{id}.json"
)
metadata = await get_metadata.json()
element = metadata["d"][
0
if metadata["d"][0]["id"]
not in ["/imdbpicks/summer-watch-guide", "/emmys"]
else 1
]
for element in metadata["d"]:
if "/" not in element["id"]:
break
name = element["l"]
year = element.get("y")
if "yr" in element:
year_end = int(element["yr"].split("-")[1])
except Exception as e:
logger.warning(f"Exception while getting metadata for {id}: {e}")
return {
"streams": [
{
"name": "[⚠️] Comet",
"description": f"Can't get metadata for {id}",
"url": "https://comet.fast",
}
]
}
name = translate(name)
log_name = name
if type == "series":
log_name = f"{name} S{season:02d}E{episode:02d}"
prepare_debrid_config(config)
if config["debridApiKey"] == "":
services = ["realdebrid", "alldebrid", "premiumize", "torbox", "debridlink", "stremthru"]
debrid_emoji = "⬇️"
else:
services = [config["debridService"]]
debrid_emoji = "⚡"
results = []
if (
is_proxy_stream_enabled(config)
and not is_proxy_stream_authed(config)
):
results.append(
{
"name": "[⚠️] Comet",
"description": "Debrid Stream Proxy Password incorrect.\nStreams will not be proxied.",
"url": "https://comet.fast",
}
)
indexers = config["indexers"].copy()
if settings.SCRAPE_TORRENTIO:
indexers.append("torrentio")
if settings.SCRAPE_MEDIAFUSION:
indexers.append("mediafusion")
if settings.ZILEAN_URL:
indexers.append("dmm")
indexers_json = orjson.dumps(indexers).decode("utf-8")
all_sorted_ranked_files = {}
trackers_found = (
set()
) # we want to check that we have a cache for each of the user's trackers
the_time = time.time()
cache_ttl = settings.CACHE_TTL
for debrid_service in services:
cached_results = await database.fetch_all(
f"""
SELECT info_hash, tracker, data
FROM cache
WHERE debridService = :debrid_service
AND name = :name
AND ((cast(:season as INTEGER) IS NULL AND season IS NULL) OR season = cast(:season as INTEGER))
AND ((cast(:episode as INTEGER) IS NULL AND episode IS NULL) OR episode = cast(:episode as INTEGER))
AND tracker IN (SELECT cast(value as TEXT) FROM {'json_array_elements_text' if settings.DATABASE_TYPE == 'postgresql' else 'json_each'}(:indexers))
AND timestamp + :cache_ttl >= :current_time
""",
{
"debrid_service": debrid_service,
"name": name,
"season": season,
"episode": episode,
"indexers": indexers_json,
"cache_ttl": cache_ttl,
"current_time": the_time,
},
)
for result in cached_results:
trackers_found.add(result["tracker"].lower())
hash = result["info_hash"]
if "searched" in hash:
continue
all_sorted_ranked_files[hash] = orjson.loads(result["data"])
if len(all_sorted_ranked_files) != 0 and set(indexers).issubset(trackers_found):
debrid_extension = get_debrid_extension(
debrid_service, config["debridApiKey"]
)
balanced_hashes = get_balanced_hashes(all_sorted_ranked_files, config)
for resolution in balanced_hashes:
for hash in balanced_hashes[resolution]:
data = all_sorted_ranked_files[hash]["data"]
the_stream = {
"name": f"[{debrid_extension}{debrid_emoji}] Comet {data['resolution']}",
"description": format_title(data, config),
"torrentTitle": (
data["torrent_title"] if "torrent_title" in data else None
),
"torrentSize": (
data["torrent_size"] if "torrent_size" in data else None
),
"behaviorHints": {
"filename": data["raw_title"],
"bingeGroup": "comet|" + hash,
},
}
if config["debridApiKey"] != "":
the_stream["url"] = (
f"{request.url.scheme}://{request.url.netloc}/{b64config}/playback/{hash}/{data['index']}"
)
else:
the_stream["infoHash"] = hash
index = str(data["index"])
the_stream["fileIdx"] = (
1 if "|" in index else int(index)
) # 1 because for Premiumize it's impossible to get the file index
the_stream["sources"] = trackers
results.append(the_stream)
logger.info(
f"{len(all_sorted_ranked_files)} cached results found for {log_name}"
)
return {"streams": results}
if config["debridApiKey"] == "":
return {
"streams": [
{
"name": "[⚠️] Comet",
"description": "No cache found for Direct Torrenting.",
"url": "https://comet.fast",
}
]
}
logger.info(f"No cache found for {log_name} with user configuration")
debrid = getDebrid(session, config, get_client_ip(request))
check_premium = await debrid.check_premium()
if not check_premium:
additional_info = ""
if config["debridService"] == "alldebrid":
additional_info = "\nCheck your email!"
return {
"streams": [
{
"name": "[⚠️] Comet",
"description": f"Invalid {config['debridService']} account.{additional_info}",
"url": "https://comet.fast",
}
]
}
indexer_manager_type = settings.INDEXER_MANAGER_TYPE
search_indexer = len(config["indexers"]) != 0
torrents = []
tasks = []
if indexer_manager_type and search_indexer:
logger.info(
f"Start of {indexer_manager_type} search for {log_name} with indexers {config['indexers']}"
)
search_terms = [name]
if type == "series":
if not kitsu:
search_terms.append(f"{name} S{season:02d}E{episode:02d}")
else:
search_terms.append(f"{name} {episode}")
tasks.extend(
get_indexer_manager(
session, indexer_manager_type, config["indexers"], term
)
for term in search_terms
)
else:
logger.info(
f"No indexer {'manager ' if not indexer_manager_type else ''}{'selected by user' if indexer_manager_type else 'defined'} for {log_name}"
)
if settings.ZILEAN_URL:
tasks.append(get_zilean(session, name, log_name, season, episode))
if settings.SCRAPE_TORRENTIO:
tasks.append(get_torrentio(log_name, type, full_id))
if settings.SCRAPE_MEDIAFUSION:
tasks.append(get_mediafusion(log_name, type, full_id))
search_response = await asyncio.gather(*tasks)
for results in search_response:
for result in results:
torrents.append(result)
logger.info(
f"{len(torrents)} unique torrents found for {log_name}"
+ (
" with "
+ ", ".join(
part
for part in [
indexer_manager_type,
"Zilean" if settings.ZILEAN_URL else None,
"Torrentio" if settings.SCRAPE_TORRENTIO else None,
"MediaFusion" if settings.SCRAPE_MEDIAFUSION else None,
]
if part
)
if any(
[
indexer_manager_type,
settings.ZILEAN_URL,
settings.SCRAPE_TORRENTIO,
settings.SCRAPE_MEDIAFUSION,
]
)
else ""
)
) )
if metadata is None:
logger.log("SCRAPER", f"❌ Failed to fetch metadata for {media_id}")
return {
"streams": [
{
"name": "[⚠️] Comet",
"description": "Unable to get metadata.",
"url": "https://comet.fast",
}
]
}
if len(torrents) == 0: title = metadata["title"]
return {"streams": []} year = metadata["year"]
year_end = metadata["year_end"]
season = metadata["season"]
episode = metadata["episode"]
if settings.TITLE_MATCH_CHECK: log_title = f"({media_id}) {title}"
aliases = await get_aliases( if media_type == "series":
session, "movies" if type == "movie" else "shows", id log_title += f" S{season:02d}E{episode:02d}"
)
indexed_torrents = [(i, torrents[i]["Title"]) for i in range(len(torrents))] logger.log("SCRAPER", f"🔍 Starting search for {log_title}")
chunk_size = 50
chunks = [
indexed_torrents[i : i + chunk_size]
for i in range(0, len(indexed_torrents), chunk_size)
]
remove_adult_content = ( id, season, episode = parse_media_id(media_type, media_id)
settings.REMOVE_ADULT_CONTENT and config["removeTrash"] media_only_id = id
)
tasks = []
for chunk in chunks:
tasks.append(
filter(chunk, name, year, year_end, aliases, remove_adult_content)
)
filtered_torrents = await asyncio.gather(*tasks) debrid_service = config["debridService"]
index_less = 0 torrent_manager = TorrentManager(
for result in filtered_torrents: debrid_service,
for filtered in result: config["debridApiKey"],
if not filtered[1]: get_client_ip(request),
del torrents[filtered[0] - index_less] media_type,
index_less += 1 media_id,
continue media_only_id,
title,
logger.info( year,
f"{len(torrents)} torrents passed title match check for {log_name}" year_end,
)
if len(torrents) == 0:
return {"streams": []}
tasks = []
for i in range(len(torrents)):
tasks.append(get_torrent_hash(session, (i, torrents[i])))
torrent_hashes = await asyncio.gather(*tasks)
index_less = 0
for hash in torrent_hashes:
if not hash[1]:
del torrents[hash[0] - index_less]
index_less += 1
continue
torrents[hash[0] - index_less]["InfoHash"] = hash[1]
logger.info(f"{len(torrents)} info hashes found for {log_name}")
if len(torrents) == 0:
return {"streams": []}
files = await debrid.get_files(
list({hash[1] for hash in torrent_hashes if hash[1] is not None}),
type,
season, season,
episode, episode,
kitsu, aliases,
video_id=full_id, settings.REMOVE_ADULT_CONTENT and config["removeTrash"],
) )
ranked_files = set() await torrent_manager.get_cached_torrents()
torrents_by_hash = {torrent["InfoHash"]: torrent for torrent in torrents} logger.log(
for hash in files: "SCRAPER", f"📦 Found cached torrents: {len(torrent_manager.torrents)}"
try:
ranked_file = rtn.rank(
torrents_by_hash[hash]["Title"],
hash,
remove_trash=False, # user can choose if he wants to remove it
)
ranked_files.add(ranked_file)
except:
pass
sorted_ranked_files = sort_torrents(ranked_files)
len_sorted_ranked_files = len(sorted_ranked_files)
logger.info(
f"{len_sorted_ranked_files} cached files found on {config['debridService']} for {log_name}"
) )
if len_sorted_ranked_files == 0: is_first = await is_first_search(media_id)
return {"streams": []} has_cached_results = len(torrent_manager.torrents) > 0
sorted_ranked_files = { cached_results = []
key: (value.model_dump() if isinstance(value, Torrent) else value) non_cached_results = []
for key, value in sorted_ranked_files.items()
} if not has_cached_results:
for hash in sorted_ranked_files: # needed for caching logger.log("SCRAPER", f"🔎 Starting new search for {log_title}")
sorted_ranked_files[hash]["data"]["title"] = files[hash]["title"] await database.execute(
sorted_ranked_files[hash]["data"]["torrent_title"] = torrents_by_hash[hash][ f"INSERT {'OR IGNORE ' if settings.DATABASE_TYPE == 'sqlite' else ''}INTO ongoing_searches VALUES (:media_id, :timestamp){' ON CONFLICT DO NOTHING' if settings.DATABASE_TYPE == 'postgresql' else ''}",
"Title" {"media_id": media_id, "timestamp": time.time()},
]
sorted_ranked_files[hash]["data"]["tracker"] = torrents_by_hash[hash][
"Tracker"
]
torrent_size = torrents_by_hash[hash]["Size"]
sorted_ranked_files[hash]["data"]["size"] = (
files[hash]["size"]
) )
sorted_ranked_files[hash]["data"]["torrent_size"] = ( background_tasks.add_task(remove_ongoing_search_from_database, media_id)
torrent_size if torrent_size else files[hash]["size"]
await torrent_manager.scrape_torrents(session)
logger.log(
"SCRAPER",
f"📥 Scraped torrents: {len(torrent_manager.torrents)}",
)
elif is_first:
logger.log(
"SCRAPER",
f"🔄 Starting background scrape + availability check for {log_title}",
)
await database.execute(
f"INSERT {'OR IGNORE ' if settings.DATABASE_TYPE == 'sqlite' else ''}INTO ongoing_searches VALUES (:media_id, :timestamp){' ON CONFLICT DO NOTHING' if settings.DATABASE_TYPE == 'postgresql' else ''}",
{"media_id": media_id, "timestamp": time.time()},
) )
sorted_ranked_files[hash]["data"]["index"] = files[hash]["index"]
background_tasks.add_task( background_tasks.add_task(
add_torrent_to_cache, config, name, season, episode, sorted_ranked_files background_scrape, torrent_manager, media_id, debrid_service
) )
logger.info(f"Results have been cached for {log_name}") cached_results.append(
{
"name": "[🔄] Comet",
"description": "First search for this media - More results will be available in a few seconds...",
"url": "https://comet.fast",
}
)
debrid_extension = get_debrid_extension(config["debridService"], config["debridApiKey"]) await torrent_manager.get_cached_availability()
balanced_hashes = get_balanced_hashes(sorted_ranked_files, config)
results = []
if ( if (
is_proxy_stream_enabled(config) (
and not is_proxy_stream_authed(config) not has_cached_results
or sum(
1
for torrent in torrent_manager.torrents.values()
if torrent["cached"]
)
== 0
)
and len(torrent_manager.torrents) > 0
and debrid_service != "torrent"
): ):
results.append( logger.log("SCRAPER", "🔄 Checking availability on debrid service...")
await torrent_manager.get_and_cache_debrid_availability(session)
if debrid_service != "torrent":
cached_count = sum(
1 for torrent in torrent_manager.torrents.values() if torrent["cached"]
)
logger.log(
"SCRAPER",
f"💾 Available cached torrents on {debrid_service}: {cached_count}/{len(torrent_manager.torrents)}",
)
initial_torrent_count = len(torrent_manager.torrents)
torrent_manager.rank_torrents(
config["rtnSettings"],
config["rtnRanking"],
config["maxResultsPerResolution"],
config["maxSize"],
config["cachedOnly"],
config["removeTrash"],
)
logger.log(
"SCRAPER",
f"⚖️ Torrents after RTN filtering: {len(torrent_manager.ranked_torrents)}/{initial_torrent_count}",
)
debrid_extension = get_debrid_extension(debrid_service)
if (
config["debridStreamProxyPassword"] != ""
and settings.PROXY_DEBRID_STREAM
and settings.PROXY_DEBRID_STREAM_PASSWORD
!= config["debridStreamProxyPassword"]
):
cached_results.append(
{ {
"name": "[⚠️] Comet", "name": "[⚠️] Comet",
"description": "Debrid Stream Proxy Password incorrect.\nStreams will not be proxied.", "description": "Debrid Stream Proxy Password incorrect.\nStreams will not be proxied.",
@@ -482,209 +250,137 @@ async def stream(
} }
) )
for resolution in balanced_hashes: result_season = season if season is not None else "n"
for hash in balanced_hashes[resolution]: result_episode = episode if episode is not None else "n"
data = sorted_ranked_files[hash]["data"]
index = data['index'] torrents = torrent_manager.torrents
if index == -1: for info_hash in torrent_manager.ranked_torrents:
index = data['title'] torrent = torrents[info_hash]
url = f"{request.url.scheme}://{request.url.netloc}/{b64config}/playback/{hash}/{index}" rtn_data = torrent["parsed"]
results.append(
{ debrid_emoji = (
"name": f"[{debrid_extension}⚡] Comet {data['resolution']}", "🧲"
"description": format_title(data, config), if debrid_service == "torrent"
"torrentTitle": data["torrent_title"], else ("⚡" if torrent["cached"] else "⬇️")
"torrentSize": data["torrent_size"], )
"url": url,
"behaviorHints": { the_stream = {
"filename": data["raw_title"], "name": f"[{debrid_extension}{debrid_emoji}] Comet {rtn_data.resolution}",
"bingeGroup": "comet|" + hash, "description": format_title(
}, rtn_data,
} torrent["title"],
torrent["seeders"],
torrent["size"],
torrent["tracker"],
config["resultFormat"],
),
"behaviorHints": {
"bingeGroup": "comet|" + info_hash,
"videoSize": torrent["size"],
"filename": rtn_data.raw_title,
},
}
if debrid_service == "torrent":
the_stream["infoHash"] = info_hash
if torrent["fileIndex"] is not None:
the_stream["fileIdx"] = torrent["fileIndex"]
if len(torrent["sources"]) == 0:
the_stream["sources"] = trackers
else:
the_stream["sources"] = torrent["sources"]
else:
the_stream["url"] = (
f"{request.url.scheme}://{request.url.netloc}/{b64config}/playback/{info_hash}/{torrent['fileIndex'] if torrent['cached'] and torrent['fileIndex'] is not None else 'n'}/{title}/{result_season}/{result_episode}"
) )
return {"streams": results} if torrent["cached"]:
cached_results.append(the_stream)
else:
non_cached_results.append(the_stream)
return {"streams": cached_results + non_cached_results}
@streams.head("/{b64config}/playback/{hash}/{index}") @streams.get("/{b64config}/playback/{hash}/{index}/{name}/{season}/{episode}")
async def playback(b64config: str, hash: str, index: str): async def playback(
return RedirectResponse("https://stremio.fast", status_code=302) request: Request,
b64config: str,
hash: str,
class CustomORJSONResponse(Response): index: str,
media_type = "application/json" name: str,
season: str,
def render(self, content) -> bytes: episode: str,
assert orjson is not None, "orjson must be installed" ):
return orjson.dumps(content, option=orjson.OPT_INDENT_2)
@streams.get("/active-connections", response_class=CustomORJSONResponse)
async def active_connections(request: Request, password: str):
if password != settings.DASHBOARD_ADMIN_PASSWORD:
return "Invalid Password"
active_connections = await database.fetch_all("SELECT * FROM active_connections")
return {
"total_connections": len(active_connections),
"active_connections": active_connections,
}
@streams.get("/{b64config}/playback/{hash}/{index}")
async def playback(request: Request, b64config: str, hash: str, index: str):
config = config_check(b64config) config = config_check(b64config)
if not config: # if not config:
return FileResponse("comet/assets/invalidconfig.mp4") # return FileResponse("comet/assets/invalidconfig.mp4")
prepare_debrid_config(config) season = int(season) if season != "n" else None
episode = int(episode) if episode != "n" else None
async with aiohttp.ClientSession(raise_for_status=True) as session: async with aiohttp.ClientSession() as session:
# Check for cached download link
cached_link = await database.fetch_one( cached_link = await database.fetch_one(
f"SELECT link, timestamp FROM download_links WHERE debrid_key = '{config['debridApiKey']}' AND hash = '{hash}' AND file_index = '{index}'" f"SELECT download_url FROM download_links_cache WHERE debrid_key = '{config['debridApiKey']}' AND info_hash = '{hash}' AND ((cast(:season as INTEGER) IS NULL AND season IS NULL) OR season = cast(:season as INTEGER)) AND ((cast(:episode as INTEGER) IS NULL AND episode IS NULL) OR episode = cast(:episode as INTEGER)) AND timestamp + 3600 >= :current_time",
{
"current_time": time.time(),
"season": season,
"episode": season,
},
) )
current_time = time.time() download_url = None
download_link = None
if cached_link: if cached_link:
link = cached_link["link"] download_url = cached_link["download_url"]
timestamp = cached_link["timestamp"]
if current_time - timestamp < 3600:
download_link = link
else:
# Cache expired, remove old entry
await database.execute(
f"DELETE FROM download_links WHERE debrid_key = '{config['debridApiKey']}' AND hash = '{hash}' AND file_index = '{index}'"
)
ip = get_client_ip(request) ip = get_client_ip(request)
if download_url is None:
if not download_link: debrid = get_debrid(
debrid = getDebrid(
session, session,
config, None,
ip None,
if ( config["debridService"],
not is_proxy_stream_enabled(config) config["debridApiKey"],
or not is_proxy_stream_authed(config) ip,
)
else "",
) )
download_link = await debrid.generate_download_link(hash, index) download_url = await debrid.generate_download_link(
if not download_link: hash, index, name, season, episode
)
if not download_url:
return FileResponse("comet/assets/uncached.mp4") return FileResponse("comet/assets/uncached.mp4")
# Cache the new download link query = f"""
INSERT {"OR IGNORE " if settings.DATABASE_TYPE == "sqlite" else ""}
INTO download_links_cache
VALUES (:debrid_key, :info_hash, :season, :episode, :download_url, :timestamp)
{" ON CONFLICT DO NOTHING" if settings.DATABASE_TYPE == "postgresql" else ""}
"""
await database.execute( await database.execute(
f"INSERT {'OR IGNORE ' if settings.DATABASE_TYPE == 'sqlite' else ''}INTO download_links (debrid_key, hash, file_index, link, timestamp) VALUES (:debrid_key, :hash, :file_index, :link, :timestamp){' ON CONFLICT DO NOTHING' if settings.DATABASE_TYPE == 'postgresql' else ''}", query,
{ {
"debrid_key": config["debridApiKey"], "debrid_key": config["debridApiKey"],
"hash": hash, "info_hash": hash,
"file_index": index, "season": season,
"link": download_link, "episode": episode,
"timestamp": current_time, "download_url": download_url,
"timestamp": time.time(),
}, },
) )
if should_skip_proxy_stream(config):
return RedirectResponse(download_link, status_code=302)
if ( if (
is_proxy_stream_enabled(config) settings.PROXY_DEBRID_STREAM
and is_proxy_stream_authed(config) and settings.PROXY_DEBRID_STREAM_PASSWORD
== config["debridStreamProxyPassword"]
): ):
if settings.PROXY_DEBRID_STREAM_MAX_CONNECTIONS != -1: return await custom_handle_stream_request(
active_ip_connections = await database.fetch_all( request.method,
"SELECT ip, COUNT(*) as connections FROM active_connections GROUP BY ip" download_url,
) mediaflow_proxy.utils.http_utils.get_proxy_headers(request),
if any( media_id=hash,
connection["ip"] == ip ip=ip,
and connection["connections"] )
>= settings.PROXY_DEBRID_STREAM_MAX_CONNECTIONS
for connection in active_ip_connections
):
return FileResponse("comet/assets/proxylimit.mp4")
proxy = None return RedirectResponse(download_url, status_code=302)
class Streamer:
def __init__(self, id: str):
self.id = id
self.client = httpx.AsyncClient(proxy=proxy, timeout=None)
self.response = None
async def stream_content(self, headers: dict):
async with self.client.stream(
"GET", download_link, headers=headers
) as self.response:
async for chunk in self.response.aiter_raw():
yield chunk
async def close(self):
await database.execute(
f"DELETE FROM active_connections WHERE id = '{self.id}'"
)
if self.response is not None:
await self.response.aclose()
if self.client is not None:
await self.client.aclose()
range_header = request.headers.get("range", "bytes=0-")
try:
if config["debridService"] != "torbox":
response = await session.head(
download_link, headers={"Range": range_header}
)
else:
response = await session.get(
download_link, headers={"Range": range_header}
)
except aiohttp.ClientResponseError as e:
if e.status == 503 and config["debridService"] == "alldebrid":
proxy = (
settings.DEBRID_PROXY_URL
) # proxy is needed only to proxy alldebrid streams
response = await session.head(
download_link, headers={"Range": range_header}, proxy=proxy
)
else:
logger.warning(f"Exception while proxying {download_link}: {e}")
return
if response.status == 206 or (
response.status == 200 and config["debridService"] == "torbox"
):
id = str(uuid.uuid4())
await database.execute(
f"INSERT {'OR IGNORE ' if settings.DATABASE_TYPE == 'sqlite' else ''}INTO active_connections (id, ip, content, timestamp) VALUES (:id, :ip, :content, :timestamp){' ON CONFLICT DO NOTHING' if settings.DATABASE_TYPE == 'postgresql' else ''}",
{
"id": id,
"ip": ip,
"content": str(response.url),
"timestamp": current_time,
},
)
streamer = Streamer(id)
return StreamingResponse(
streamer.stream_content({"Range": range_header}),
status_code=206,
headers={
"Content-Range": response.headers["Content-Range"],
"Content-Length": response.headers["Content-Length"],
"Accept-Ranges": "bytes",
},
background=BackgroundTask(streamer.close),
)
return FileResponse("comet/assets/uncached.mp4")
return RedirectResponse(download_link, status_code=302)
comet/debrid/alldebrid.py
+3 -167
View File
@@ -1,172 +1,8 @@
import aiohttp import aiohttp
import asyncio
from RTN import parse
from comet.utils.general import is_video
from comet.utils.logger import logger
from comet.utils.models import settings
class AllDebrid: class AllDebrid:
def __init__(self, session: aiohttp.ClientSession, debrid_api_key: str): def __init__(
session.headers["Authorization"] = f"Bearer {debrid_api_key}" self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
self.session = session
self.proxy = None
self.api_url = "http://api.alldebrid.com/v4"
self.agent = "comet"
async def check_premium(self):
try:
check_premium = await self.session.get(
f"{self.api_url}/user?agent={self.agent}"
)
check_premium = await check_premium.text()
if '"isPremium":true' in check_premium:
return True
except Exception as e:
logger.warning(
f"Exception while checking premium status on All-Debrid: {e}"
)
return False
async def get_instant(self, chunk: list):
try:
get_instant = await self.session.get(
f"{self.api_url}/magnet/instant?agent={self.agent}&magnets[]={'&magnets[]='.join(chunk)}"
)
return await get_instant.json()
except Exception as e:
logger.warning(
f"Exception while checking hashes instant availability on All-Debrid: {e}"
)
async def get_files(
self, torrent_hashes: list, type: str, season: str, episode: str, kitsu: bool, **kwargs
): ):
chunk_size = 500 pass
chunks = [
torrent_hashes[i : i + chunk_size]
for i in range(0, len(torrent_hashes), chunk_size)
]
tasks = []
for chunk in chunks:
tasks.append(self.get_instant(chunk))
responses = await asyncio.gather(*tasks)
availability = [response for response in responses if response]
files = {}
if type == "series":
for result in availability:
if "status" not in result or result["status"] != "success":
continue
for magnet in result["data"]["magnets"]:
if not magnet["instant"]:
continue
for file in magnet["files"]:
filename = file["n"]
pack = False
if "e" in file: # PACK
filename = file["e"][0]["n"]
pack = True
if not is_video(filename):
continue
if "sample" in filename.lower():
continue
filename_parsed = parse(filename)
if episode not in filename_parsed.episodes:
continue
if kitsu:
if filename_parsed.seasons:
continue
else:
if season not in filename_parsed.seasons:
continue
files[magnet["hash"]] = {
"index": magnet["files"].index(file),
"title": filename,
"size": file["e"][0]["s"] if pack else file["s"],
}
break
else:
for result in availability:
if "status" not in result or result["status"] != "success":
continue
for magnet in result["data"]["magnets"]:
if not magnet["instant"]:
continue
for file in magnet["files"]:
filename = file["n"]
if not is_video(filename):
continue
if "sample" in filename.lower():
continue
files[magnet["hash"]] = {
"index": magnet["files"].index(file),
"title": filename,
"size": file["s"],
}
break
return files
async def generate_download_link(self, hash: str, index: str):
try:
check_blacklisted = await self.session.get(
f"{self.api_url}/magnet/upload?agent=comet&magnets[]={hash}"
)
check_blacklisted = await check_blacklisted.text()
if "NO_SERVER" in check_blacklisted:
self.proxy = settings.DEBRID_PROXY_URL
if not self.proxy:
logger.warning(
"All-Debrid blacklisted server's IP. No proxy found."
)
else:
logger.warning(
f"All-Debrid blacklisted server's IP. Switching to proxy {self.proxy} for {hash}|{index}"
)
upload_magnet = await self.session.get(
f"{self.api_url}/magnet/upload?agent=comet&magnets[]={hash}",
proxy=self.proxy,
)
upload_magnet = await upload_magnet.json()
get_magnet_status = await self.session.get(
f"{self.api_url}/magnet/status?agent=comet&id={upload_magnet['data']['magnets'][0]['id']}",
proxy=self.proxy,
)
get_magnet_status = await get_magnet_status.json()
unlock_link = await self.session.get(
f"{self.api_url}/link/unlock?agent=comet&link={get_magnet_status['data']['magnets']['links'][int(index)]['link']}",
proxy=self.proxy,
)
unlock_link = await unlock_link.json()
return unlock_link["data"]["link"]
except Exception as e:
logger.warning(
f"Exception while getting download link from All-Debrid for {hash}|{index}: {e}"
)
comet/debrid/debridlink.py
+3 -135
View File
@@ -1,140 +1,8 @@
import aiohttp import aiohttp
import asyncio
from RTN import parse
from comet.utils.general import is_video
from comet.utils.logger import logger
class DebridLink: class DebridLink:
def __init__(self, session: aiohttp.ClientSession, debrid_api_key: str): def __init__(
session.headers["Authorization"] = f"Bearer {debrid_api_key}" self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
self.session = session
self.proxy = None
self.api_url = "https://debrid-link.com/api/v2"
async def check_premium(self):
try:
check_premium = await self.session.get(f"{self.api_url}/account/infos")
check_premium = await check_premium.text()
if '"accountType":1' in check_premium:
return True
except Exception as e:
logger.warning(
f"Exception while checking premium status on Debrid-Link: {e}"
)
return False
async def get_instant(self, chunk: list):
responses = []
for hash in chunk:
try:
add_torrent = await self.session.post(
f"{self.api_url}/seedbox/add",
data={"url": hash, "wait": True, "async": True},
)
add_torrent = await add_torrent.json()
torrent_id = add_torrent["value"]["id"]
await self.session.delete(f"{self.api_url}/seedbox/{torrent_id}/remove")
responses.append(add_torrent)
except:
pass
return responses
async def get_files(
self, torrent_hashes: list, type: str, season: str, episode: str, kitsu: bool, **kwargs
): ):
chunk_size = 10 pass
chunks = [
torrent_hashes[i : i + chunk_size]
for i in range(0, len(torrent_hashes), chunk_size)
]
tasks = []
for chunk in chunks:
tasks.append(self.get_instant(chunk))
responses = await asyncio.gather(*tasks)
availability = []
for response_list in responses:
for response in response_list:
availability.append(response)
files = {}
if type == "series":
for result in availability:
torrent_files = result["value"]["files"]
for file in torrent_files:
if file["downloadPercent"] != 100:
continue
filename = file["name"]
if not is_video(filename):
continue
if "sample" in filename.lower():
continue
filename_parsed = parse(filename)
if episode not in filename_parsed.episodes:
continue
if kitsu:
if filename_parsed.seasons:
continue
else:
if season not in filename_parsed.seasons:
continue
files[result["value"]["hashString"]] = {
"index": torrent_files.index(file),
"title": filename,
"size": file["size"],
}
break
else:
for result in availability:
value = result["value"]
torrent_files = value["files"]
for file in torrent_files:
if file["downloadPercent"] != 100:
continue
filename = file["name"]
if not is_video(filename):
continue
if "sample" in filename.lower():
continue
files[value["hashString"]] = {
"index": torrent_files.index(file),
"title": filename,
"size": file["size"],
}
return files
async def generate_download_link(self, hash: str, index: str):
try:
add_torrent = await self.session.post(
f"{self.api_url}/seedbox/add", data={"url": hash, "async": True}
)
add_torrent = await add_torrent.json()
return add_torrent["value"]["files"][int(index)]["downloadUrl"]
except Exception as e:
logger.warning(
f"Exception while getting download link from Debrid-Link for {hash}|{index}: {e}"
)
comet/debrid/easydebrid.py
+8
View File
@@ -0,0 +1,8 @@
import aiohttp
class EasyDebrid:
def __init__(
self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
):
pass
comet/debrid/manager.py
+96 -22
View File
@@ -1,35 +1,109 @@
import aiohttp import aiohttp
from comet.utils.config import should_use_stremthru
from .realdebrid import RealDebrid from .realdebrid import RealDebrid
from .alldebrid import AllDebrid from .alldebrid import AllDebrid
from .premiumize import Premiumize from .premiumize import Premiumize
from .torbox import TorBox from .torbox import TorBox
from .debridlink import DebridLink from .debridlink import DebridLink
from .torrent import Torrent
from .stremthru import StremThru from .stremthru import StremThru
from .easydebrid import EasyDebrid
from .offcloud import Offcloud
from .pikpak import PikPak
debrid_services = {
"realdebrid": {
"extension": "RD",
"cache_availability_endpoint": False,
"class": RealDebrid,
},
"alldebrid": {
"extension": "AD",
"cache_availability_endpoint": False,
"class": AllDebrid,
},
"premiumize": {
"extension": "PM",
"cache_availability_endpoint": True,
"class": Premiumize,
},
"torbox": {"extension": "TB", "cache_availability_endpoint": True, "class": TorBox},
"debridlink": {
"extension": "DL",
"cache_availability_endpoint": False,
"class": DebridLink,
},
"stremthru": {
"extension": "ST",
"cache_availability_endpoint": True,
"class": StremThru,
},
"easydebrid": {
"extension": "ED",
"cache_availability_endpoint": True,
"class": EasyDebrid,
},
"offcloud": {
"extension": "OC",
"cache_availability_endpoint": False,
"class": Offcloud,
},
"pikpak": {
"extension": "PP",
"cache_availability_endpoint": False,
"class": PikPak,
},
"torrent": {
"extension": "TORRENT",
"cache_availability_endpoint": False,
"class": Torrent,
},
}
def getDebrid(session: aiohttp.ClientSession, config: dict, ip: str): def get_debrid_extension(debrid_service: str):
debrid_service = config["debridService"] original_extension = debrid_services[debrid_service]["extension"]
debrid_api_key = config["debridApiKey"]
if should_use_stremthru(config): return original_extension
return StremThru(
session=session,
url=config["stremthruUrl"], def build_stremthru_token(debrid_service: str, debrid_api_key: str):
debrid_service=debrid_service, return f"{debrid_service}:{debrid_api_key}"
token=debrid_api_key,
ip=ip,
def get_debrid(
session: aiohttp.ClientSession,
video_id: str,
media_only_id: str,
debrid_service: str,
debrid_api_key: str,
ip: str,
):
if debrid_service != "torrent":
return debrid_services["stremthru"]["class"](
session,
video_id,
media_only_id,
build_stremthru_token(debrid_service, debrid_api_key),
ip,
) )
if debrid_service == "realdebrid":
return RealDebrid(session, debrid_api_key, ip) async def retrieve_debrid_availability(
elif debrid_service == "alldebrid": session: aiohttp.ClientSession,
return AllDebrid(session, debrid_api_key) video_id: str,
elif debrid_service == "premiumize": media_only_id: str,
return Premiumize(session, debrid_api_key) debrid_service: str,
elif debrid_service == "torbox": debrid_api_key: str,
return TorBox(session, debrid_api_key) ip: str,
elif debrid_service == "debridlink": info_hashes: list,
return DebridLink(session, debrid_api_key) seeders_map: dict,
tracker_map: dict,
sources_map: dict,
):
if debrid_service == "torrent":
return []
return await get_debrid(
session, video_id, media_only_id, debrid_service, debrid_api_key, ip
).get_availability(info_hashes, seeders_map, tracker_map, sources_map)
comet/debrid/offcloud.py
+8
View File
@@ -0,0 +1,8 @@
import aiohttp
class Offcloud:
def __init__(
self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
):
pass
comet/debrid/pikpak.py
+8
View File
@@ -0,0 +1,8 @@
import aiohttp
class PikPak:
def __init__(
self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
):
pass
comet/debrid/premiumize.py
+3 -172
View File
@@ -1,177 +1,8 @@
import aiohttp import aiohttp
import asyncio
from RTN import parse
from comet.utils.general import is_video
from comet.utils.logger import logger
class Premiumize: class Premiumize:
def __init__(self, session: aiohttp.ClientSession, debrid_api_key: str): def __init__(
self.session = session self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
self.proxy = None
self.api_url = "https://premiumize.me/api"
self.debrid_api_key = debrid_api_key
async def check_premium(self):
try:
check_premium = await self.session.get(
f"{self.api_url}/account/info?apikey={self.debrid_api_key}"
)
check_premium = await check_premium.text()
if (
'"status":"success"' in check_premium
and '"premium_until":null' not in check_premium
):
return True
except Exception as e:
logger.warning(
f"Exception while checking premium status on Premiumize: {e}"
)
return False
async def get_instant(self, chunk: list):
try:
response = await self.session.get(
f"{self.api_url}/cache/check?apikey={self.debrid_api_key}&items[]={'&items[]='.join(chunk)}"
)
response = await response.json()
response["hashes"] = chunk
return response
except Exception as e:
logger.warning(
f"Exception while checking hash instant availability on Premiumize: {e}"
)
async def get_files(
self, torrent_hashes: list, type: str, season: str, episode: str, kitsu: bool, **kwargs
): ):
chunk_size = 100 pass
chunks = [
torrent_hashes[i : i + chunk_size]
for i in range(0, len(torrent_hashes), chunk_size)
]
tasks = []
for chunk in chunks:
tasks.append(self.get_instant(chunk))
responses = await asyncio.gather(*tasks)
availability = []
for response in responses:
if not response:
continue
availability.append(response)
files = {}
if type == "series":
for result in availability:
if result["status"] != "success":
continue
responses = result["response"]
filenames = result["filename"]
filesizes = result["filesize"]
hashes = result["hashes"]
for index, response in enumerate(responses):
if not response:
continue
if not filesizes[index]:
continue
filename = filenames[index]
if "sample" in filename.lower():
continue
filename_parsed = parse(filename)
if episode not in filename_parsed.episodes:
continue
if kitsu:
if filename_parsed.seasons:
continue
else:
if season not in filename_parsed.seasons:
continue
files[hashes[index]] = {
"index": f"{season}|{episode}",
"title": filename,
"size": int(filesizes[index]),
}
else:
for result in availability:
if result["status"] != "success":
continue
responses = result["response"]
filenames = result["filename"]
filesizes = result["filesize"]
hashes = result["hashes"]
for index, response in enumerate(responses):
if response is False:
continue
if not filesizes[index]:
continue
filename = filenames[index]
if "sample" in filename.lower():
continue
files[hashes[index]] = {
"index": 0,
"title": filename,
"size": int(filesizes[index]),
}
return files
async def generate_download_link(self, hash: str, index: str):
try:
add_magnet = await self.session.post(
f"{self.api_url}/transfer/directdl?apikey={self.debrid_api_key}&src=magnet:?xt=urn:btih:{hash}",
)
add_magnet = await add_magnet.json()
season = None
if "|" in index:
index = index.split("|")
season = int(index[0])
episode = int(index[1])
content = add_magnet["content"]
for file in content:
filename = file["path"]
if "/" in filename:
filename = filename.split("/")[1]
if not is_video(filename):
content.remove(file)
continue
if season is not None:
filename_parsed = parse(filename)
if (
season in filename_parsed.seasons
and episode in filename_parsed.episodes
):
return file["link"]
max_size_item = max(content, key=lambda x: x["size"])
return max_size_item["link"]
except Exception as e:
logger.warning(
f"Exception while getting download link from Premiumize for {hash}|{index}: {e}"
)
comet/debrid/realdebrid.py
+3 -187
View File
@@ -1,192 +1,8 @@
import aiohttp import aiohttp
import asyncio
from RTN import parse
from comet.utils.general import is_video
from comet.utils.logger import logger
from comet.utils.models import settings
class RealDebrid: class RealDebrid:
def __init__(self, session: aiohttp.ClientSession, debrid_api_key: str, ip: str): def __init__(
session.headers["Authorization"] = f"Bearer {debrid_api_key}" self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
self.session = session
self.ip = ip
self.proxy = None
self.api_url = "https://api.real-debrid.com/rest/1.0"
async def check_premium(self):
try:
check_premium = await self.session.get(f"{self.api_url}/user")
check_premium = await check_premium.text()
if '"type": "premium"' in check_premium:
return True
except Exception as e:
logger.warning(
f"Exception while checking premium status on Real-Debrid: {e}"
)
return False
async def get_instant(self, chunk: list):
try:
response = await self.session.get(
f"{self.api_url}/torrents/instantAvailability/{'/'.join(chunk)}"
)
return await response.json()
except Exception as e:
logger.warning(
f"Exception while checking hash instant availability on Real-Debrid: {e}"
)
async def get_files(
self, torrent_hashes: list, type: str, season: str, episode: str, kitsu: bool, **kwargs
): ):
chunk_size = 100 pass
chunks = [
torrent_hashes[i : i + chunk_size]
for i in range(0, len(torrent_hashes), chunk_size)
]
tasks = []
for chunk in chunks:
tasks.append(self.get_instant(chunk))
responses = await asyncio.gather(*tasks)
availability = {}
for response in responses:
if response is not None:
availability.update(response)
files = {}
if type == "series":
for hash, details in availability.items():
if "rd" not in details:
continue
for variants in details["rd"]:
for index, file in variants.items():
filename = file["filename"]
if not is_video(filename):
continue
if "sample" in filename.lower():
continue
filename_parsed = parse(filename)
if episode not in filename_parsed.episodes:
continue
if kitsu:
if filename_parsed.seasons:
continue
else:
if season not in filename_parsed.seasons:
continue
files[hash] = {
"index": index,
"title": filename,
"size": file["filesize"],
}
break
else:
for hash, details in availability.items():
if "rd" not in details:
continue
for variants in details["rd"]:
for index, file in variants.items():
filename = file["filename"]
if not is_video(filename):
continue
if "sample" in filename.lower():
continue
files[hash] = {
"index": index,
"title": filename,
"size": file["filesize"],
}
break
return files
async def generate_download_link(self, hash: str, index: str):
try:
check_blacklisted = await self.session.get("https://real-debrid.com/vpn")
check_blacklisted = await check_blacklisted.text()
if (
"Your ISP or VPN provider IP address is currently blocked on our website"
in check_blacklisted
):
self.proxy = settings.DEBRID_PROXY_URL
if not self.proxy:
logger.warning(
"Real-Debrid blacklisted server's IP. No proxy found."
)
else:
logger.warning(
f"Real-Debrid blacklisted server's IP. Switching to proxy {self.proxy} for {hash}|{index}"
)
add_magnet = await self.session.post(
f"{self.api_url}/torrents/addMagnet",
data={"magnet": f"magnet:?xt=urn:btih:{hash}", "ip": self.ip},
proxy=self.proxy,
)
add_magnet = await add_magnet.json()
get_magnet_info = await self.session.get(
add_magnet["uri"], proxy=self.proxy
)
get_magnet_info = await get_magnet_info.json()
await self.session.post(
f"{self.api_url}/torrents/selectFiles/{add_magnet['id']}",
data={
"files": ",".join(
str(file["id"])
for file in get_magnet_info["files"]
if is_video(file["path"])
),
"ip": self.ip,
},
proxy=self.proxy,
)
get_magnet_info = await self.session.get(
add_magnet["uri"], proxy=self.proxy
)
get_magnet_info = await get_magnet_info.json()
index = int(index)
realIndex = index
for file in get_magnet_info["files"]:
if file["id"] == realIndex:
break
if file["selected"] != 1:
index -= 1
unrestrict_link = await self.session.post(
f"{self.api_url}/unrestrict/link",
data={"link": get_magnet_info["links"][index - 1], "ip": self.ip},
proxy=self.proxy,
)
unrestrict_link = await unrestrict_link.json()
return unrestrict_link["download"]
except Exception as e:
logger.warning(
f"Exception while getting download link from Real-Debrid for {hash}|{index}: {e}"
)
comet/debrid/stremthru.py
+135 -100
View File
@@ -1,59 +1,44 @@
import asyncio
from typing import Optional
import aiohttp import aiohttp
from RTN import parse import asyncio
from RTN import parse, title_match
from comet.utils.models import settings
from comet.utils.general import is_video from comet.utils.general import is_video
from comet.utils.debrid import cache_availability
from comet.utils.logger import logger from comet.utils.logger import logger
from comet.utils.torrent import torrent_update_queue
class StremThru: class StremThru:
def __init__( def __init__(
self, self,
session: aiohttp.ClientSession, session: aiohttp.ClientSession,
url: str, video_id: str,
media_only_id: str,
token: str, token: str,
debrid_service: str,
ip: str, ip: str,
): ):
if not self.is_supported_store(debrid_service): store, token = self.parse_store_creds(token)
raise ValueError(f"unsupported store: {debrid_service}") session.headers["X-StremThru-Store-Name"] = store
session.headers["X-StremThru-Store-Authorization"] = f"Bearer {token}"
store, token = self.parse_store_creds(debrid_service, token)
if store == "stremthru":
session.headers["Proxy-Authorization"] = f"Basic {token}"
else:
session.headers["X-StremThru-Store-Name"] = store
session.headers["X-StremThru-Store-Authorization"] = f"Bearer {token}"
session.headers["User-Agent"] = "comet" session.headers["User-Agent"] = "comet"
self.session = session self.session = session
self.base_url = f"{url}/v0/store" self.base_url = f"{settings.STREMTHRU_URL}/v0/store"
self.name = f"StremThru[{debrid_service}]" if debrid_service else "StremThru" self.name = f"StremThru-{store}"
self.real_debrid_name = store
self.client_ip = ip self.client_ip = ip
self.sid = video_id
self.media_only_id = media_only_id
@staticmethod def parse_store_creds(self, token: str):
def parse_store_creds(debrid_service, token: str = ""):
if debrid_service != "stremthru":
return debrid_service, token
if ":" in token: if ":" in token:
parts = token.split(":") parts = token.split(":", 1)
return parts[0], parts[1] return parts[0], parts[1]
return debrid_service, token
@staticmethod return token, ""
def is_supported_store(name: Optional[str]):
return (
name == "stremthru"
or name == "alldebrid"
or name == "debridlink"
or name == "easydebrid"
or name == "premiumize"
or name == "realdebrid"
or name == "torbox"
)
async def check_premium(self): async def check_premium(self):
try: try:
@@ -69,11 +54,9 @@ class StremThru:
return False return False
async def get_instant(self, magnets: list, sid: Optional[str] = None): async def get_instant(self, magnets: list):
try: try:
url = f"{self.base_url}/magnets/check?magnet={','.join(magnets)}&client_ip={self.client_ip}" url = f"{self.base_url}/magnets/check?magnet={','.join(magnets)}&client_ip={self.client_ip}&sid={self.sid}"
if sid:
url = f"{url}&sid={sid}"
magnet = await self.session.get(url) magnet = await self.session.get(url)
return await magnet.json() return await magnet.json()
except Exception as e: except Exception as e:
@@ -81,17 +64,17 @@ class StremThru:
f"Exception while checking hash instant availability on {self.name}: {e}" f"Exception while checking hash instant availability on {self.name}: {e}"
) )
async def get_files( async def get_availability(
self, self,
torrent_hashes: list, torrent_hashes: list,
type: str, seeders_map: dict,
season: str, tracker_map: dict,
episode: str, sources_map: dict,
kitsu: bool,
video_id: Optional[str] = None,
**kwargs,
): ):
chunk_size = 25 if not await self.check_premium():
return []
chunk_size = 50
chunks = [ chunks = [
torrent_hashes[i : i + chunk_size] torrent_hashes[i : i + chunk_size]
for i in range(0, len(torrent_hashes), chunk_size) for i in range(0, len(torrent_hashes), chunk_size)
@@ -99,7 +82,7 @@ class StremThru:
tasks = [] tasks = []
for chunk in chunks: for chunk in chunks:
tasks.append(self.get_instant(chunk, sid=video_id)) tasks.append(self.get_instant(chunk))
responses = await asyncio.gather(*tasks) responses = await asyncio.gather(*tasks)
@@ -109,62 +92,83 @@ class StremThru:
if response and "data" in response if response and "data" in response
] ]
files = {} is_offcloud = self.real_debrid_name == "offcloud"
if type == "series": files = []
for magnets in availability: cached_count = 0
for magnet in magnets: for result in availability:
if magnet["status"] != "cached": for torrent in result:
continue if torrent["status"] != "cached":
continue
for file in magnet["files"]: cached_count += 1
filename = file["name"] hash = torrent["hash"]
seeders = seeders_map[hash]
tracker = tracker_map[hash]
sources = sources_map[hash]
if not is_video(filename) or "sample" in filename: if is_offcloud:
file_info = {
"info_hash": hash,
"index": None,
"title": None,
"size": None,
"season": None,
"episode": None,
"parsed": None,
}
files.append(file_info)
else:
for file in torrent["files"]:
filename = file["name"].split("/")[-1]
if not is_video(filename) or "sample" in filename.lower():
continue continue
filename_parsed = parse(filename) filename_parsed = parse(filename)
if episode not in filename_parsed.episodes: season = (
filename_parsed.seasons[0]
if filename_parsed.seasons
else None
)
episode = (
filename_parsed.episodes[0]
if filename_parsed.episodes
else None
)
if ":" in self.sid and (season is None or episode is None):
continue continue
if kitsu: index = file["index"] if file["index"] != -1 else None
if filename_parsed.seasons: size = file["size"] if file["size"] != -1 else None
continue
else:
if season not in filename_parsed.seasons:
continue
files[magnet["hash"]] = { file_info = {
"index": file["index"], "info_hash": hash,
"index": index,
"title": filename, "title": filename,
"size": file["size"], "size": size,
"season": season,
"episode": episode,
"parsed": filename_parsed,
"seeders": seeders,
"tracker": tracker,
"sources": sources,
} }
break files.append(file_info)
else: await torrent_update_queue.add_torrent_info(file_info, self.media_only_id)
for magnets in availability:
for magnet in magnets:
if magnet["status"] != "cached":
continue
for file in magnet["files"]:
filename = file["name"]
if not is_video(filename) or "sample" in filename:
continue
files[magnet["hash"]] = {
"index": file["index"],
"title": filename,
"size": file["size"],
}
break
logger.log(
"SCRAPER",
f"{self.name}: Found {cached_count} cached torrents with {len(files)} valid files",
)
return files return files
async def generate_download_link(self, hash: str, index: str): async def generate_download_link(
self, hash: str, index: str, name: str, season: int, episode: int
):
try: try:
magnet = await self.session.post( magnet = await self.session.post(
f"{self.base_url}/magnets?client_ip={self.client_ip}", f"{self.base_url}/magnets?client_ip={self.client_ip}",
@@ -175,26 +179,57 @@ class StremThru:
if magnet["data"]["status"] != "downloaded": if magnet["data"]["status"] != "downloaded":
return return
file = next( name_parsed = parse(name)
( target_file = None
file
for file in magnet["data"]["files"]
if str(file["index"]) == index or file["name"] == index
),
None,
)
if not file: files = []
for file in magnet["data"]["files"]:
filename = file["name"]
filename_parsed = parse(filename)
if not is_video(filename) or not title_match(
name_parsed.parsed_title, filename_parsed.parsed_title
):
continue
file_season = (
filename_parsed.seasons[0] if filename_parsed.seasons else None
)
file_episode = (
filename_parsed.episodes[0] if filename_parsed.episodes else None
)
file_index = file["index"] if file["index"] != -1 else None
file_size = file["size"] if file["size"] != -1 else None
file_info = {
"info_hash": hash,
"index": file_index,
"title": filename,
"size": file_size,
"season": file_season,
"episode": file_episode,
"parsed": filename_parsed,
}
files.append(file_info)
if str(file["index"]) == index:
target_file = file
if season == file_season and episode == file_episode:
target_file = file
if len(files) > 0:
asyncio.create_task(cache_availability(self.real_debrid_name, files))
if not target_file:
return return
link = await self.session.post( link = await self.session.post(
f"{self.base_url}/link/generate?client_ip={self.client_ip}", f"{self.base_url}/link/generate?client_ip={self.client_ip}",
json={"link": file["link"]}, json={"link": target_file["link"]},
) )
link = await link.json() link = await link.json()
return link["data"]["link"] return link["data"]["link"]
except Exception as e: except Exception as e:
logger.warning( logger.warning(f"Exception while getting download link for {hash}: {e}")
f"Exception while getting download link from {self.name} for {hash}|{index}: {e}"
)
comet/debrid/torbox.py
+3 -156
View File
@@ -1,161 +1,8 @@
import aiohttp import aiohttp
import asyncio
from RTN import parse
from comet.utils.general import is_video
from comet.utils.logger import logger
class TorBox: class TorBox:
def __init__(self, session: aiohttp.ClientSession, debrid_api_key: str): def __init__(
session.headers["Authorization"] = f"Bearer {debrid_api_key}" self, session: aiohttp.ClientSession, video_id, debrid_api_key: str, ip: str
self.session = session
self.proxy = None
self.api_url = "https://api.torbox.app/v1/api"
self.debrid_api_key = debrid_api_key
async def check_premium(self):
try:
check_premium = await self.session.get(
f"{self.api_url}/user/me?settings=false"
)
check_premium = await check_premium.text()
if '"success":true' in check_premium:
return True
except Exception as e:
logger.warning(f"Exception while checking premium status on TorBox: {e}")
return False
async def get_instant(self, chunk: list):
try:
response = await self.session.get(
f"{self.api_url}/torrents/checkcached?hash={','.join(chunk)}&format=list&list_files=true"
)
return await response.json()
except Exception as e:
logger.warning(
f"Exception while checking hash instant availability on TorBox: {e}"
)
async def get_files(
self, torrent_hashes: list, type: str, season: str, episode: str, kitsu: bool, **kwargs
): ):
chunk_size = 50 pass
chunks = [
torrent_hashes[i : i + chunk_size]
for i in range(0, len(torrent_hashes), chunk_size)
]
tasks = []
for chunk in chunks:
tasks.append(self.get_instant(chunk))
responses = await asyncio.gather(*tasks)
availability = [response for response in responses if response is not None]
files = {}
if type == "series":
for result in availability:
if not result["success"] or not result["data"]:
continue
for torrent in result["data"]:
torrent_files = torrent["files"]
for file in torrent_files:
filename = file["name"].split("/")[1]
if not is_video(filename):
continue
if "sample" in filename.lower():
continue
filename_parsed = parse(filename)
if episode not in filename_parsed.episodes:
continue
if kitsu:
if filename_parsed.seasons:
continue
else:
if season not in filename_parsed.seasons:
continue
files[torrent["hash"]] = {
"index": torrent_files.index(file),
"title": filename,
"size": file["size"],
}
break
else:
for result in availability:
if not result["success"] or not result["data"]:
continue
for torrent in result["data"]:
torrent_files = torrent["files"]
for file in torrent_files:
filename = file["name"].split("/")[1]
if not is_video(filename):
continue
if "sample" in filename.lower():
continue
files[torrent["hash"]] = {
"index": torrent_files.index(file),
"title": filename,
"size": file["size"],
}
break
return files
async def generate_download_link(self, hash: str, index: str):
try:
get_torrents = await self.session.get(
f"{self.api_url}/torrents/mylist?bypass_cache=true"
)
get_torrents = await get_torrents.json()
exists = False
for torrent in get_torrents["data"]:
if torrent["hash"] == hash:
torrent_id = torrent["id"]
exists = True
break
if not exists:
create_torrent = await self.session.post(
f"{self.api_url}/torrents/createtorrent",
data={"magnet": f"magnet:?xt=urn:btih:{hash}"},
)
create_torrent = await create_torrent.json()
torrent_id = create_torrent["data"]["torrent_id"]
# get_torrents = await self.session.get(
# f"{self.api_url}/torrents/mylist?bypass_cache=true"
# )
# get_torrents = await get_torrents.json()
# for torrent in get_torrents["data"]:
# if torrent["id"] == torrent_id:
# file_id = torrent["files"][int(index)]["id"]
# Useless, we already have file index
get_download_link = await self.session.get(
f"{self.api_url}/torrents/requestdl?token={self.debrid_api_key}&torrent_id={torrent_id}&file_id={index}&zip=false",
)
get_download_link = await get_download_link.json()
return get_download_link["data"]
except Exception as e:
logger.warning(
f"Exception while getting download link from TorBox for {hash}|{index}: {e}"
)
comet/debrid/torrent.py
+3
View File
@@ -0,0 +1,3 @@
class Torrent:
def __init__(self):
pass
comet/main.py
+236 -177
View File
@@ -1,177 +1,236 @@
import contextlib import contextlib
import signal import signal
import sys import sys
import threading import threading
import time import time
import traceback import traceback
from contextlib import asynccontextmanager import uvicorn
import os
import uvicorn
from fastapi import FastAPI from contextlib import asynccontextmanager
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles from fastapi import FastAPI
from starlette.middleware.base import BaseHTTPMiddleware from fastapi.middleware.cors import CORSMiddleware
from starlette.requests import Request from fastapi.staticfiles import StaticFiles
from starlette.middleware.base import BaseHTTPMiddleware
from comet.api.core import main from starlette.requests import Request
from comet.api.stream import streams
from comet.utils.db import setup_database, teardown_database from comet.api.core import main
from comet.utils.logger import logger from comet.api.stream import streams
from comet.utils.models import settings from comet.utils.database import setup_database, teardown_database
from comet.utils.trackers import download_best_trackers
from comet.utils.logger import logger
class LoguruMiddleware(BaseHTTPMiddleware): from comet.utils.models import settings
async def dispatch(self, request: Request, call_next):
start_time = time.time()
try: class LoguruMiddleware(BaseHTTPMiddleware):
response = await call_next(request) async def dispatch(self, request: Request, call_next):
except Exception as e: start_time = time.time()
logger.exception(f"Exception during request processing: {e}") try:
raise response = await call_next(request)
finally: except Exception as e:
process_time = time.time() - start_time logger.exception(f"Exception during request processing: {e}")
logger.log( raise
"API", finally:
f"{request.method} {request.url.path} - {response.status_code if 'response' in locals() else '500'} - {process_time:.2f}s", process_time = time.time() - start_time
) logger.log(
return response "API",
f"{request.method} {request.url.path} - {response.status_code if 'response' in locals() else '500'} - {process_time:.2f}s",
)
@asynccontextmanager return response
async def lifespan(app: FastAPI):
await setup_database()
yield @asynccontextmanager
await teardown_database() async def lifespan(app: FastAPI):
await setup_database()
await download_best_trackers()
app = FastAPI( yield
title="Comet", await teardown_database()
summary="Stremio's fastest torrent/debrid search add-on.",
version="1.0.0",
lifespan=lifespan, app = FastAPI(
redoc_url=None, title="Comet",
) summary="Stremio's fastest torrent/debrid search add-on.",
lifespan=lifespan,
redoc_url=None,
app.add_middleware(LoguruMiddleware) )
app.add_middleware(
CORSMiddleware,
allow_origins=["*"], app.add_middleware(LoguruMiddleware)
allow_credentials=True, app.add_middleware(
allow_methods=["*"], CORSMiddleware,
allow_headers=["*"], allow_origins=["*"],
) allow_credentials=True,
allow_methods=["*"],
app.mount("/static", StaticFiles(directory="comet/templates"), name="static") allow_headers=["*"],
)
app.include_router(main)
app.include_router(streams) app.mount("/static", StaticFiles(directory="comet/templates"), name="static")
app.include_router(main)
class Server(uvicorn.Server): app.include_router(streams)
def install_signal_handlers(self):
pass
class Server(uvicorn.Server):
@contextlib.contextmanager def install_signal_handlers(self):
def run_in_thread(self): pass
thread = threading.Thread(target=self.run, name="Comet")
thread.start() @contextlib.contextmanager
try: def run_in_thread(self):
while not self.started: thread = threading.Thread(target=self.run, name="Comet")
time.sleep(1e-3) thread.start()
yield try:
except Exception as e: while not self.started:
logger.error(f"Error in server thread: {e}") time.sleep(1e-3)
logger.exception(traceback.format_exc()) yield
raise e except Exception as e:
finally: logger.error(f"Error in server thread: {e}")
self.should_exit = True logger.exception(traceback.format_exc())
sys.exit(0) raise e
finally:
self.should_exit = True
def signal_handler(sig, frame): sys.exit(0)
# This will handle kubernetes/docker shutdowns better
# Toss anything that needs to be gracefully shutdown here
logger.log("COMET", "Exiting Gracefully.") def signal_handler(sig, frame):
sys.exit(0) # This will handle kubernetes/docker shutdowns better
# Toss anything that needs to be gracefully shutdown here
logger.log("COMET", "Exiting Gracefully.")
signal.signal(signal.SIGINT, signal_handler) sys.exit(0)
signal.signal(signal.SIGTERM, signal_handler)
config = uvicorn.Config( signal.signal(signal.SIGINT, signal_handler)
app, signal.signal(signal.SIGTERM, signal_handler)
host=settings.FASTAPI_HOST,
port=settings.FASTAPI_PORT,
proxy_headers=True, def start_log():
forwarded_allow_ips="*", logger.log(
workers=settings.FASTAPI_WORKERS, "COMET",
log_config=None, f"Server started on http://{settings.FASTAPI_HOST}:{settings.FASTAPI_PORT} - {settings.FASTAPI_WORKERS} workers",
) )
server = Server(config=config) logger.log(
"COMET",
f"Dashboard Admin Password: {settings.DASHBOARD_ADMIN_PASSWORD} - http://{settings.FASTAPI_HOST}:{settings.FASTAPI_PORT}/dashboard",
def start_log(): )
logger.log( logger.log(
"COMET", "COMET",
f"Server started on http://{settings.FASTAPI_HOST}:{settings.FASTAPI_PORT} - {settings.FASTAPI_WORKERS} workers", f"Database ({settings.DATABASE_TYPE}): {settings.DATABASE_PATH if settings.DATABASE_TYPE == 'sqlite' else settings.DATABASE_URL} - TTL: metadata={settings.METADATA_CACHE_TTL}s, torrents={settings.TORRENT_CACHE_TTL}s, debrid={settings.DEBRID_CACHE_TTL}s",
) )
logger.log( logger.log("COMET", f"Debrid Proxy: {settings.DEBRID_PROXY_URL}")
"COMET",
f"Dashboard Admin Password: {settings.DASHBOARD_ADMIN_PASSWORD} - http://{settings.FASTAPI_HOST}:{settings.FASTAPI_PORT}/active-connections?password={settings.DASHBOARD_ADMIN_PASSWORD}", if settings.INDEXER_MANAGER_TYPE:
) logger.log(
logger.log( "COMET",
"COMET", f"Indexer Manager: {settings.INDEXER_MANAGER_TYPE}|{settings.INDEXER_MANAGER_URL} - Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s",
f"Database ({settings.DATABASE_TYPE}): {settings.DATABASE_PATH if settings.DATABASE_TYPE == 'sqlite' else settings.DATABASE_URL} - TTL: {settings.CACHE_TTL}s", )
) logger.log("COMET", f"Indexers: {', '.join(settings.INDEXER_MANAGER_INDEXERS)}")
logger.log("COMET", f"Debrid Proxy: {settings.DEBRID_PROXY_URL}") logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s")
logger.log(
if settings.INDEXER_MANAGER_TYPE: "COMET", f"Download Torrent Files: {bool(settings.DOWNLOAD_TORRENT_FILES)}"
logger.log( )
"COMET", else:
f"Indexer Manager: {settings.INDEXER_MANAGER_TYPE}|{settings.INDEXER_MANAGER_URL} - Timeout: {settings.INDEXER_MANAGER_TIMEOUT}s", logger.log("COMET", "Indexer Manager: False")
)
logger.log("COMET", f"Indexers: {', '.join(settings.INDEXER_MANAGER_INDEXERS)}") zilean_url = f" - {settings.ZILEAN_URL}"
logger.log("COMET", f"Get Torrent Timeout: {settings.GET_TORRENT_TIMEOUT}s") logger.log(
else: "COMET",
logger.log("COMET", "Indexer Manager: False") f"Zilean Scraper: {bool(settings.SCRAPE_ZILEAN)}{zilean_url if settings.SCRAPE_ZILEAN else ''}",
)
if settings.ZILEAN_URL:
logger.log( torrentio_url = f" - {settings.TORRENTIO_URL}"
"COMET", logger.log(
f"Zilean: {settings.ZILEAN_URL} - Take first: {settings.ZILEAN_TAKE_FIRST}", "COMET",
) f"Torrentio Scraper: {bool(settings.SCRAPE_TORRENTIO)}{torrentio_url if settings.SCRAPE_TORRENTIO else ''}",
else: )
logger.log("COMET", "Zilean: False")
mediafusion_url = f" - {settings.MEDIAFUSION_URL}"
logger.log("COMET", f"Torrentio Scraper: {bool(settings.SCRAPE_TORRENTIO)}") logger.log(
"COMET",
mediafusion_url = f" - {settings.MEDIAFUSION_URL}" f"MediaFusion Scraper: {bool(settings.SCRAPE_MEDIAFUSION)}{mediafusion_url if settings.SCRAPE_MEDIAFUSION else ''}",
logger.log( )
"COMET",
f"MediaFusion Scraper: {bool(settings.SCRAPE_MEDIAFUSION)}{mediafusion_url if settings.SCRAPE_MEDIAFUSION else ''}", logger.log(
) "COMET",
f"Debrid Stream Proxy: {bool(settings.PROXY_DEBRID_STREAM)} - Password: {settings.PROXY_DEBRID_STREAM_PASSWORD} - Max Connections: {settings.PROXY_DEBRID_STREAM_MAX_CONNECTIONS} - Default Debrid Service: {settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE} - Default Debrid API Key: {settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY}",
logger.log( )
"COMET",
f"Debrid Stream Proxy: {bool(settings.PROXY_DEBRID_STREAM)} - Password: {settings.PROXY_DEBRID_STREAM_PASSWORD} - Max Connections: {settings.PROXY_DEBRID_STREAM_MAX_CONNECTIONS} - Default Debrid Service: {settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE} - Default Debrid API Key: {settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY}", logger.log("COMET", f"StremThru URL: {settings.STREMTHRU_URL}")
)
if settings.STREMTHRU_DEFAULT_URL: logger.log("COMET", f"Remove Adult Content: {bool(settings.REMOVE_ADULT_CONTENT)}")
logger.log("COMET", f"Default StremThru URL: {settings.STREMTHRU_DEFAULT_URL}") logger.log("COMET", f"Custom Header HTML: {bool(settings.CUSTOM_HEADER_HTML)}")
logger.log("COMET", f"Title Match Check: {bool(settings.TITLE_MATCH_CHECK)}")
logger.log("COMET", f"Remove Adult Content: {bool(settings.REMOVE_ADULT_CONTENT)}")
logger.log("COMET", f"Custom Header HTML: {bool(settings.CUSTOM_HEADER_HTML)}") def run_with_uvicorn():
"""Run the server with uvicorn only"""
config = uvicorn.Config(
with server.run_in_thread(): app,
start_log() host=settings.FASTAPI_HOST,
try: port=settings.FASTAPI_PORT,
while True: proxy_headers=True,
time.sleep(1) # Keep the main thread alive forwarded_allow_ips="*",
except KeyboardInterrupt: workers=settings.FASTAPI_WORKERS,
logger.log("COMET", "Server stopped by user") log_config=None,
except Exception as e: )
logger.error(f"Unexpected error: {e}") server = Server(config=config)
logger.exception(traceback.format_exc())
finally: with server.run_in_thread():
logger.log("COMET", "Server Shutdown") start_log()
try:
while True:
time.sleep(1) # Keep the main thread alive
except KeyboardInterrupt:
logger.log("COMET", "Server stopped by user")
except Exception as e:
logger.error(f"Unexpected error: {e}")
logger.exception(traceback.format_exc())
finally:
logger.log("COMET", "Server Shutdown")
def run_with_gunicorn():
"""Run the server with gunicorn and uvicorn workers"""
import gunicorn.app.base
class StandaloneApplication(gunicorn.app.base.BaseApplication):
def __init__(self, app, options=None):
self.options = options or {}
self.application = app
super().__init__()
def load_config(self):
config = {
key: value for key, value in self.options.items()
if key in self.cfg.settings and value is not None
}
for key, value in config.items():
self.cfg.set(key.lower(), value)
def load(self):
return self.application
workers = settings.FASTAPI_WORKERS
if workers <= 1:
workers = (os.cpu_count() or 1) * 2 + 1
options = {
"bind": f"{settings.FASTAPI_HOST}:{settings.FASTAPI_PORT}",
"workers": workers,
"worker_class": "uvicorn.workers.UvicornWorker",
"timeout": 120,
"keepalive": 5,
"preload_app": True,
"proxy_protocol": True,
"forwarded_allow_ips": "*",
}
start_log()
logger.log("COMET", f"Starting with gunicorn using {workers} workers")
StandaloneApplication(app, options).run()
if __name__ == "__main__":
if os.name == "nt" or not settings.USE_GUNICORN:
run_with_uvicorn()
else:
run_with_gunicorn()
comet/metadata/__init__.py
View File
comet/metadata/imdb.py
+20
View File
@@ -0,0 +1,20 @@
import aiohttp
from comet.utils.logger import logger
async def get_imdb_metadata(session: aiohttp.ClientSession, id: str):
try:
response = await session.get(
f"https://v3.sg.media-imdb.com/suggestion/a/{id}.json"
)
metadata = await response.json()
for element in metadata["d"]:
if "/" not in element["id"]:
title = element["l"]
year = element.get("y")
year_end = int(element["yr"].split("-")[1]) if "yr" in element else None
return title, year, year_end
except Exception as e:
logger.warning(f"Exception while getting IMDB metadata for {id}: {e}")
return None, None, None, None, None
comet/metadata/kitsu.py
+44
View File
@@ -0,0 +1,44 @@
import aiohttp
from comet.utils.logger import logger
async def get_kitsu_metadata(session: aiohttp.ClientSession, id: str):
try:
response = await session.get(f"https://kitsu.io/api/edge/anime/{id}")
metadata = await response.json()
attributes = metadata["data"]["attributes"]
year = int(attributes["createdAt"].split("-")[0])
year_end = int(attributes["updatedAt"].split("-")[0])
return attributes["canonicalTitle"], year, year_end
except Exception as e:
logger.warning(f"Exception while getting Kitsu metadata for {id}: {e}")
return None, None, None
async def get_kitsu_aliases(session: aiohttp.ClientSession, id: str):
aliases = {}
try:
response = await session.get(f"https://find-my-anime.dtimur.de/api?id={id}&provider=Kitsu")
data = await response.json()
aliases["ez"] = []
aliases["ez"].append(data[0]["title"])
for synonym in data[0]["synonyms"]:
aliases["ez"].append(synonym)
total_aliases = len(aliases["ez"])
if total_aliases > 0:
logger.log(
"SCRAPER",
f"📜 Found {total_aliases} Kitsu aliases for {id}",
)
return aliases
except Exception:
pass
logger.log("SCRAPER", f"📜 No Kitsu aliases found for {id}")
return {}
comet/metadata/manager.py
+107
View File
@@ -0,0 +1,107 @@
import aiohttp
import asyncio
import time
import orjson
from RTN.patterns import normalize_title
from comet.utils.models import database, settings
from comet.utils.general import parse_media_id
from .kitsu import get_kitsu_metadata, get_kitsu_aliases
from .imdb import get_imdb_metadata
from .trakt import get_trakt_aliases
class MetadataScraper:
def __init__(self, session: aiohttp.ClientSession):
self.session = session
async def fetch_metadata_and_aliases(self, media_type: str, media_id: str):
id, season, episode = parse_media_id(media_type, media_id)
get_cached = await self.get_cached(
id, season if not "kitsu" in media_id else 1, episode
)
if get_cached is not None:
return get_cached[0], get_cached[1]
is_kitsu = "kitsu" in media_id
metadata_task = asyncio.create_task(self.get_metadata(id, season, episode, is_kitsu))
aliases_task = asyncio.create_task(self.get_aliases(media_type, id, is_kitsu))
metadata, aliases = await asyncio.gather(metadata_task, aliases_task)
await self.cache_metadata(id, metadata, aliases)
return metadata, aliases
async def get_cached(self, media_id: str, season: int, episode: int):
row = await database.fetch_one(
"""
SELECT title, year, year_end, aliases
FROM metadata_cache
WHERE media_id = :media_id
AND timestamp + :cache_ttl >= :current_time
""",
{
"media_id": media_id,
"cache_ttl": settings.METADATA_CACHE_TTL,
"current_time": time.time(),
},
)
if row is not None:
metadata = {
"title": row["title"],
"year": row["year"],
"year_end": row["year_end"],
"season": season,
"episode": episode,
}
return metadata, orjson.loads(row["aliases"])
return None
async def cache_metadata(self, media_id: str, metadata: dict, aliases: dict):
await database.execute(
f"""
INSERT {"OR IGNORE " if settings.DATABASE_TYPE == "sqlite" else ""}
INTO metadata_cache
VALUES (:media_id, :title, :year, :year_end, :aliases, :timestamp)
{" ON CONFLICT DO NOTHING" if settings.DATABASE_TYPE == "postgresql" else ""}
""",
{
"media_id": media_id,
"title": metadata["title"],
"year": metadata["year"],
"year_end": metadata["year_end"],
"aliases": orjson.dumps(aliases).decode("utf-8"),
"timestamp": time.time(),
},
)
def normalize_metadata(self, metadata: dict, season: int, episode: int):
title, year, year_end = metadata
if title is None: # metadata retrieving failed
return None
return {
"title": normalize_title(title),
"year": year,
"year_end": year_end,
"season": season,
"episode": episode,
}
async def get_metadata(self, id: str, season: int, episode: int, is_kitsu: bool):
if is_kitsu:
raw_metadata = await get_kitsu_metadata(self.session, id)
return self.normalize_metadata(raw_metadata, 1, episode)
else:
raw_metadata = await get_imdb_metadata(self.session, id)
return self.normalize_metadata(raw_metadata, season, episode)
async def get_aliases(self, media_type: str, media_id: str, is_kitsu: bool):
if is_kitsu:
return await get_kitsu_aliases(self.session, media_id)
return await get_trakt_aliases(self.session, media_type, media_id)
comet/metadata/trakt.py
+31
View File
@@ -0,0 +1,31 @@
import aiohttp
from comet.utils.logger import logger
async def get_trakt_aliases(
session: aiohttp.ClientSession, media_type: str, media_id: str
):
aliases = set()
try:
response = await session.get(
f"https://api.trakt.tv/{'movies' if media_type == 'movie' else 'shows'}/{media_id}/aliases"
)
data = await response.json()
for aliase in data:
aliases.add(aliase["title"])
total_aliases = len(aliases)
if total_aliases > 0:
logger.log(
"SCRAPER",
f"📜 Found {total_aliases} Trakt aliases for {media_id}",
)
return {"ez": list(aliases)}
except Exception:
pass
logger.log("SCRAPER", f"📜 No Trakt aliases found for {media_id}")
return {}
comet/scrapers/__init__.py
View File
comet/scrapers/jackett.py
+128
View File
@@ -0,0 +1,128 @@
import aiohttp
import asyncio
from comet.utils.models import settings
from comet.utils.logger import logger
from comet.utils.torrent import (
download_torrent,
extract_torrent_metadata,
extract_trackers_from_magnet,
add_torrent_queue,
)
async def process_torrent(
session: aiohttp.ClientSession, result: dict, media_id: str, season: int
):
base_torrent = {
"title": result["Title"],
"infoHash": None,
"fileIndex": None,
"seeders": result["Seeders"],
"size": result["Size"],
"tracker": result["Tracker"],
"sources": [],
}
torrents = []
if result["Link"] is not None:
content, magnet_hash, magnet_url = await download_torrent(
session, result["Link"]
)
if content:
metadata = extract_torrent_metadata(content)
if metadata:
for file in metadata["files"]:
torrent = base_torrent.copy()
torrent["title"] = file["name"]
torrent["infoHash"] = metadata["info_hash"].lower()
torrent["fileIndex"] = file["index"]
torrent["size"] = file["size"]
torrent["sources"] = metadata["announce_list"]
torrents.append(torrent)
return torrents
if magnet_hash:
base_torrent["infoHash"] = magnet_hash.lower()
base_torrent["sources"] = extract_trackers_from_magnet(magnet_url)
await add_torrent_queue.add_torrent(
magnet_url,
base_torrent["seeders"],
base_torrent["tracker"],
media_id,
season,
)
torrents.append(base_torrent)
return torrents
if "InfoHash" in result and result["InfoHash"]:
base_torrent["infoHash"] = result["InfoHash"].lower()
if result["MagnetUri"] is not None:
base_torrent["sources"] = extract_trackers_from_magnet(result["MagnetUri"])
await add_torrent_queue.add_torrent(
result["MagnetUri"],
base_torrent["seeders"],
base_torrent["tracker"],
media_id,
season,
)
torrents.append(base_torrent)
return torrents
async def fetch_jackett_results(
session: aiohttp.ClientSession, indexer: str, query: str
):
try:
response = await session.get(
f"{settings.INDEXER_MANAGER_URL}/api/v2.0/indexers/all/results?apikey={settings.INDEXER_MANAGER_API_KEY}&Query={query}&Tracker[]={indexer}",
timeout=aiohttp.ClientTimeout(total=settings.INDEXER_MANAGER_TIMEOUT),
)
response = await response.json()
return response.get("Results", [])
except Exception as e:
logger.warning(
f"Exception while fetching Jackett results for indexer {indexer}: {e}"
)
return []
async def get_jackett(manager, session: aiohttp.ClientSession, title: str, seen: set):
torrents = []
try:
tasks = [
fetch_jackett_results(session, indexer, title)
for indexer in settings.INDEXER_MANAGER_INDEXERS
]
all_results = await asyncio.gather(*tasks)
torrent_tasks = []
for result_set in all_results:
for result in result_set:
if result["Details"] in seen:
continue
seen.add(result["Details"])
torrent_tasks.append(
process_torrent(
session, result, manager.media_only_id, manager.season
)
)
processed_torrents = await asyncio.gather(*torrent_tasks)
torrents = [
t for sublist in processed_torrents for t in sublist if t["infoHash"]
]
except Exception as e:
logger.warning(
f"Exception while getting torrents for {title} with Jackett: {e}"
)
await manager.filter_manager(torrents)
comet/scrapers/manager.py
+349
View File
@@ -0,0 +1,349 @@
import aiohttp
import asyncio
import orjson
import time
from RTN import (
parse,
title_match,
get_rank,
check_fetch,
sort_torrents,
ParsedData,
BestRanking,
Torrent,
)
from comet.utils.models import settings, database, CometSettingsModel
from comet.utils.general import default_dump
from comet.utils.debrid import cache_availability, get_cached_availability
from comet.debrid.manager import retrieve_debrid_availability
from .zilean import get_zilean
from .torrentio import get_torrentio
from .mediafusion import get_mediafusion
from .jackett import get_jackett
from .prowlarr import get_prowlarr
class TorrentManager:
def __init__(
self,
debrid_service: str,
debrid_api_key: str,
ip: str,
media_type: str,
media_full_id: str,
media_only_id: str,
title: str,
year: int,
year_end: int,
season: int,
episode: int,
aliases: dict,
remove_adult_content: bool,
):
self.debrid_service = debrid_service
self.debrid_api_key = debrid_api_key
self.ip = ip
self.media_type = media_type
self.media_id = media_full_id
self.media_only_id = media_only_id
self.title = title
self.year = year
self.year_end = year_end
self.season = season
self.episode = episode
self.aliases = aliases
self.remove_adult_content = remove_adult_content
self.seen_hashes = set()
self.torrents = {}
self.ready_to_cache = []
self.ranked_torrents = {}
async def scrape_torrents(
self,
session: aiohttp.ClientSession,
):
tasks = []
if settings.SCRAPE_TORRENTIO:
tasks.append(get_torrentio(self, self.media_type, self.media_id))
if settings.SCRAPE_MEDIAFUSION:
tasks.append(get_mediafusion(self, self.media_type, self.media_id))
if settings.SCRAPE_ZILEAN:
tasks.append(
get_zilean(self, session, self.title, self.season, self.episode)
)
if settings.INDEXER_MANAGER_API_KEY:
queries = [self.title]
if self.media_type == "series":
queries.append(f"{self.title} S{self.season:02d}")
queries.append(f"{self.title} S{self.season:02d}E{self.episode:02d}")
seen_already = set()
for query in queries:
if settings.INDEXER_MANAGER_TYPE == "jackett":
tasks.append(get_jackett(self, session, query, seen_already))
elif settings.INDEXER_MANAGER_TYPE == "prowlarr":
tasks.append(get_prowlarr(self, session, query, seen_already))
await asyncio.gather(*tasks)
asyncio.create_task(self.cache_torrents())
for torrent in self.ready_to_cache:
season = torrent["parsed"].seasons[0] if torrent["parsed"].seasons else None
episode = (
torrent["parsed"].episodes[0] if torrent["parsed"].episodes else None
)
if (season is not None and season != self.season) or (
episode is not None and episode != self.episode
):
continue
info_hash = torrent["infoHash"]
self.torrents[info_hash] = {
"fileIndex": torrent["fileIndex"],
"title": torrent["title"],
"seeders": torrent["seeders"],
"size": torrent["size"],
"tracker": torrent["tracker"],
"sources": torrent["sources"],
"parsed": torrent["parsed"],
}
async def get_cached_torrents(self):
rows = await database.fetch_all(
"""
SELECT info_hash, file_index, title, seeders, size, tracker, sources, parsed
FROM torrents
WHERE media_id = :media_id
AND ((season IS NOT NULL AND season = cast(:season as INTEGER)) OR (season IS NULL AND cast(:season as INTEGER) IS NULL))
AND (episode IS NULL OR episode = cast(:episode as INTEGER))
AND timestamp + :cache_ttl >= :current_time
""",
{
"media_id": self.media_only_id,
"season": self.season,
"episode": self.episode,
"cache_ttl": settings.TORRENT_CACHE_TTL,
"current_time": time.time(),
},
)
for row in rows:
info_hash = row["info_hash"]
self.torrents[info_hash] = {
"fileIndex": row["file_index"],
"title": row["title"],
"seeders": row["seeders"],
"size": row["size"],
"tracker": row["tracker"],
"sources": orjson.loads(row["sources"]),
"parsed": ParsedData(**orjson.loads(row["parsed"])),
}
async def cache_torrents(self):
current_time = time.time()
values = [
{
"media_id": self.media_only_id,
"info_hash": torrent["infoHash"],
"file_index": torrent["fileIndex"],
"season": torrent["parsed"].seasons[0]
if torrent["parsed"].seasons
else self.season,
"episode": torrent["parsed"].episodes[0]
if torrent["parsed"].episodes
else None,
"title": torrent["title"],
"seeders": torrent["seeders"],
"size": torrent["size"],
"tracker": torrent["tracker"],
"sources": orjson.dumps(torrent["sources"]).decode("utf-8"),
"parsed": orjson.dumps(torrent["parsed"], default_dump).decode("utf-8"),
"timestamp": current_time,
}
for torrent in self.ready_to_cache
]
query = f"""
INSERT {"OR IGNORE " if settings.DATABASE_TYPE == "sqlite" else ""}
INTO torrents
VALUES (:media_id, :info_hash, :file_index, :season, :episode, :title, :seeders, :size, :tracker, :sources, :parsed, :timestamp)
{" ON CONFLICT DO NOTHING" if settings.DATABASE_TYPE == "postgresql" else ""}
"""
await database.execute_many(query, values)
async def filter(self, torrents: list):
title = self.title
year = self.year
year_end = self.year_end
aliases = self.aliases
remove_adult_content = self.remove_adult_content
for torrent in torrents:
parsed = parse(torrent["title"])
if remove_adult_content and parsed.adult:
continue
if parsed.parsed_title and not title_match(
title, parsed.parsed_title, aliases=aliases
):
continue
if year and parsed.year:
if year_end is not None:
if not (year <= parsed.year <= year_end):
continue
else:
if year < (parsed.year - 1) or year > (parsed.year + 1):
continue
torrent["parsed"] = parsed
self.ready_to_cache.append(torrent)
async def filter_manager(self, torrents: list):
new_torrents = [
torrent
for torrent in torrents
if (torrent["infoHash"], torrent["title"]) not in self.seen_hashes
]
self.seen_hashes.update(
(torrent["infoHash"], torrent["title"]) for torrent in new_torrents
)
chunk_size = 50
tasks = [
self.filter(new_torrents[i : i + chunk_size])
for i in range(0, len(new_torrents), chunk_size)
]
await asyncio.gather(*tasks)
def rank_torrents(
self,
rtn_settings: CometSettingsModel,
rtn_ranking: BestRanking,
max_results_per_resolution: int,
max_size: int,
cached_only: int,
remove_trash: int,
):
ranked_torrents = set()
for info_hash, torrent in self.torrents.items():
if (
cached_only
and self.debrid_service != "torrent"
and not torrent["cached"]
):
continue
if max_size != 0 and torrent["size"] > max_size:
continue
parsed = torrent["parsed"]
raw_title = torrent["title"]
is_fetchable, failed_keys = check_fetch(parsed, rtn_settings)
rank = get_rank(parsed, rtn_settings, rtn_ranking)
if remove_trash:
if (
not is_fetchable
or rank < rtn_settings.options["remove_ranks_under"]
):
continue
try:
ranked_torrents.add(
Torrent(
infohash=info_hash,
raw_title=raw_title,
data=parsed,
fetch=is_fetchable,
rank=rank,
lev_ratio=0.0,
)
)
except Exception:
pass
self.ranked_torrents = sort_torrents(
ranked_torrents, max_results_per_resolution
)
async def get_and_cache_debrid_availability(self, session: aiohttp.ClientSession):
info_hashes = list(self.torrents.keys())
seeders_map = {hash: self.torrents[hash]["seeders"] for hash in info_hashes}
tracker_map = {hash: self.torrents[hash]["tracker"] for hash in info_hashes}
sources_map = {hash: self.torrents[hash]["sources"] for hash in info_hashes}
availability = await retrieve_debrid_availability(
session,
self.media_id,
self.media_only_id,
self.debrid_service,
self.debrid_api_key,
self.ip,
info_hashes,
seeders_map,
tracker_map,
sources_map,
)
if len(availability) == 0:
return
for file in availability:
season = file["season"]
episode = file["episode"]
if (season is not None and season != self.season) or (
episode is not None and episode != self.episode
):
continue
info_hash = file["info_hash"]
self.torrents[info_hash]["cached"] = True
if file["parsed"] is not None:
self.torrents[info_hash]["parsed"] = file["parsed"]
if file["index"] is not None:
self.torrents[info_hash]["fileIndex"] = file["index"]
if file["title"] is not None:
self.torrents[info_hash]["title"] = file["title"]
if file["size"] is not None:
self.torrents[info_hash]["size"] = file["size"]
asyncio.create_task(cache_availability(self.debrid_service, availability))
async def get_cached_availability(self):
info_hashes = list(self.torrents.keys())
for hash in info_hashes:
self.torrents[hash]["cached"] = False
if self.debrid_service == "torrent" or len(self.torrents) == 0:
return
rows = await get_cached_availability(
self.debrid_service, info_hashes, self.season, self.episode
)
for row in rows:
info_hash = row["info_hash"]
self.torrents[info_hash]["cached"] = True
if row["parsed"] is not None:
self.torrents[info_hash]["parsed"] = ParsedData(
**orjson.loads(row["parsed"])
)
if row["file_index"] is not None:
self.torrents[info_hash]["fileIndex"] = row["file_index"]
if row["title"] is not None:
self.torrents[info_hash]["title"] = row["title"]
if row["size"] is not None:
self.torrents[info_hash]["size"] = row["size"]
comet/scrapers/mediafusion.py
+58
View File
@@ -0,0 +1,58 @@
from curl_cffi import requests
from comet.utils.models import settings
from comet.utils.logger import logger
async def get_mediafusion(manager, media_type: str, media_id: str):
torrents = []
try:
try:
get_mediafusion = requests.get(
f"{settings.MEDIAFUSION_URL}/D-zn4qJLK4wUZVWscY9ESCnoZBEiNJCZ9uwfCvmxuliDjY7vkc-fu0OdxUPxwsP3_A/stream/{media_type}/{media_id}.json"
).json()
except Exception as e:
logger.warning(
f"Failed to get MediaFusion results without proxy for {media_id}: {e}"
)
get_mediafusion = requests.get(
f"{settings.MEDIAFUSION_URL}/stream/{media_type}/{media_id}.json",
proxies={
"http": settings.DEBRID_PROXY_URL,
"https": settings.DEBRID_PROXY_URL,
},
).json()
for torrent in get_mediafusion["streams"]:
title_full = torrent["description"]
lines = title_full.split("\n")
title = lines[0].replace("📂 ", "").replace("/", "")
seeders = None
if "👤" in lines[1]:
seeders = int(lines[1].split("👤 ")[1].split("\n")[0])
tracker = lines[-1].split("🔗 ")[1]
torrents.append(
{
"title": title,
"infoHash": torrent["infoHash"].lower(),
"fileIndex": torrent["fileIdx"] if "fileIdx" in torrent else None,
"seeders": seeders,
"size": torrent["behaviorHints"][
"videoSize"
], # not the pack size but still useful for prowlarr userss
"tracker": f"MediaFusion|{tracker}",
"sources": torrent["sources"] if "sources" in torrent else [],
}
)
except Exception as e:
logger.warning(
f"Exception while getting torrents for {media_id} with MediaFusion, your IP is most likely blacklisted (you should try proxying Comet): {e}"
)
pass
await manager.filter_manager(torrents)
comet/scrapers/prowlarr.py
+124
View File
@@ -0,0 +1,124 @@
import aiohttp
import asyncio
from comet.utils.models import settings
from comet.utils.logger import logger
from comet.utils.torrent import (
download_torrent,
extract_torrent_metadata,
extract_trackers_from_magnet,
add_torrent_queue,
)
async def process_torrent(
session: aiohttp.ClientSession, result: dict, media_id: str, season: int
):
base_torrent = {
"title": result["title"],
"infoHash": None,
"fileIndex": None,
"seeders": result["seeders"],
"size": result["size"],
"tracker": result["indexer"],
"sources": [],
}
torrents = []
if "downloadUrl" in result:
content, magnet_hash, magnet_url = await download_torrent(
session, result["downloadUrl"]
)
if content:
metadata = extract_torrent_metadata(content)
if metadata:
for file in metadata["files"]:
torrent = base_torrent.copy()
torrent["title"] = file["name"]
torrent["infoHash"] = metadata["info_hash"].lower()
torrent["fileIndex"] = file["index"]
torrent["size"] = file["size"]
torrent["sources"] = metadata["announce_list"]
torrents.append(torrent)
return torrents
if magnet_hash:
base_torrent["infoHash"] = magnet_hash.lower()
base_torrent["sources"] = extract_trackers_from_magnet(magnet_url)
await add_torrent_queue.add_torrent(
magnet_url,
base_torrent["seeders"],
base_torrent["tracker"],
media_id,
season,
)
torrents.append(base_torrent)
return torrents
if "infoHash" in result and result["infoHash"]:
base_torrent["infoHash"] = result["infoHash"].lower()
if "guid" in result and result["guid"].startswith("magnet:"):
base_torrent["sources"] = extract_trackers_from_magnet(result["guid"])
await add_torrent_queue.add_torrent(
result["guid"],
base_torrent["seeders"],
base_torrent["tracker"],
media_id,
season,
)
torrents.append(base_torrent)
return torrents
async def get_prowlarr(manager, session: aiohttp.ClientSession, title: str, seen: set):
torrents = []
try:
indexers = [indexer.lower() for indexer in settings.INDEXER_MANAGER_INDEXERS]
get_indexers = await session.get(
f"{settings.INDEXER_MANAGER_URL}/api/v1/indexer",
headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY},
)
get_indexers = await get_indexers.json()
indexers_id = []
for indexer in get_indexers:
if (
indexer["name"].lower() in indexers
or indexer["definitionName"].lower() in indexers
):
indexers_id.append(indexer["id"])
response = await session.get(
f"{settings.INDEXER_MANAGER_URL}/api/v1/search?query={title}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search",
headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY},
)
response = await response.json()
torrent_tasks = []
for result in response:
if result["infoUrl"] in seen:
continue
seen.add(result["infoUrl"])
torrent_tasks.append(
process_torrent(session, result, manager.media_only_id, manager.season)
)
processed_torrents = await asyncio.gather(*torrent_tasks)
torrents = [
t for sublist in processed_torrents for t in sublist if t["infoHash"]
]
except Exception as e:
logger.warning(
f"Exception while getting torrents for {title} with Prowlarr: {e}"
)
await manager.filter_manager(torrents)
comet/scrapers/torrentio.py
+65
View File
@@ -0,0 +1,65 @@
import re
from curl_cffi import requests
from comet.utils.models import settings
from comet.utils.logger import logger
from comet.utils.general import size_to_bytes
data_pattern = re.compile(
r"(?:👤 (\d+) )?💾 ([\d.]+ [KMGT]B)(?: ⚙️ (\w+))?", re.IGNORECASE
)
async def get_torrentio(manager, media_type: str, media_id: str):
torrents = []
try:
try:
get_torrentio = requests.get(
f"{settings.TORRENTIO_URL}/stream/{media_type}/{media_id}.json"
).json()
except Exception as e:
logger.warning(
f"Failed to get Torrentio results without proxy for {media_id}: {e}"
)
get_torrentio = requests.get(
f"{settings.TORRENTIO_URL}/stream/{media_type}/{media_id}.json",
proxies={
"http": settings.DEBRID_PROXY_URL,
"https": settings.DEBRID_PROXY_URL,
},
).json()
for torrent in get_torrentio["streams"]:
title_full = torrent["title"]
title = (
title_full.split("\n")[0]
if settings.TORRENTIO_URL == "https://torrentio.strem.fun"
else title_full.split("\n💾")[0].split("\n")[-1]
)
match = data_pattern.search(title_full)
seeders = int(match.group(1)) if match.group(1) else None
size = size_to_bytes(match.group(2))
tracker = match.group(3) if match.group(3) else "KnightCrawler"
torrents.append(
{
"title": title,
"infoHash": torrent["infoHash"].lower(),
"fileIndex": torrent["fileIdx"] if "fileIdx" in torrent else None,
"seeders": seeders,
"size": size,
"tracker": f"Torrentio|{tracker}",
"sources": torrent["sources"] if "sources" in torrent else [],
}
)
except Exception as e:
logger.warning(
f"Exception while getting torrents for {media_id} with Torrentio, your IP is most likely blacklisted (you should try proxying Comet): {e}"
)
await manager.filter_manager(torrents)
comet/scrapers/zilean.py
+33
View File
@@ -0,0 +1,33 @@
import aiohttp
from comet.utils.models import settings
from comet.utils.logger import logger
async def get_zilean(
manager, session: aiohttp.ClientSession, title: str, season: int, episode: int
):
torrents = []
try:
show = f"&season={season}&episode={episode}"
get_dmm = await session.get(
f"{settings.ZILEAN_URL}/dmm/filtered?query={title}{show if season else ''}"
)
get_dmm = await get_dmm.json()
for result in get_dmm:
object = {
"title": result["raw_title"],
"infoHash": result["info_hash"].lower(),
"fileIndex": None,
"seeders": None,
"size": int(result["size"]),
"tracker": "DMM",
"sources": [],
}
torrents.append(object)
except Exception as e:
logger.warning(f"Exception while getting torrents for {title} with Zilean: {e}")
await manager.filter_manager(torrents)
comet/templates/index.html
+232 -119
View File
@@ -12,8 +12,8 @@
<title>Comet - Stremio's fastest torrent/debrid search add-on.</title> <title>Comet - Stremio's fastest torrent/debrid search add-on.</title>
<link id="favicon" rel="icon" type="image/x-icon" href="https://i.imgur.com/jmVoVMu.jpeg"> <link id="favicon" rel="icon" type="image/x-icon" href="https://i.imgur.com/jmVoVMu.jpeg">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@shoelace-style/shoelace@2.15.1/cdn/themes/dark.css" /> <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@shoelace-style/shoelace@2.19.0/cdn/themes/dark.css" />
<script type="module" src="https://cdn.jsdelivr.net/npm/@shoelace-style/shoelace@2.15.1/cdn/shoelace-autoloader.js"></script> <script type="module" src="https://cdn.jsdelivr.net/npm/@shoelace-style/shoelace@2.19.0/cdn/shoelace-autoloader.js"></script>
<style> <style>
:not(:defined) { :not(:defined) {
@@ -512,19 +512,26 @@
<div class="form-container"> <div class="form-container">
<div class="form-item"> <div class="form-item">
<sl-select id="indexers" multiple clearable label="Indexers" placeholder="Select indexers" {% if not indexerManager %} disabled {% endif %}></sl-select> <sl-select id="languages_required" multiple clearable label="Required Languages" placeholder="Select required languages"></sl-select>
</div> </div>
<div class="form-item"> <div class="form-item">
<sl-select id="languages" multiple clearable label="Languages" placeholder="Select languages"></sl-select> <sl-select id="languages_preferred" multiple clearable label="Preferred Languages" placeholder="Select preferred languages"></sl-select>
</div> </div>
<div class="form-item"> <div class="form-item">
<sl-select id="resolutions" multiple clearable label="Resolutions" placeholder="Select resolutions"></sl-select> <sl-select id="languages_excluded" multiple clearable label="Excluded Languages" placeholder="Select excluded languages"></sl-select>
</div> </div>
<div class="form-item"> <div class="form-item">
<sl-input id="maxResults" type="number" min=0 value=0 label="Max Results" placeholder="Enter max results"></sl-input> <sl-select id="resolutions" multiple clearable label="Resolutions" placeholder="Select resolutions">
<sl-option value="r2160p">4K - 2160p</sl-option>
<sl-option value="r1080p">Full HD - 1080p</sl-option>
<sl-option value="r720p">HD - 720p</sl-option>
<sl-option value="r480p">SD - 480p</sl-option>
<sl-option value="r360p">LD - 360p</sl-option>
<sl-option value="unknown">Unknown</sl-option>
</sl-select>
</div> </div>
<div class="form-item"> <div class="form-item">
@@ -535,37 +542,39 @@
<sl-input id="maxSize" type="number" min=0 value=0 label="Max Size (GB)" placeholder="Enter max size in gigabytes"></sl-input> <sl-input id="maxSize" type="number" min=0 value=0 label="Max Size (GB)" placeholder="Enter max size in gigabytes"></sl-input>
</div> </div>
<div class="form-item"> <div class="form-item" {% if not proxyDebridStream %}style="display: none"{% endif %}>
<sl-input id="debridStreamProxyPassword" label="Debrid Stream Proxy Password" placeholder="{% if not proxyDebridStream %}Configuration required, see docs{% else %}Enter password{% endif %}" help-text="Debrid Stream Proxying allows you to use your Debrid Service from multiple IPs at same time!" {% if not proxyDebridStream %} disabled {% endif %}></sl-input> <sl-input id="debridStreamProxyPassword" label="Debrid Stream Proxy Password" placeholder="Enter password" help-text="Debrid Stream Proxying allows you to use your Debrid Service from multiple IPs at same time!"></sl-input>
</div> </div>
<div class="form-item"> <div class="form-item">
<sl-select id="debridService" value="torbox" label="Debrid Service" placeholder="Select debrid service"> <sl-select id="debridService" value="torrent" label="Debrid Service" placeholder="Select debrid service">
<sl-option value="torrent">Torrent</sl-option>
<sl-option value="torbox">TorBox</sl-option> <sl-option value="torbox">TorBox</sl-option>
<sl-option value="easydebrid">EasyDebrid</sl-option>
<sl-option value="realdebrid">Real-Debrid</sl-option>
<sl-option value="debridlink">Debrid-Link</sl-option> <sl-option value="debridlink">Debrid-Link</sl-option>
<sl-option value="alldebrid">All-Debrid</sl-option> <sl-option value="alldebrid">All-Debrid</sl-option>
<sl-option value="premiumize">Premiumize</sl-option> <sl-option value="premiumize">Premiumize</sl-option>
<sl-option value="realdebrid">Real-Debrid</sl-option> <sl-option value="offcloud">Offcloud</sl-option>
<sl-option value="stremthru">StremThru</sl-option> <sl-option value="offcloud">PikPak</sl-option>
</sl-select> </sl-select>
</div> </div>
<div class="form-item"> <div class="form-item">
<label for="debridService"> <label for="debridService">
Debrid API Key - Debrid API Key
<a id="apiKeyLink" href="https://torbox.app/settings" target="_blank">Get It Here</a> <a id="apiKeyLink" href="" target="_blank"></a>
</label> </label>
<sl-input id="debridApiKey" placeholder="Enter API key" help-text="If no key is specified, you'll get direct torrents!"></sl-input> <sl-input id="debridApiKey" placeholder="Enter API key"></sl-input>
</div>
<div class="form-item">
<sl-input id="stremthruUrl" label="StremThru URL" placeholder="{{stremthruDefaultUrl}}" help-text="URL for StremThru."></sl-input>
</div> </div>
<sl-details summary="Advanced Settings"> <sl-details summary="Advanced Settings">
<div class="form-item"> <div class="form-item">
<sl-checkbox checked id="removeTrash" help-text="Remove all trash from results (Adult Content, CAM, Clean Audio, PDTV, R5, Screener, Size, Telecine and Telesync)">Remove Trash</sl-checkbox> <sl-input id="rankThreshold" type="number" value="-10000000000" label="Minimum Rank Threshold" placeholder="Enter minimum rank threshold" help-text="Torrents ranked below this value will be excluded from results."></sl-input>
<sl-checkbox id="reverseResultOrder" help-text="Reverse the order of the results for each resolution (useful for those who need small file sizes)">Reverse Result Order</sl-checkbox> <sl-checkbox id="cachedOnly" help-text="Show only content that has already been cached.">Show Cached Only</sl-checkbox>
<sl-checkbox id="allowEnglishInLanguages" help-text="Allows English language torrents to be included even if you excluded English from your language preferences.">Allow English in Languages</sl-checkbox>
<sl-checkbox id="removeUnknownLanguages" help-text="Exclude torrents that don't have a language in the title.">Remove Unknown Languages</sl-checkbox>
<sl-checkbox checked id="removeTrash" help-text="Remove all trash from results (if disabled, your filters will not be applied - Adult Content, CAM, Clean Audio, PDTV, R5, Screener, Size, Telecine and Telesync)">Remove Trash</sl-checkbox>
<sl-select id="resultFormat" multiple label="Result Format" placeholder="Select what to show in result title" hoist max-options-visible=10> <sl-select id="resultFormat" multiple label="Result Format" placeholder="Select what to show in result title" hoist max-options-visible=10>
</sl-select> </sl-select>
</div> </div>
@@ -576,24 +585,51 @@
const selectedService = event.target.value; const selectedService = event.target.value;
const apiKeyLink = document.getElementById("apiKeyLink"); const apiKeyLink = document.getElementById("apiKeyLink");
const apiKeyInput = document.getElementById("debridApiKey"); const apiKeyInput = document.getElementById("debridApiKey");
debridStreamProxyPasswordInput = document.getElementById("debridStreamProxyPassword");
if (selectedService === "realdebrid") { if (selectedService === "realdebrid") {
apiKeyLink.href = "https://real-debrid.com/apitoken"; apiKeyLink.href = "https://real-debrid.com/apitoken";
apiKeyLink.textContent = "Get It Here"
} else if (selectedService === "alldebrid") { } else if (selectedService === "alldebrid") {
apiKeyLink.href = "https://alldebrid.com/apikeys"; apiKeyLink.href = "https://alldebrid.com/apikeys";
apiKeyLink.textContent = "Get It Here"
} else if (selectedService === "premiumize") { } else if (selectedService === "premiumize") {
apiKeyLink.href = "https://premiumize.me/account"; apiKeyLink.href = "https://premiumize.me/account";
apiKeyLink.textContent = "Get It Here"
} else if (selectedService === "torbox") { } else if (selectedService === "torbox") {
apiKeyLink.href = "https://torbox.app/settings"; apiKeyLink.href = "https://torbox.app/settings";
apiKeyLink.textContent = "Get It Here"
} else if (selectedService === "easydebrid") {
apiKeyLink.href = "https://paradise-cloud.com/products/easydebrid";
apiKeyLink.textContent = "Get It Here"
} else if (selectedService === "debridlink") { } else if (selectedService === "debridlink") {
apiKeyLink.href = "https://debrid-link.com/webapp/apikey"; apiKeyLink.href = "https://debrid-link.com/webapp/apikey";
} else if (selectedService === "stremthru") { apiKeyLink.textContent = "Get It Here"
apiKeyLink.href = "https://github.com/MunifTanjim/stremthru?tab=readme-ov-file#configuration"; } else if (selectedService === "offcloud") {
apiKeyLink.href = "https://offcloud.com";
apiKeyLink.textContent = "Get It Here"
} else if (selectedService === "pikpak") {
apiKeyLink.href = "https://mypikpak.com";
apiKeyLink.textContent = "Get It Here"
} else if (selectedService === "torrent") {
apiKeyLink.href = "";
apiKeyLink.textContent = ""
} }
if (selectedService === "stremthru") {
apiKeyInput.helpText = "Enter Credential ('store_name:store_token' or base64 encoded basic token from 'STREMTHRU_PROXY_AUTH' config)" if (selectedService === "offcloud" || selectedService === "pikpak") {
apiKeyInput.helpText = "Format: `email:password`"
} else if (selectedService != "torrent") {
apiKeyInput.helpText = "Format: `api-key`"
} else { } else {
apiKeyInput.helpText = "If no key is specified, you'll get direct torrents!" apiKeyInput.helpText = ""
}
if (selectedService === "torrent") {
debridStreamProxyPasswordInput.disabled = true;
apiKeyInput.disabled = true;
} else {
debridStreamProxyPasswordInput.disabled = false;
apiKeyInput.disabled = false;
} }
}); });
</script> </script>
@@ -613,59 +649,55 @@
<script type="module"> <script type="module">
const languagesEmojis = { const languagesEmojis = {
"Unknown": "❓", "multi": "🌎",
"Multi": "🌎", "en": "🇬🇧", // English
"English": "🇬🇧", "ja": "🇯🇵", // Japanese
"Japanese": "🇯🇵", "zh": "🇨🇳", // Chinese
"Chinese": "🇨🇳", "ru": "🇷🇺", // Russian
"Russian": "🇷🇺", "ar": "🇸🇦", // Arabic
"Arabic": "🇸🇦", "pt": "🇵🇹", // Portuguese
"Portuguese": "🇵🇹", "es": "🇪🇸", // Spanish
"Spanish": "🇪🇸", "fr": "🇫🇷", // French
"French": "🇫🇷", "de": "🇩🇪", // German
"German": "🇩🇪", "it": "🇮🇹", // Italian
"Italian": "🇮🇹", "ko": "🇰🇷", // Korean
"Korean": "🇰🇷", "hi": "🇮🇳", // Hindi
"Hindi": "🇮🇳", "bn": "🇧🇩", // Bengali
"Bengali": "🇧🇩", "pa": "🇵🇰", // Punjabi
"Punjabi": "🇵🇰", "mr": "🇮🇳", // Marathi
"Marathi": "🇮🇳", "gu": "🇮🇳", // Gujarati
"Gujarati": "🇮🇳", "ta": "🇮🇳", // Tamil
"Tamil": "🇮🇳", "te": "🇮🇳", // Telugu
"Telugu": "🇮🇳", "kn": "🇮🇳", // Kannada
"Kannada": "🇮🇳", "ml": "🇮🇳", // Malayalam
"Malayalam": "🇮🇳", "th": "🇹🇭", // Thai
"Thai": "🇹🇭", "vi": "🇻🇳", // Vietnamese
"Vietnamese": "🇻🇳", "id": "🇮🇩", // Indonesian
"Indonesian": "🇮🇩", "tr": "🇹🇷", // Turkish
"Turkish": "🇹🇷", "he": "🇮🇱", // Hebrew
"Hebrew": "🇮🇱", "fa": "🇮🇷", // Persian
"Persian": "🇮🇷", "uk": "🇺🇦", // Ukrainian
"Ukrainian": "🇺🇦", "el": "🇬🇷", // Greek
"Greek": "🇬🇷", "lt": "🇱🇹", // Lithuanian
"Lithuanian": "🇱🇹", "lv": "🇱🇻", // Latvian
"Latvian": "🇱🇻", "et": "🇪🇪", // Estonian
"Estonian": "🇪🇪", "pl": "🇵🇱", // Polish
"Polish": "🇵🇱", "cs": "🇨🇿", // Czech
"Czech": "🇨🇿", "sk": "🇸🇰", // Slovak
"Slovak": "🇸🇰", "hu": "🇭🇺", // Hungarian
"Hungarian": "🇭🇺", "ro": "🇷🇴", // Romanian
"Romanian": "🇷🇴", "bg": "🇧🇬", // Bulgarian
"Bulgarian": "🇧🇬", "sr": "🇷🇸", // Serbian
"Serbian": "🇷🇸", "hr": "🇭🇷", // Croatian
"Croatian": "🇭🇷", "sl": "🇸🇮", // Slovenian
"Slovenian": "🇸🇮", "nl": "🇳🇱", // Dutch
"Dutch": "🇳🇱", "da": "🇩🇰", // Danish
"Danish": "🇩🇰", "fi": "🇫🇮", // Finnish
"Finnish": "🇫🇮", "sv": "🇸🇪", // Swedish
"Swedish": "🇸🇪", "no": "🇳🇴", // Norwegian
"Norwegian": "🇳🇴", "ms": "🇲🇾", // Malay
"Malay": "🇲🇾", "la": "💃🏻", // Latino
"Latino": "💃🏻",
}; };
let defaultLanguages = [];
let defaultResolutions = [];
let defaultResultFormat = []; let defaultResultFormat = [];
document.addEventListener("DOMContentLoaded", async () => { document.addEventListener("DOMContentLoaded", async () => {
@@ -680,14 +712,14 @@
]); ]);
const webConfig = {{webConfig|tojson}}; const webConfig = {{webConfig|tojson}};
populateSelect("indexers", webConfig.indexers);
populateSelect("languages", webConfig.languages); const resolutionsSelect = document.getElementById("resolutions");
populateSelect("resolutions", webConfig.resolutions); resolutionsSelect.value = ["r2160p", "r1080p", "r720p", "r480p", "r360p", "unknown"];
populateSelect("resultFormat", webConfig.resultFormat); populateSelect("resultFormat", webConfig.resultFormat);
document.body.classList.add("ready");
defaultLanguages = webConfig.languages;
defaultResolutions = webConfig.resolutions;
defaultResultFormat = webConfig.resultFormat; defaultResultFormat = webConfig.resultFormat;
const resultFormatSelect = document.getElementById("resultFormat");
resultFormatSelect.value = defaultResultFormat;
// try populate the form from previous settings // try populate the form from previous settings
try { try {
@@ -697,28 +729,86 @@
console.log("Failed to retrieve or parse settings from URL:", error); console.log("Failed to retrieve or parse settings from URL:", error);
} }
// trigger sl-change event on debridService to set initial state
const debridServiceSelect = document.getElementById("debridService");
debridServiceSelect.dispatchEvent(new Event("sl-change"));
// Populate language selects
populateSelect("languages_required", Object.keys(languagesEmojis));
populateSelect("languages_excluded", Object.keys(languagesEmojis));
populateSelect("languages_preferred", Object.keys(languagesEmojis));
document.body.classList.add("ready");
}); });
function populateSelect(selectId, options) { function populateSelect(selectId, options) {
const selectElement = document.getElementById(selectId); const selectElement = document.getElementById(selectId);
const languageNames = {
"multi": "Multi",
"en": "English",
"ja": "Japanese",
"zh": "Chinese",
"ru": "Russian",
"ar": "Arabic",
"pt": "Portuguese",
"es": "Spanish",
"fr": "French",
"de": "German",
"it": "Italian",
"ko": "Korean",
"hi": "Hindi",
"bn": "Bengali",
"pa": "Punjabi",
"mr": "Marathi",
"gu": "Gujarati",
"ta": "Tamil",
"te": "Telugu",
"kn": "Kannada",
"ml": "Malayalam",
"th": "Thai",
"vi": "Vietnamese",
"id": "Indonesian",
"tr": "Turkish",
"he": "Hebrew",
"fa": "Persian",
"uk": "Ukrainian",
"el": "Greek",
"lt": "Lithuanian",
"lv": "Latvian",
"et": "Estonian",
"pl": "Polish",
"cs": "Czech",
"sk": "Slovak",
"hu": "Hungarian",
"ro": "Romanian",
"bg": "Bulgarian",
"sr": "Serbian",
"hr": "Croatian",
"sl": "Slovenian",
"nl": "Dutch",
"da": "Danish",
"fi": "Finnish",
"sv": "Swedish",
"no": "Norwegian",
"ms": "Malay",
"la": "Latino"
};
options.forEach(option => { options.forEach(option => {
const optionElement = document.createElement("sl-option"); const optionElement = document.createElement("sl-option");
optionElement.value = option; optionElement.value = option;
if (selectId === "languages") { if (selectId.startsWith('languages')) {
// For languages, prepend the flag emoji if it exists
const flag = languagesEmojis[option] || ''; const flag = languagesEmojis[option] || '';
optionElement.textContent = `${flag} ${option}`; const fullName = languageNames[option] || option;
optionElement.textContent = `${flag} ${fullName}`;
} else { } else {
// For other selects, just use the option text as is
optionElement.textContent = option; optionElement.textContent = option;
} }
selectElement.appendChild(optionElement); selectElement.appendChild(optionElement);
}); });
// Set the default value
selectElement.value = options; // Assuming first option as default
} }
const installButton = document.querySelector("#install"); const installButton = document.querySelector("#install");
@@ -727,37 +817,52 @@
const copyAlert = document.querySelector('#copyAlert'); const copyAlert = document.querySelector('#copyAlert');
function getSettings() { function getSettings() {
const indexers = Array.from(document.getElementById("indexers").selectedOptions).map(option => option.value); const languages_required = Array.from(document.getElementById("languages_required").selectedOptions).map(option => option.value);
const languages = Array.from(document.getElementById("languages").selectedOptions).map(option => option.value); const languages_excluded = Array.from(document.getElementById("languages_excluded").selectedOptions).map(option => option.value);
const resolutions = Array.from(document.getElementById("resolutions").selectedOptions).map(option => option.value); const languages_preferred = Array.from(document.getElementById("languages_preferred").selectedOptions).map(option => option.value);
const maxResults = document.getElementById("maxResults").value; const selectedResolutions = Array.from(document.getElementById("resolutions").selectedOptions).map(option => option.value);
const allResolutions = ["r2160p", "r1080p", "r720p", "r480p", "r360p", "unknown"];
const resolutions = allResolutions.reduce((obj, res) => {
// Only include resolutions that are NOT selected (disabled) as false
if (!selectedResolutions.includes(res)) {
obj[res] = false;
}
return obj;
}, {});
const rankThreshold = document.getElementById("rankThreshold").value;
const maxResultsPerResolution = document.getElementById("maxResultsPerResolution").value; const maxResultsPerResolution = document.getElementById("maxResultsPerResolution").value;
const maxSize = document.getElementById("maxSize").value; const maxSize = document.getElementById("maxSize").value;
const reverseResultOrder = document.getElementById("reverseResultOrder").checked; const cachedOnly = document.getElementById("cachedOnly").checked;
const removeTrash = document.getElementById("removeTrash").checked; const removeTrash = document.getElementById("removeTrash").checked;
const allowEnglishInLanguages = document.getElementById("allowEnglishInLanguages").checked;
const removeUnknownLanguages = document.getElementById("removeUnknownLanguages").checked;
const resultFormat = Array.from(document.getElementById("resultFormat").selectedOptions).map(option => option.value); const resultFormat = Array.from(document.getElementById("resultFormat").selectedOptions).map(option => option.value);
const debridService = document.getElementById("debridService").value; const debridService = document.getElementById("debridService").value;
const debridApiKey = document.getElementById("debridApiKey").value; const debridApiKey = document.getElementById("debridApiKey").value;
const stremthruUrl = document.getElementById("stremthruUrl").value;
const debridStreamProxyPassword = document.getElementById("debridStreamProxyPassword").value; const debridStreamProxyPassword = document.getElementById("debridStreamProxyPassword").value;
const selectedLanguages = languages.length === defaultLanguages.length && languages.every((val, index) => val === defaultLanguages[index]) ? ["All"] : languages;
const selectedResolutions = resolutions.length === defaultResolutions.length && resolutions.every((val, index) => val === defaultResolutions[index]) ? ["All"] : resolutions; const selectedResultFormat = resultFormat.length === defaultResultFormat.length && resultFormat.every((val, index) => val === defaultResultFormat[index]) ? ["all"] : resultFormat;
const selectedResultFormat = resultFormat.length === defaultResultFormat.length && resultFormat.every((val, index) => val === defaultResultFormat[index]) ? ["All"] : resultFormat;
return { return {
indexers: indexers,
maxResults: parseInt(maxResults),
maxResultsPerResolution: parseInt(maxResultsPerResolution), maxResultsPerResolution: parseInt(maxResultsPerResolution),
maxSize: parseFloat(maxSize * 1073741824), maxSize: parseFloat(maxSize * 1073741824),
reverseResultOrder: reverseResultOrder, cachedOnly: cachedOnly,
removeTrash: removeTrash, removeTrash: removeTrash,
resultFormat: selectedResultFormat, resultFormat: selectedResultFormat,
resolutions: selectedResolutions,
languages: selectedLanguages,
debridService: debridService, debridService: debridService,
debridApiKey: debridApiKey, debridApiKey: debridApiKey,
stremthruUrl: stremthruUrl,
debridStreamProxyPassword: debridStreamProxyPassword, debridStreamProxyPassword: debridStreamProxyPassword,
languages: {
required: languages_required,
exclude: languages_excluded,
preferred: languages_preferred
},
resolutions: resolutions,
options: {
remove_ranks_under: parseFloat(rankThreshold),
allow_english_in_languages: allowEnglishInLanguages,
remove_unknown_languages: removeUnknownLanguages
}
}; };
} }
@@ -802,10 +907,8 @@
} }
function populateFormFromSettings(settings) { function populateFormFromSettings(settings) {
if (settings.maxResults !== null) if (settings.cachedOnly !== null)
document.getElementById("maxResults").value = settings.maxResults; document.getElementById("cachedOnly").checked = settings.cachedOnly;
if (settings.reverseResultOrder !== null)
document.getElementById("reverseResultOrder").checked = settings.reverseResultOrder;
if (settings.removeTrash !== null) if (settings.removeTrash !== null)
document.getElementById("removeTrash").checked = settings.removeTrash; document.getElementById("removeTrash").checked = settings.removeTrash;
if (settings.maxResultsPerResolution !== null) if (settings.maxResultsPerResolution !== null)
@@ -818,18 +921,28 @@
document.getElementById("debridApiKey").value = settings.debridApiKey; document.getElementById("debridApiKey").value = settings.debridApiKey;
if (settings.debridStreamProxyPassword !== null) if (settings.debridStreamProxyPassword !== null)
document.getElementById("debridStreamProxyPassword").value = settings.debridStreamProxyPassword; document.getElementById("debridStreamProxyPassword").value = settings.debridStreamProxyPassword;
if (settings.indexers !== null) if (settings.resultFormat !== null && settings.resultFormat != "all")
document.getElementById("indexers").value = settings.indexers;
if (settings.languages !== null && settings.languages != "All")
document.getElementById("languages").value = settings.languages;
if (settings.stremthruUrl !== null)
document.getElementById("stremthruUrl").value = settings.stremthruUrl;
if (settings.resolutions !== null && settings.resolutions != "All")
document.getElementById("resolutions").value = settings.resolutions;
if (settings.resultFormat !== null && settings.resultFormat != "All")
document.getElementById("resultFormat").value = settings.resultFormat; document.getElementById("resultFormat").value = settings.resultFormat;
if (settings.languages) {
if (settings.languages.required)
document.getElementById("languages_required").value = settings.languages.required;
if (settings.languages.exclude)
document.getElementById("languages_excluded").value = settings.languages.exclude;
if (settings.languages.preferred)
document.getElementById("languages_preferred").value = settings.languages.preferred;
}
if (settings.resolutions) {
const allResolutions = ["r2160p", "r1080p", "r720p", "r480p", "r360p", "unknown"];
const enabledResolutions = allResolutions.filter(res => settings.resolutions[res] !== false);
document.getElementById("resolutions").value = enabledResolutions;
}
if (settings.options?.remove_ranks_under !== undefined)
document.getElementById("rankThreshold").value = settings.options.remove_ranks_under;
if (settings.options?.allow_english_in_languages !== undefined)
document.getElementById("allowEnglishInLanguages").checked = settings.options.allow_english_in_languages;
if (settings.options?.remove_unknown_languages !== undefined)
document.getElementById("removeUnknownLanguages").checked = settings.options.remove_unknown_languages;
} }
</script> </script>
</div> </div>
</div> </div>
comet/utils/database.py
+318
View File
@@ -0,0 +1,318 @@
import os
import time
import traceback
from comet.utils.logger import logger
from comet.utils.models import database, settings
DATABASE_VERSION = "1.0"
async def setup_database():
try:
if settings.DATABASE_TYPE == "sqlite":
os.makedirs(os.path.dirname(settings.DATABASE_PATH), exist_ok=True)
if not os.path.exists(settings.DATABASE_PATH):
open(settings.DATABASE_PATH, "a").close()
await database.connect()
await database.execute(
"""
CREATE TABLE IF NOT EXISTS db_version (
id INTEGER PRIMARY KEY CHECK (id = 1),
version TEXT
)
"""
)
current_version = await database.fetch_val(
"""
SELECT version FROM db_version WHERE id = 1
"""
)
if current_version != DATABASE_VERSION:
logger.log("COMET", f"Database: Migration from {current_version} to {DATABASE_VERSION} version")
if settings.DATABASE_TYPE == "sqlite":
tables = await database.fetch_all(
"""
SELECT name FROM sqlite_master
WHERE type='table' AND name != 'db_version' AND name != 'sqlite_sequence'
"""
)
for table in tables:
await database.execute(f"DROP TABLE IF EXISTS {table['name']}")
else:
await database.execute(
"""
DO $$ DECLARE
r RECORD;
BEGIN
FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema() AND tablename != 'db_version') LOOP
EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE';
END LOOP;
END $$;
"""
)
await database.execute(
"""
INSERT INTO db_version VALUES (1, :version)
ON CONFLICT (id) DO UPDATE SET version = :version
""",
{"version": DATABASE_VERSION},
)
logger.log("COMET", f"Database: Migration to version {DATABASE_VERSION} completed")
await database.execute(
"""
CREATE TABLE IF NOT EXISTS ongoing_searches (
media_id TEXT PRIMARY KEY,
timestamp INTEGER
)
"""
)
await database.execute(
"""
CREATE TABLE IF NOT EXISTS first_searches (
media_id TEXT PRIMARY KEY,
timestamp INTEGER
)
"""
)
await database.execute(
"""
CREATE TABLE IF NOT EXISTS metadata_cache (
media_id TEXT PRIMARY KEY,
title TEXT,
year INTEGER,
year_end INTEGER,
aliases TEXT,
timestamp INTEGER
)
"""
)
await database.execute(
"""
CREATE TABLE IF NOT EXISTS torrents (
media_id TEXT,
info_hash TEXT,
file_index INTEGER,
season INTEGER,
episode INTEGER,
title TEXT,
seeders INTEGER,
size BIGINT,
tracker TEXT,
sources TEXT,
parsed TEXT,
timestamp INTEGER
)
"""
)
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS torrents_series_both_idx
ON torrents (media_id, info_hash, season, episode)
WHERE season IS NOT NULL AND episode IS NOT NULL
"""
)
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS torrents_season_only_idx
ON torrents (media_id, info_hash, season)
WHERE season IS NOT NULL AND episode IS NULL
"""
)
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS torrents_episode_only_idx
ON torrents (media_id, info_hash, episode)
WHERE season IS NULL AND episode IS NOT NULL
"""
)
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS torrents_no_season_episode_idx
ON torrents (media_id, info_hash)
WHERE season IS NULL AND episode IS NULL
"""
)
await database.execute(
"""
CREATE TABLE IF NOT EXISTS debrid_availability (
debrid_service TEXT,
info_hash TEXT,
file_index TEXT,
title TEXT,
season INTEGER,
episode INTEGER,
size BIGINT,
parsed TEXT,
timestamp INTEGER
)
"""
)
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS debrid_series_both_idx
ON debrid_availability (debrid_service, info_hash, season, episode)
WHERE season IS NOT NULL AND episode IS NOT NULL
"""
)
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS debrid_season_only_idx
ON debrid_availability (debrid_service, info_hash, season)
WHERE season IS NOT NULL AND episode IS NULL
"""
)
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS debrid_episode_only_idx
ON debrid_availability (debrid_service, info_hash, episode)
WHERE season IS NULL AND episode IS NOT NULL
"""
)
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS debrid_no_season_episode_idx
ON debrid_availability (debrid_service, info_hash)
WHERE season IS NULL AND episode IS NULL
"""
)
await database.execute(
"""
CREATE TABLE IF NOT EXISTS download_links_cache (
debrid_key TEXT,
info_hash TEXT,
season INTEGER,
episode INTEGER,
download_url TEXT,
timestamp INTEGER
)
"""
)
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS download_links_series_both_idx
ON download_links_cache (debrid_key, info_hash, season, episode)
WHERE season IS NOT NULL AND episode IS NOT NULL
"""
)
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS download_links_season_only_idx
ON download_links_cache (debrid_key, info_hash, season)
WHERE season IS NOT NULL AND episode IS NULL
"""
)
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS download_links_episode_only_idx
ON download_links_cache (debrid_key, info_hash, episode)
WHERE season IS NULL AND episode IS NOT NULL
"""
)
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS download_links_no_season_episode_idx
ON download_links_cache (debrid_key, info_hash)
WHERE season IS NULL AND episode IS NULL
"""
)
await database.execute(
"""
CREATE TABLE IF NOT EXISTS active_connections (
id TEXT PRIMARY KEY,
ip TEXT,
content TEXT,
timestamp INTEGER
)
"""
)
if settings.DATABASE_TYPE == "sqlite":
await database.execute("PRAGMA busy_timeout=30000") # 30 seconds timeout
await database.execute("PRAGMA journal_mode=OFF")
await database.execute("PRAGMA synchronous=OFF")
await database.execute("PRAGMA temp_store=MEMORY")
await database.execute("PRAGMA mmap_size=30000000000")
await database.execute("PRAGMA page_size=4096")
await database.execute("PRAGMA cache_size=-2000")
await database.execute("PRAGMA foreign_keys=OFF")
await database.execute("PRAGMA count_changes=OFF")
await database.execute("PRAGMA secure_delete=OFF")
await database.execute("PRAGMA auto_vacuum=OFF")
await database.execute("DELETE FROM ongoing_searches")
await database.execute(
"""
DELETE FROM first_searches
WHERE timestamp + :cache_ttl < :current_time;
""",
{"cache_ttl": settings.TORRENT_CACHE_TTL, "current_time": time.time()},
)
await database.execute(
"""
DELETE FROM metadata_cache
WHERE timestamp + :cache_ttl < :current_time;
""",
{"cache_ttl": settings.METADATA_CACHE_TTL, "current_time": time.time()},
)
await database.execute(
"""
DELETE FROM torrents
WHERE timestamp + :cache_ttl < :current_time;
""",
{"cache_ttl": settings.TORRENT_CACHE_TTL, "current_time": time.time()},
)
await database.execute(
"""
DELETE FROM debrid_availability
WHERE timestamp + :cache_ttl < :current_time;
""",
{"cache_ttl": settings.DEBRID_CACHE_TTL, "current_time": time.time()},
)
await database.execute("DELETE FROM download_links_cache")
await database.execute("DELETE FROM active_connections")
except Exception as e:
logger.error(f"Error setting up the database: {e}")
logger.exception(traceback.format_exc())
async def teardown_database():
try:
await database.disconnect()
except Exception as e:
logger.error(f"Error tearing down the database: {e}")
logger.exception(traceback.format_exc())
comet/utils/debrid.py
+173
View File
@@ -0,0 +1,173 @@
import time
import orjson
from comet.utils.models import settings, database
from comet.utils.general import default_dump
async def cache_availability(debrid_service: str, availability: list):
current_time = time.time()
values = [
{
"debrid_service": debrid_service,
"info_hash": file["info_hash"],
"file_index": str(file["index"]) if file["index"] is not None else None,
"title": file["title"],
"season": file["season"],
"episode": file["episode"],
"size": file["size"] if file["index"] is not None else None,
"parsed": orjson.dumps(file["parsed"], default_dump).decode("utf-8")
if file["parsed"] is not None
else None,
"timestamp": current_time,
}
for file in availability
]
if settings.DATABASE_TYPE == "sqlite":
query = """
INSERT OR REPLACE
INTO debrid_availability
VALUES (:debrid_service, :info_hash, :file_index, :title, :season, :episode, :size, :parsed, :timestamp)
"""
await database.execute_many(query, values)
elif settings.DATABASE_TYPE == "postgresql":
both_values = []
season_only_values = []
episode_only_values = []
no_season_episode_values = []
for val in values:
if val["season"] is not None and val["episode"] is not None:
both_values.append(val)
elif val["season"] is not None and val["episode"] is None:
season_only_values.append(val)
elif val["season"] is None and val["episode"] is not None:
episode_only_values.append(val)
else:
no_season_episode_values.append(val)
# handle each case separately with appropriate ON CONFLICT clauses
if both_values:
query = """
INSERT INTO debrid_availability
VALUES (:debrid_service, :info_hash, :file_index, :title, :season, :episode, :size, :parsed, :timestamp)
ON CONFLICT (debrid_service, info_hash, season, episode)
WHERE season IS NOT NULL AND episode IS NOT NULL
DO UPDATE SET
title = EXCLUDED.title,
file_index = EXCLUDED.file_index,
size = EXCLUDED.size,
parsed = EXCLUDED.parsed,
timestamp = EXCLUDED.timestamp
"""
await database.execute_many(query, both_values)
if season_only_values:
query = """
INSERT INTO debrid_availability
VALUES (:debrid_service, :info_hash, :file_index, :title, :season, :episode, :size, :parsed, :timestamp)
ON CONFLICT (debrid_service, info_hash, season)
WHERE season IS NOT NULL AND episode IS NULL
DO UPDATE SET
title = EXCLUDED.title,
file_index = EXCLUDED.file_index,
size = EXCLUDED.size,
parsed = EXCLUDED.parsed,
timestamp = EXCLUDED.timestamp
"""
await database.execute_many(query, season_only_values)
if episode_only_values:
query = """
INSERT INTO debrid_availability
VALUES (:debrid_service, :info_hash, :file_index, :title, :season, :episode, :size, :parsed, :timestamp)
ON CONFLICT (debrid_service, info_hash, episode)
WHERE season IS NULL AND episode IS NOT NULL
DO UPDATE SET
title = EXCLUDED.title,
file_index = EXCLUDED.file_index,
size = EXCLUDED.size,
parsed = EXCLUDED.parsed,
timestamp = EXCLUDED.timestamp
"""
await database.execute_many(query, episode_only_values)
if no_season_episode_values:
query = """
INSERT INTO debrid_availability
VALUES (:debrid_service, :info_hash, :file_index, :title, :season, :episode, :size, :parsed, :timestamp)
ON CONFLICT (debrid_service, info_hash)
WHERE season IS NULL AND episode IS NULL
DO UPDATE SET
title = EXCLUDED.title,
file_index = EXCLUDED.file_index,
size = EXCLUDED.size,
parsed = EXCLUDED.parsed,
timestamp = EXCLUDED.timestamp
"""
await database.execute_many(query, no_season_episode_values)
else:
query = """
INSERT
INTO debrid_availability
VALUES (:debrid_service, :info_hash, :file_index, :title, :season, :episode, :size, :parsed, :timestamp)
"""
await database.execute_many(query, values)
async def get_cached_availability(
debrid_service: str, info_hashes: list, season: int = None, episode: int = None
):
base_query = f"""
SELECT info_hash, file_index, title, size, parsed
FROM debrid_availability
WHERE info_hash IN (SELECT cast(value as TEXT) FROM {"json_array_elements_text" if settings.DATABASE_TYPE == "postgresql" else "json_each"}(:info_hashes))
AND debrid_service = :debrid_service
AND timestamp + :cache_ttl >= :current_time
"""
params = {
"info_hashes": orjson.dumps(info_hashes).decode("utf-8"),
"debrid_service": debrid_service,
"cache_ttl": settings.DEBRID_CACHE_TTL,
"current_time": time.time(),
"season": season,
"episode": episode,
}
if debrid_service == "offcloud":
query = (
base_query
+ """
AND ((cast(:season as INTEGER) IS NULL AND season IS NULL) OR season = cast(:season as INTEGER))
AND ((cast(:episode as INTEGER) IS NULL AND episode IS NULL) OR episode = cast(:episode as INTEGER))
"""
)
results = await database.fetch_all(query, params)
found_hashes = {r["info_hash"] for r in results}
remaining_hashes = [h for h in info_hashes if h not in found_hashes]
if remaining_hashes:
null_title_params = {
"info_hashes": orjson.dumps(remaining_hashes).decode("utf-8"),
"debrid_service": debrid_service,
"cache_ttl": settings.DEBRID_CACHE_TTL,
"current_time": time.time(),
}
null_title_query = base_query + " AND title IS NULL"
null_results = await database.fetch_all(null_title_query, null_title_params)
results.extend(null_results)
else:
query = (
base_query
+ """
AND ((cast(:season as INTEGER) IS NULL AND season IS NULL) OR season = cast(:season as INTEGER))
AND ((cast(:episode as INTEGER) IS NULL AND episode IS NULL) OR episode = cast(:episode as INTEGER))
"""
)
results = await database.fetch_all(query, params)
return results
comet/utils/general.py
+179 -657
View File
@@ -1,20 +1,101 @@
import base64 import base64
import hashlib
import re
import aiohttp
import bencodepy
import PTT
import asyncio
import orjson import orjson
import time
import copy
from RTN import parse, title_match from RTN import ParsedData
from curl_cffi import requests
from fastapi import Request from fastapi import Request
from comet.utils.logger import logger from comet.utils.models import (
from comet.utils.models import database, settings, ConfigModel ConfigModel,
default_config,
settings,
rtn_settings_default,
rtn_ranking_default,
)
def config_check(b64config: str):
try:
config = orjson.loads(base64.b64decode(b64config).decode())
if "indexers" in config:
return False
validated_config = ConfigModel(**config)
validated_config = validated_config.model_dump()
for key in list(validated_config["options"].keys()):
if key not in [
"remove_ranks_under",
"allow_english_in_languages",
"remove_unknown_languages",
]:
validated_config["options"].pop(key)
validated_config["options"]["remove_all_trash"] = validated_config[
"removeTrash"
]
rtn_settings = rtn_settings_default.model_copy(
update={
"resolutions": rtn_settings_default.resolutions.model_copy(
update=validated_config["resolutions"]
),
"options": rtn_settings_default.options.model_copy(
update=validated_config["options"]
),
"languages": rtn_settings_default.languages.model_copy(
update=validated_config["languages"]
),
}
)
validated_config["rtnSettings"] = rtn_settings
validated_config["rtnRanking"] = rtn_ranking_default
if (
settings.PROXY_DEBRID_STREAM
and settings.PROXY_DEBRID_STREAM_PASSWORD
== validated_config["debridStreamProxyPassword"]
and validated_config["debridApiKey"] == ""
):
validated_config["debridService"] = (
settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE
)
validated_config["debridApiKey"] = (
settings.PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY
)
return validated_config
except Exception:
return default_config # if it doesn't pass, return default config
def bytes_to_size(bytes: int):
sizes = ["Bytes", "KB", "MB", "GB", "TB"]
if bytes == 0:
return "0 Byte"
i = 0
while bytes >= 1024 and i < len(sizes) - 1:
bytes /= 1024
i += 1
return f"{round(bytes, 2)} {sizes[i]}"
def size_to_bytes(size_str: str):
sizes = ["b", "kb", "mb", "gb", "tb"]
value, unit = size_str.split()
value = float(value)
unit = unit.lower()
if unit not in sizes:
return None
multiplier = 1024 ** sizes.index(unit)
return int(value * multiplier)
languages_emojis = { languages_emojis = {
"unknown": "❓", # Unknown "unknown": "❓", # Unknown
@@ -78,104 +159,78 @@ def get_language_emoji(language: str):
) )
translation_table = { def format_metadata(data: ParsedData):
"ā": "a", extras = []
"ă": "a", if data.quality:
"ą": "a", extras.append(data.quality)
"ć": "c", if data.hdr:
"č": "c", extras.extend(data.hdr)
"ç": "c", if data.codec:
"ĉ": "c", extras.append(data.codec)
"ċ": "c", if data.audio:
"ď": "d", extras.extend(data.audio)
"đ": "d", if data.channels:
"è": "e", extras.extend(data.channels)
"é": "e", if data.bit_depth:
"ê": "e", extras.append(data.bit_depth)
"ë": "e", if data.network:
"ē": "e", extras.append(data.network)
"ĕ": "e", if data.group:
"ę": "e", extras.append(data.group)
"ě": "e",
"ĝ": "g",
"ğ": "g",
"ġ": "g",
"ģ": "g",
"ĥ": "h",
"î": "i",
"ï": "i",
"ì": "i",
"í": "i",
"ī": "i",
"ĩ": "i",
"ĭ": "i",
"ı": "i",
"ĵ": "j",
"ķ": "k",
"ĺ": "l",
"ļ": "l",
"ł": "l",
"ń": "n",
"ň": "n",
"ñ": "n",
"ņ": "n",
"ʼn": "n",
"ó": "o",
"ô": "o",
"õ": "o",
"ö": "o",
"ø": "o",
"ō": "o",
"ő": "o",
"œ": "oe",
"ŕ": "r",
"ř": "r",
"ŗ": "r",
"š": "s",
"ş": "s",
"ś": "s",
"ș": "s",
"ß": "ss",
"ť": "t",
"ţ": "t",
"ū": "u",
"ŭ": "u",
"ũ": "u",
"û": "u",
"ü": "u",
"ù": "u",
"ú": "u",
"ų": "u",
"ű": "u",
"ŵ": "w",
"ý": "y",
"ÿ": "y",
"ŷ": "y",
"ž": "z",
"ż": "z",
"ź": "z",
"æ": "ae",
"ǎ": "a",
"ǧ": "g",
"ə": "e",
"ƒ": "f",
"ǐ": "i",
"ǒ": "o",
"ǔ": "u",
"ǚ": "u",
"ǜ": "u",
"ǹ": "n",
"ǻ": "a",
"ǽ": "ae",
"ǿ": "o",
}
translation_table = str.maketrans(translation_table) return "|".join(extras)
info_hash_pattern = re.compile(r"\b([a-fA-F0-9]{40})\b")
def translate(title: str): def format_title(
return title.translate(translation_table) data: ParsedData,
ttitle: str,
seeders: int,
size: int,
tracker: str,
result_format: list,
):
has_all = "all" in result_format
title = ""
if has_all or "title" in result_format:
title += f"{ttitle}\n"
if has_all or "metadata" in result_format:
metadata = format_metadata(data)
if metadata != "":
title += f"💿 {metadata}\n"
if (has_all or "seeders" in result_format) and seeders is not None:
title += f"👤 {seeders} "
if has_all or "size" in result_format:
title += f"💾 {bytes_to_size(size)} "
if has_all or "tracker" in result_format:
title += f"🔎 {tracker}"
if has_all or "languages" in result_format:
languages = data.languages
if languages:
formatted_languages = "/".join(
get_language_emoji(language) for language in languages
)
languages_str = "\n" + formatted_languages
title += f"{languages_str}"
if title == "":
# Without this, Streamio shows SD as the result, which is confusing
title = "Empty result format configuration"
return title
def get_client_ip(request: Request):
return (
request.headers["cf-connecting-ip"]
if "cf-connecting-ip" in request.headers
else request.client.host
)
def is_video(title: str): def is_video(title: str):
@@ -221,553 +276,20 @@ def is_video(title: str):
return title.endswith(video_extensions) return title.endswith(video_extensions)
def bytes_to_size(bytes: int): def default_dump(obj):
sizes = ["Bytes", "KB", "MB", "GB", "TB"] if isinstance(obj, ParsedData):
if bytes == 0: return obj.model_dump()
return "0 Byte"
i = 0 def parse_media_id(media_type: str, media_id: str):
while bytes >= 1024 and i < len(sizes) - 1: if media_type == "series":
bytes /= 1024 info = media_id.split(":")
i += 1
if "kitsu" in media_id:
return f"{round(bytes, 2)} {sizes[i]}" return info[1], 1, int(info[2])
return info[0], int(info[1]), int(info[2])
def size_to_bytes(size_str: str): elif media_type == "movie" and "kitsu" in media_id:
sizes = ["bytes", "kb", "mb", "gb", "tb"] info = media_id.split(":")
try: return info[0], int(info[1]), None
value, unit = size_str.split() return media_id, None, None
value = float(value)
unit = unit.lower()
if unit not in sizes:
return None
multiplier = 1024 ** sizes.index(unit)
return int(value * multiplier)
except:
return None
def config_check(b64config: str):
try:
config = orjson.loads(base64.b64decode(b64config).decode())
validated_config = ConfigModel(**config)
return validated_config.model_dump()
except:
return False
def get_debrid_extension(debridService: str, debridApiKey: str = None):
if debridApiKey == "":
return "TORRENT"
debrid_extensions = {
"realdebrid": "RD",
"alldebrid": "AD",
"premiumize": "PM",
"torbox": "TB",
"debridlink": "DL",
"easydebrid": "ED",
"stremthru": "ST",
}
extension = debrid_extensions.get(debridService, None)
if extension == "ST" and debridApiKey and ":" in debridApiKey:
ext = debrid_extensions[debridApiKey.split(":")[0]]
return f"{extension}({ext})" if extension != ext else extension
return extension
async def get_indexer_manager(
session: aiohttp.ClientSession,
indexer_manager_type: str,
indexers: list,
query: str,
):
results = []
try:
indexers = [indexer.replace("_", " ") for indexer in indexers]
if indexer_manager_type == "jackett":
async def fetch_jackett_results(
session: aiohttp.ClientSession, indexer: str, query: str
):
try:
async with session.get(
f"{settings.INDEXER_MANAGER_URL}/api/v2.0/indexers/all/results?apikey={settings.INDEXER_MANAGER_API_KEY}&Query={query}&Tracker[]={indexer}",
timeout=aiohttp.ClientTimeout(
total=settings.INDEXER_MANAGER_TIMEOUT
),
) as response:
response_json = await response.json()
return response_json.get("Results", [])
except Exception as e:
logger.warning(
f"Exception while fetching Jackett results for indexer {indexer}: {e}"
)
return []
tasks = [
fetch_jackett_results(session, indexer, query) for indexer in indexers
]
all_results = await asyncio.gather(*tasks)
for result_set in all_results:
results.extend(result_set)
elif indexer_manager_type == "prowlarr":
get_indexers = await session.get(
f"{settings.INDEXER_MANAGER_URL}/api/v1/indexer",
headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY},
)
get_indexers = await get_indexers.json()
indexers_id = []
for indexer in get_indexers:
if (
indexer["name"].lower() in indexers
or indexer["definitionName"].lower() in indexers
):
indexers_id.append(indexer["id"])
response = await session.get(
f"{settings.INDEXER_MANAGER_URL}/api/v1/search?query={query}&indexerIds={'&indexerIds='.join(str(indexer_id) for indexer_id in indexers_id)}&type=search",
headers={"X-Api-Key": settings.INDEXER_MANAGER_API_KEY},
)
response = await response.json()
for result in response:
result["InfoHash"] = (
result["infoHash"] if "infoHash" in result else None
)
result["Title"] = result["title"]
result["Size"] = result["size"]
result["Link"] = (
result["downloadUrl"] if "downloadUrl" in result else None
)
result["Tracker"] = result["indexer"]
results.append(result)
except Exception as e:
logger.warning(
f"Exception while getting {indexer_manager_type} results for {query} with {indexers}: {e}"
)
pass
return results
async def get_zilean(
session: aiohttp.ClientSession, name: str, log_name: str, season: int, episode: int
):
results = []
try:
show = f"&season={season}&episode={episode}"
get_dmm = await session.get(
f"{settings.ZILEAN_URL}/dmm/filtered?query={name}{show if season else ''}"
)
get_dmm = await get_dmm.json()
if isinstance(get_dmm, list):
take_first = get_dmm[: settings.ZILEAN_TAKE_FIRST]
for result in take_first:
object = {
"Title": result["raw_title"],
"InfoHash": result["info_hash"],
"Size": int(result["size"]),
"Tracker": "DMM",
}
results.append(object)
logger.info(f"{len(results)} torrents found for {log_name} with Zilean")
except Exception as e:
logger.warning(
f"Exception while getting torrents for {log_name} with Zilean: {e}"
)
pass
return results
async def get_torrentio(log_name: str, type: str, full_id: str):
results = []
try:
try:
get_torrentio = requests.get(
f"https://torrentio.strem.fun/stream/{type}/{full_id}.json"
).json()
except:
get_torrentio = requests.get(
f"https://torrentio.strem.fun/stream/{type}/{full_id}.json",
proxies={
"http": settings.DEBRID_PROXY_URL,
"https": settings.DEBRID_PROXY_URL,
},
).json()
for torrent in get_torrentio["streams"]:
title_full = torrent["title"]
title = title_full.split("\n")[0]
tracker = title_full.split("⚙️ ")[1].split("\n")[0]
size = size_to_bytes(title_full.split("💾 ")[1].split(" ⚙️")[0])
results.append(
{
"Title": title,
"InfoHash": torrent["infoHash"],
"Size": size,
"Tracker": f"Torrentio|{tracker}",
}
)
logger.info(f"{len(results)} torrents found for {log_name} with Torrentio")
except Exception as e:
logger.warning(
f"Exception while getting torrents for {log_name} with Torrentio, your IP is most likely blacklisted (you should try proxying Comet): {e}"
)
pass
return results
async def get_mediafusion(log_name: str, type: str, full_id: str):
results = []
try:
try:
get_mediafusion = requests.get(
f"{settings.MEDIAFUSION_URL}/stream/{type}/{full_id}.json"
).json()
except:
get_mediafusion = requests.get(
f"{settings.MEDIAFUSION_URL}/stream/{type}/{full_id}.json",
proxies={
"http": settings.DEBRID_PROXY_URL,
"https": settings.DEBRID_PROXY_URL,
},
).json()
for torrent in get_mediafusion["streams"]:
title_full = torrent["description"]
title = title_full.split("\n")[0].replace("📂 ", "").replace("/", "")
tracker = title_full.split("🔗 ")[1]
results.append(
{
"Title": title,
"InfoHash": torrent["infoHash"],
"Size": torrent["behaviorHints"][
"videoSize"
], # not the pack size but still useful for prowlarr userss
"Tracker": f"MediaFusion|{tracker}",
}
)
logger.info(f"{len(results)} torrents found for {log_name} with MediaFusion")
except Exception as e:
logger.warning(
f"Exception while getting torrents for {log_name} with MediaFusion, your IP is most likely blacklisted (you should try proxying Comet): {e}"
)
pass
return results
async def filter(
torrents: list,
name: str,
year: int,
year_end: int,
aliases: dict,
remove_adult_content: bool,
):
results = []
for torrent in torrents:
index = torrent[0]
title = torrent[1]
if "\n" in title: # Torrentio title parsing
title = title.split("\n")[1]
parsed = parse(title)
if remove_adult_content and parsed.adult:
results.append((index, False))
continue
if parsed.parsed_title and not title_match(
name, parsed.parsed_title, aliases=aliases
):
results.append((index, False))
continue
if year and parsed.year:
if year_end is not None:
if not (year <= parsed.year <= year_end):
results.append((index, False))
continue
else:
if year < (parsed.year - 1) or year > (parsed.year + 1):
results.append((index, False))
continue
results.append((index, True))
return results
async def get_torrent_hash(session: aiohttp.ClientSession, torrent: tuple):
index = torrent[0]
torrent = torrent[1]
if "InfoHash" in torrent and torrent["InfoHash"] is not None:
return (index, torrent["InfoHash"].lower())
url = torrent["Link"]
try:
timeout = aiohttp.ClientTimeout(total=settings.GET_TORRENT_TIMEOUT)
response = await session.get(url, allow_redirects=False, timeout=timeout)
if response.status == 200:
torrent_data = await response.read()
torrent_dict = bencodepy.decode(torrent_data)
info = bencodepy.encode(torrent_dict[b"info"])
hash = hashlib.sha1(info).hexdigest()
else:
location = response.headers.get("Location", "")
if not location:
return (index, None)
match = info_hash_pattern.search(location)
if not match:
return (index, None)
hash = match.group(1).upper()
return (index, hash.lower())
except Exception as e:
logger.warning(
f"Exception while getting torrent info hash for {torrent['indexer'] if 'indexer' in torrent else (torrent['Tracker'] if 'Tracker' in torrent else '')}|{url}: {e}"
)
return (index, None)
def get_balanced_hashes(hashes: dict, config: dict):
max_results = config["maxResults"]
max_results_per_resolution = config["maxResultsPerResolution"]
max_size = config["maxSize"]
config_resolutions = [resolution.lower() for resolution in config["resolutions"]]
include_all_resolutions = "all" in config_resolutions
remove_trash = config["removeTrash"]
languages = [language.lower() for language in config["languages"]]
include_all_languages = "all" in languages
if not include_all_languages:
config_languages = [
code
for code, name in PTT.parse.LANGUAGES_TRANSLATION_TABLE.items()
if name.lower() in languages
]
hashes_by_resolution = {}
for hash, hash_data in hashes.items():
if remove_trash and not hash_data["fetch"]:
continue
hash_info = hash_data["data"]
if max_size != 0 and hash_info["size"] > max_size:
continue
if (
not include_all_languages
and not any(lang in hash_info["languages"] for lang in config_languages)
and ("multi" not in languages if hash_info["dubbed"] else True)
and not (len(hash_info["languages"]) == 0 and "unknown" in languages)
):
continue
resolution = hash_info["resolution"]
if not include_all_resolutions and resolution not in config_resolutions:
continue
if resolution not in hashes_by_resolution:
hashes_by_resolution[resolution] = []
hashes_by_resolution[resolution].append(hash)
if config["reverseResultOrder"]:
hashes_by_resolution = {
res: lst[::-1] for res, lst in hashes_by_resolution.items()
}
total_resolutions = len(hashes_by_resolution)
if max_results == 0 and max_results_per_resolution == 0 or total_resolutions == 0:
return hashes_by_resolution
hashes_per_resolution = (
max_results // total_resolutions
if max_results > 0
else max_results_per_resolution
)
extra_hashes = max_results % total_resolutions
balanced_hashes = {}
for resolution, hash_list in hashes_by_resolution.items():
selected_count = hashes_per_resolution + (1 if extra_hashes > 0 else 0)
if max_results_per_resolution > 0:
selected_count = min(selected_count, max_results_per_resolution)
balanced_hashes[resolution] = hash_list[:selected_count]
if extra_hashes > 0:
extra_hashes -= 1
selected_total = sum(len(hashes) for hashes in balanced_hashes.values())
if selected_total < max_results:
missing_hashes = max_results - selected_total
for resolution, hash_list in hashes_by_resolution.items():
if missing_hashes <= 0:
break
current_count = len(balanced_hashes[resolution])
available_hashes = hash_list[current_count : current_count + missing_hashes]
balanced_hashes[resolution].extend(available_hashes)
missing_hashes -= len(available_hashes)
return balanced_hashes
def format_metadata(data: dict):
extras = []
if data["quality"]:
extras.append(data["quality"])
if data["hdr"]:
extras.extend(data["hdr"])
if data["codec"]:
extras.append(data["codec"])
if data["audio"]:
extras.extend(data["audio"])
if data["channels"]:
extras.extend(data["channels"])
if data["bit_depth"]:
extras.append(data["bit_depth"])
if data["network"]:
extras.append(data["network"])
if data["group"]:
extras.append(data["group"])
return "|".join(extras)
def format_title(data: dict, config: dict):
result_format = config["resultFormat"]
has_all = "All" in result_format
title = ""
if has_all or "Title" in result_format:
title += f"{data['title']}\n"
if has_all or "Metadata" in result_format:
metadata = format_metadata(data)
if metadata != "":
title += f"💿 {metadata}\n"
if has_all or "Size" in result_format:
title += f"💾 {bytes_to_size(data['size'])} "
if has_all or "Tracker" in result_format:
title += f"🔎 {data['tracker'] if 'tracker' in data else '?'}"
if has_all or "Languages" in result_format:
languages = data["languages"]
if data["dubbed"]:
languages.insert(0, "multi")
if languages:
formatted_languages = "/".join(
get_language_emoji(language) for language in languages
)
languages_str = "\n" + formatted_languages
title += f"{languages_str}"
if title == "":
# Without this, Streamio shows SD as the result, which is confusing
title = "Empty result format configuration"
return title
def get_client_ip(request: Request):
return (
request.headers["cf-connecting-ip"]
if "cf-connecting-ip" in request.headers
else request.client.host
)
async def get_aliases(session: aiohttp.ClientSession, media_type: str, media_id: str):
aliases = {}
try:
response = await session.get(
f"https://api.trakt.tv/{media_type}/{media_id}/aliases"
)
for aliase in await response.json():
country = aliase["country"]
if country not in aliases:
aliases[country] = []
aliases[country].append(aliase["title"])
except:
pass
return aliases
async def add_torrent_to_cache(
config: dict, name: str, season: int, episode: int, sorted_ranked_files: dict
):
# trace of which indexers were used when cache was created - not optimal
indexers = config["indexers"].copy()
if settings.SCRAPE_TORRENTIO:
indexers.append("torrentio")
if settings.SCRAPE_MEDIAFUSION:
indexers.append("mediafusion")
if settings.ZILEAN_URL:
indexers.append("dmm")
for indexer in indexers:
hash = f"searched-{indexer}-{name}-{season}-{episode}"
searched = copy.deepcopy(
sorted_ranked_files[list(sorted_ranked_files.keys())[0]]
)
searched["infohash"] = hash
searched["data"]["tracker"] = indexer
sorted_ranked_files[hash] = searched
values = [
{
"debridService": config["debridService"],
"info_hash": sorted_ranked_files[torrent]["infohash"],
"name": name,
"season": season,
"episode": episode,
"tracker": sorted_ranked_files[torrent]["data"]["tracker"]
.split("|")[0]
.lower(),
"data": orjson.dumps(sorted_ranked_files[torrent]).decode("utf-8"),
"timestamp": time.time(),
}
for torrent in sorted_ranked_files
]
query = f"""
INSERT {'OR IGNORE ' if settings.DATABASE_TYPE == 'sqlite' else ''}
INTO cache (debridService, info_hash, name, season, episode, tracker, data, timestamp)
VALUES (:debridService, :info_hash, :name, :season, :episode, :tracker, :data, :timestamp)
{' ON CONFLICT DO NOTHING' if settings.DATABASE_TYPE == 'postgresql' else ''}
"""
await database.execute_many(query, values)
comet/utils/logger.py
+41 -34
View File
@@ -1,34 +1,41 @@
import sys import sys
import logging
from loguru import logger
from loguru import logger
def setupLogger(level: str): logging.getLogger("demagnetize").setLevel(
logger.level("COMET", no=50, icon="🌠", color="<fg #7871d6>") logging.CRITICAL
logger.level("API", no=40, icon="👾", color="<fg #7871d6>") ) # disable demagnetize logging
logger.level("INFO", icon="📰", color="<fg #FC5F39>")
logger.level("DEBUG", icon="🕸️", color="<fg #DC5F00>") def setupLogger(level: str):
logger.level("WARNING", icon="⚠️", color="<fg #DC5F00>") logger.level("COMET", no=50, icon="🌠", color="<fg #7871d6>")
logger.level("API", no=45, icon="👾", color="<fg #006989>")
log_format = ( logger.level("SCRAPER", no=40, icon="👻", color="<fg #d6bb71>")
"<white>{time:YYYY-MM-DD}</white> <magenta>{time:HH:mm:ss}</magenta> | " logger.level("STREAM", no=35, icon="🎬", color="<fg #d171d6>")
"<level>{level.icon}</level> <level>{level}</level> | "
"<cyan>{module}</cyan>.<cyan>{function}</cyan> - <level>{message}</level>" logger.level("INFO", icon="📰", color="<fg #FC5F39>")
) logger.level("DEBUG", icon="🕸️", color="<fg #DC5F00>")
logger.level("WARNING", icon="⚠️", color="<fg #DC5F00>")
logger.configure(
handlers=[ log_format = (
{ "<white>{time:YYYY-MM-DD}</white> <magenta>{time:HH:mm:ss}</magenta> | "
"sink": sys.stderr, "<level>{level.icon}</level> <level>{level}</level> | "
"level": level, "<cyan>{module}</cyan>.<cyan>{function}</cyan> - <level>{message}</level>"
"format": log_format, )
"backtrace": False,
"diagnose": False, logger.configure(
"enqueue": True, handlers=[
} {
] "sink": sys.stderr,
) "level": level,
"format": log_format,
"backtrace": False,
setupLogger("DEBUG") "diagnose": False,
"enqueue": True,
}
]
)
setupLogger("DEBUG")
comet/utils/models.py
+541 -219
View File
@@ -1,27 +1,46 @@
import os
import random import random
import string import string
import RTN
from typing import List, Optional from typing import List, Optional
from databases import Database from databases import Database
from pydantic import BaseModel, field_validator from pydantic import BaseModel, field_validator
from pydantic_settings import BaseSettings, SettingsConfigDict from pydantic_settings import BaseSettings, SettingsConfigDict
from RTN import RTN, BestRanking, SettingsModel from RTN import BestRanking, SettingsModel
from RTN.models import (
ResolutionConfig,
OptionsConfig,
LanguagesConfig,
CustomRanksConfig,
CustomRank,
QualityRankModel,
RipsRankModel,
HdrRankModel,
AudioRankModel,
ExtrasRankModel,
)
class AppSettings(BaseSettings): class AppSettings(BaseSettings):
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8") model_config = SettingsConfigDict(
env_file=".env", env_file_encoding="utf-8", extra="ignore"
)
ADDON_ID: Optional[str] = "stremio.comet.fast" ADDON_ID: Optional[str] = "stremio.comet.fast"
ADDON_NAME: Optional[str] = "Comet" ADDON_NAME: Optional[str] = "Comet"
FASTAPI_HOST: Optional[str] = "0.0.0.0" FASTAPI_HOST: Optional[str] = "0.0.0.0"
FASTAPI_PORT: Optional[int] = 8000 FASTAPI_PORT: Optional[int] = 8000
FASTAPI_WORKERS: Optional[int] = 2 * (os.cpu_count() or 1) FASTAPI_WORKERS: Optional[int] = 1
DASHBOARD_ADMIN_PASSWORD: Optional[str] = None USE_GUNICORN: Optional[bool] = True
DASHBOARD_ADMIN_PASSWORD: Optional[str] = "".join(
random.choices(string.ascii_letters + string.digits, k=16)
)
DATABASE_TYPE: Optional[str] = "sqlite" DATABASE_TYPE: Optional[str] = "sqlite"
DATABASE_URL: Optional[str] = "username:password@hostname:port" DATABASE_URL: Optional[str] = "username:password@hostname:port"
DATABASE_PATH: Optional[str] = "data/comet.db" DATABASE_PATH: Optional[str] = "data/comet.db"
CACHE_TTL: Optional[int] = 86400 METADATA_CACHE_TTL: Optional[int] = 2592000 # 30 days
TORRENT_CACHE_TTL: Optional[int] = 1296000 # 15 days
DEBRID_CACHE_TTL: Optional[int] = 86400 # 1 day
DEBRID_PROXY_URL: Optional[str] = None DEBRID_PROXY_URL: Optional[str] = None
INDEXER_MANAGER_TYPE: Optional[str] = None INDEXER_MANAGER_TYPE: Optional[str] = None
INDEXER_MANAGER_URL: Optional[str] = "http://127.0.0.1:9117" INDEXER_MANAGER_URL: Optional[str] = "http://127.0.0.1:9117"
@@ -29,86 +48,498 @@ class AppSettings(BaseSettings):
INDEXER_MANAGER_TIMEOUT: Optional[int] = 30 INDEXER_MANAGER_TIMEOUT: Optional[int] = 30
INDEXER_MANAGER_INDEXERS: List[str] = [] INDEXER_MANAGER_INDEXERS: List[str] = []
GET_TORRENT_TIMEOUT: Optional[int] = 5 GET_TORRENT_TIMEOUT: Optional[int] = 5
ZILEAN_URL: Optional[str] = None DOWNLOAD_TORRENT_FILES: Optional[bool] = False
ZILEAN_TAKE_FIRST: Optional[int] = 500 SCRAPE_MEDIAFUSION: Optional[bool] = False
SCRAPE_ZILEAN: Optional[bool] = False
ZILEAN_URL: Optional[str] = "https://zilean.elfhosted.com"
SCRAPE_TORRENTIO: Optional[bool] = False SCRAPE_TORRENTIO: Optional[bool] = False
TORRENTIO_URL: Optional[str] = "https://torrentio.strem.fun"
SCRAPE_MEDIAFUSION: Optional[bool] = False SCRAPE_MEDIAFUSION: Optional[bool] = False
MEDIAFUSION_URL: Optional[str] = "https://mediafusion.elfhosted.com" MEDIAFUSION_URL: Optional[str] = "https://mediafusion.elfhosted.com"
CUSTOM_HEADER_HTML: Optional[str] = None CUSTOM_HEADER_HTML: Optional[str] = None
PROXY_DEBRID_STREAM: Optional[bool] = False PROXY_DEBRID_STREAM: Optional[bool] = False
PROXY_DEBRID_STREAM_PASSWORD: Optional[str] = None PROXY_DEBRID_STREAM_PASSWORD: Optional[str] = "".join(
random.choices(string.ascii_letters + string.digits, k=16)
)
PROXY_DEBRID_STREAM_MAX_CONNECTIONS: Optional[int] = -1 PROXY_DEBRID_STREAM_MAX_CONNECTIONS: Optional[int] = -1
PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE: Optional[str] = "realdebrid" PROXY_DEBRID_STREAM_DEBRID_DEFAULT_SERVICE: Optional[str] = "realdebrid"
PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY: Optional[str] = None PROXY_DEBRID_STREAM_DEBRID_DEFAULT_APIKEY: Optional[str] = None
TITLE_MATCH_CHECK: Optional[bool] = True STREMTHRU_URL: Optional[str] = "https://stremthru.13377001.xyz"
REMOVE_ADULT_CONTENT: Optional[bool] = False REMOVE_ADULT_CONTENT: Optional[bool] = False
STREMTHRU_DEFAULT_URL: Optional[str] = "https://stremthru.mooo.com"
STREMTHRU_AUTO_ENABLED_DEBRID_SERVICES: List[str] = [
"realdebrid",
"alldebrid",
"debridlink",
]
@field_validator("DASHBOARD_ADMIN_PASSWORD") @field_validator(
def set_dashboard_admin_password(cls, v, values): "INDEXER_MANAGER_URL",
if v is None: "ZILEAN_URL",
return "".join(random.choices(string.ascii_letters + string.digits, k=16)) "TORRENTIO_URL",
"MEDIAFUSION_URL",
"STREMTHRU_URL",
)
def remove_trailing_slash(cls, v):
if v and v.endswith("/"):
return v[:-1]
return v return v
@field_validator("INDEXER_MANAGER_TYPE") @field_validator("INDEXER_MANAGER_TYPE")
def set_indexer_manager_type(cls, v, values): def set_indexer_manager_type(cls, v, values):
if v == "None": if v is not None and v.lower() == "none":
return None return None
return v return v
@field_validator("PROXY_DEBRID_STREAM_PASSWORD") @field_validator("INDEXER_MANAGER_INDEXERS")
def set_debrid_stream_proxy_password(cls, v, values): def indexer_manager_indexers_normalization(cls, v, values):
if v is None: v = [indexer.replace(" ", "").lower() for indexer in v]
return "".join(random.choices(string.ascii_letters + string.digits, k=16))
return v return v
settings = AppSettings() settings = AppSettings()
class CometSettingsModel(SettingsModel):
model_config = SettingsConfigDict()
resolutions: ResolutionConfig = ResolutionConfig(
r2160p=True, r480p=True, r360p=True
)
options: OptionsConfig = OptionsConfig(remove_ranks_under=-10000000000)
languages: LanguagesConfig = LanguagesConfig(exclude=[])
custom_ranks: CustomRanksConfig = CustomRanksConfig(
quality=QualityRankModel(
av1=CustomRank(fetch=True),
dvd=CustomRank(fetch=True),
mpeg=CustomRank(fetch=True),
remux=CustomRank(fetch=True),
vhs=CustomRank(fetch=True),
webmux=CustomRank(fetch=True),
xvid=CustomRank(fetch=True),
),
rips=RipsRankModel(
bdrip=CustomRank(fetch=True),
dvdrip=CustomRank(fetch=True),
ppvrip=CustomRank(fetch=True),
satrip=CustomRank(fetch=True),
tvrip=CustomRank(fetch=True),
uhdrip=CustomRank(fetch=True),
vhsrip=CustomRank(fetch=True),
webdlrip=CustomRank(fetch=True),
),
hdr=HdrRankModel(
dolby_vision=CustomRank(fetch=True),
),
audio=AudioRankModel(
mono=CustomRank(fetch=True),
mp3=CustomRank(fetch=True),
),
extras=ExtrasRankModel(
three_d=CustomRank(fetch=True),
converted=CustomRank(fetch=True),
documentary=CustomRank(fetch=True),
site=CustomRank(fetch=True),
upscaled=CustomRank(fetch=True),
),
)
rtn_settings_default = CometSettingsModel()
rtn_settings_default_dumped = rtn_settings_default.model_dump()
# {
# "profile":"default",
# "require":[
# ],
# "exclude":[
# ],
# "preferred":[
# ],
# "resolutions":{
# "r2160p":true,
# "r1080p":true,
# "r720p":true,
# "r480p":true,
# "r360p":true,
# "unknown":true
# },
# "options":{
# "title_similarity":0.85,
# "remove_all_trash":true,
# "remove_ranks_under":-10000000000,
# "remove_unknown_languages":false,
# "allow_english_in_languages":false,
# "enable_fetch_speed_mode":true,
# "remove_adult_content":true
# },
# "languages":{
# "required":[
# ],
# "exclude":[
# ],
# "preferred":[
# ]
# },
# "custom_ranks":{
# "quality":{
# "av1":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "avc":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "bluray":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "dvd":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "hdtv":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "hevc":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "mpeg":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "remux":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "vhs":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "web":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "webdl":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "webmux":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "xvid":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# }
# },
# "rips":{
# "bdrip":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "brrip":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "dvdrip":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "hdrip":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "ppvrip":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "satrip":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "tvrip":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "uhdrip":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "vhsrip":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "webdlrip":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "webrip":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# }
# },
# "hdr":{
# "bit10":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "dolby_vision":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "hdr":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "hdr10plus":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "sdr":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# }
# },
# "audio":{
# "aac":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "ac3":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "atmos":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "dolby_digital":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "dolby_digital_plus":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "dts_lossy":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "dts_lossless":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "eac3":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "flac":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "mono":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "mp3":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "stereo":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "surround":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "truehd":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# }
# },
# "extras":{
# "three_d":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "converted":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "documentary":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "dubbed":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "edition":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "hardcoded":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "network":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "proper":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "repack":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "retail":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "site":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "subbed":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "upscaled":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# },
# "scene":{
# "fetch":true,
# "use_custom_rank":false,
# "rank":0
# }
# },
# "trash":{
# "cam":{
# "fetch":false,
# "use_custom_rank":false,
# "rank":0
# },
# "clean_audio":{
# "fetch":false,
# "use_custom_rank":false,
# "rank":0
# },
# "pdtv":{
# "fetch":false,
# "use_custom_rank":false,
# "rank":0
# },
# "r5":{
# "fetch":false,
# "use_custom_rank":false,
# "rank":0
# },
# "screener":{
# "fetch":false,
# "use_custom_rank":false,
# "rank":0
# },
# "size":{
# "fetch":false,
# "use_custom_rank":false,
# "rank":0
# },
# "telecine":{
# "fetch":false,
# "use_custom_rank":false,
# "rank":0
# },
# "telesync":{
# "fetch":false,
# "use_custom_rank":false,
# "rank":0
# }
# }
# }
# }
rtn_ranking_default = BestRanking()
class ConfigModel(BaseModel): class ConfigModel(BaseModel):
indexers: List[str] cachedOnly: Optional[bool] = False
languages: Optional[List[str]] = ["All"]
resolutions: Optional[List[str]] = ["All"]
reverseResultOrder: Optional[bool] = False
removeTrash: Optional[bool] = True removeTrash: Optional[bool] = True
resultFormat: Optional[List[str]] = ["All"] resultFormat: Optional[List[str]] = ["all"]
maxResults: Optional[int] = 0
maxResultsPerResolution: Optional[int] = 0 maxResultsPerResolution: Optional[int] = 0
maxSize: Optional[float] = 0 maxSize: Optional[float] = 0
debridService: str debridService: Optional[str] = "torrent"
debridApiKey: str debridApiKey: Optional[str] = ""
debridStreamProxyPassword: Optional[str] = "" debridStreamProxyPassword: Optional[str] = ""
stremthruUrl: Optional[str] = None languages: Optional[dict] = rtn_settings_default_dumped["languages"]
resolutions: Optional[dict] = rtn_settings_default_dumped["resolutions"]
@field_validator("indexers") options: Optional[dict] = rtn_settings_default_dumped["options"]
def check_indexers(cls, v, values): rtnSettings: Optional[CometSettingsModel] = rtn_settings_default
settings.INDEXER_MANAGER_INDEXERS = [ rtnRanking: Optional[BestRanking] = rtn_ranking_default
indexer.replace(" ", "_").lower()
for indexer in settings.INDEXER_MANAGER_INDEXERS
] # to equal webui
valid_indexers = [
indexer for indexer in v if indexer in settings.INDEXER_MANAGER_INDEXERS
]
# if not valid_indexers: # For only Zilean mode
# raise ValueError(
# f"At least one indexer must be from {settings.INDEXER_MANAGER_INDEXERS}"
# )
return valid_indexers
@field_validator("maxResults")
def check_max_results(cls, v):
if not isinstance(v, int):
v = 0
if v < 0:
v = 0
return v
@field_validator("maxResultsPerResolution") @field_validator("maxResultsPerResolution")
def check_max_results_per_resolution(cls, v): def check_max_results_per_resolution(cls, v):
@@ -130,132 +561,35 @@ class ConfigModel(BaseModel):
@field_validator("debridService") @field_validator("debridService")
def check_debrid_service(cls, v): def check_debrid_service(cls, v):
if v not in ["realdebrid", "alldebrid", "premiumize", "torbox", "debridlink", "stremthru"]: if v not in [
"realdebrid",
"alldebrid",
"premiumize",
"torbox",
"easydebrid",
"debridlink",
"offcloud",
"pikpak",
"torrent",
]:
raise ValueError("Invalid debridService") raise ValueError("Invalid debridService")
return v return v
default_settings = { default_config = ConfigModel().model_dump()
"profile": "default", default_config["rtnSettings"] = rtn_settings_default
"require": [], default_config["rtnRanking"] = rtn_ranking_default
"exclude": [],
"preferred": [],
"resolutions": {
"r2160p": True,
"r1080p": True,
"r720p": True,
"r480p": True,
"r360p": True,
"unknown": True,
},
"options": {
"title_similarity": 0.85,
"remove_all_trash": True,
"remove_ranks_under": -1000000000000000,
"remove_unknown_languages": False,
"allow_english_in_languages": True,
"enable_fetch_speed_mode": True,
"remove_adult_content": settings.REMOVE_ADULT_CONTENT,
},
"languages": {
"required": [],
"exclude": [
# "ar",
# "hi",
# "fr",
# "es",
# "de",
# "ru",
# "pt",
# "it"
],
"preferred": [],
},
"custom_ranks": {
"quality": {
"av1": {"fetch": True, "use_custom_rank": False, "rank": 0},
"avc": {"fetch": True, "use_custom_rank": False, "rank": 0},
"bluray": {"fetch": True, "use_custom_rank": False, "rank": 0},
"dvd": {"fetch": True, "use_custom_rank": False, "rank": 0},
"hdtv": {"fetch": True, "use_custom_rank": False, "rank": 0},
"hevc": {"fetch": True, "use_custom_rank": False, "rank": 0},
"mpeg": {"fetch": True, "use_custom_rank": False, "rank": 0},
"remux": {"fetch": True, "use_custom_rank": False, "rank": 0},
"vhs": {"fetch": True, "use_custom_rank": False, "rank": 0},
"web": {"fetch": True, "use_custom_rank": False, "rank": 0},
"webdl": {"fetch": True, "use_custom_rank": False, "rank": 0},
"webmux": {"fetch": True, "use_custom_rank": False, "rank": 0},
"xvid": {"fetch": True, "use_custom_rank": False, "rank": 0},
},
"rips": {
"bdrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
"brrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
"dvdrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
"hdrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
"ppvrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
"satrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
"tvrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
"uhdrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
"vhsrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
"webdlrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
"webrip": {"fetch": True, "use_custom_rank": False, "rank": 0},
},
"hdr": {
"bit10": {"fetch": True, "use_custom_rank": False, "rank": 0},
"dolby_vision": {"fetch": True, "use_custom_rank": False, "rank": 0},
"hdr": {"fetch": True, "use_custom_rank": False, "rank": 0},
"hdr10plus": {"fetch": True, "use_custom_rank": False, "rank": 0},
"sdr": {"fetch": True, "use_custom_rank": False, "rank": 0},
},
"audio": {
"aac": {"fetch": True, "use_custom_rank": False, "rank": 0},
"ac3": {"fetch": True, "use_custom_rank": False, "rank": 0},
"atmos": {"fetch": True, "use_custom_rank": False, "rank": 0},
"dolby_digital": {"fetch": True, "use_custom_rank": False, "rank": 0},
"dolby_digital_plus": {"fetch": True, "use_custom_rank": False, "rank": 0},
"dts_lossy": {"fetch": True, "use_custom_rank": False, "rank": 0},
"dts_lossless": {"fetch": True, "use_custom_rank": False, "rank": 0},
"eac3": {"fetch": True, "use_custom_rank": False, "rank": 0},
"flac": {"fetch": True, "use_custom_rank": False, "rank": 0},
"mono": {"fetch": True, "use_custom_rank": False, "rank": 0},
"mp3": {"fetch": True, "use_custom_rank": False, "rank": 0},
"stereo": {"fetch": True, "use_custom_rank": False, "rank": 0},
"surround": {"fetch": True, "use_custom_rank": False, "rank": 0},
"Truehd": {"fetch": True, "use_custom_rank": False, "rank": 0},
},
"extras": {
"three_d": {"fetch": True, "use_custom_rank": False, "rank": 0},
"converted": {"fetch": True, "use_custom_rank": False, "rank": 0},
"documentary": {"fetch": True, "use_custom_rank": False, "rank": 0},
"dubbed": {"fetch": True, "use_custom_rank": False, "rank": 0},
"edition": {"fetch": True, "use_custom_rank": False, "rank": 0},
"hardcoded": {"fetch": True, "use_custom_rank": False, "rank": 0},
"network": {"fetch": True, "use_custom_rank": False, "rank": 0},
"proper": {"fetch": True, "use_custom_rank": False, "rank": 0},
"repack": {"fetch": True, "use_custom_rank": False, "rank": 0},
"retail": {"fetch": True, "use_custom_rank": False, "rank": 0},
"site": {"fetch": True, "use_custom_rank": False, "rank": 0},
"subbed": {"fetch": True, "use_custom_rank": False, "rank": 0},
"upscaled": {"fetch": True, "use_custom_rank": False, "rank": 0},
"scene": {"fetch": True, "use_custom_rank": False, "rank": 0},
},
"trash": {
"cam": {"fetch": False, "use_custom_rank": False, "rank": 0},
"clean_audio": {"fetch": False, "use_custom_rank": False, "rank": 0},
"pdtv": {"fetch": False, "use_custom_rank": False, "rank": 0},
"r5": {"fetch": False, "use_custom_rank": False, "rank": 0},
"screener": {"fetch": False, "use_custom_rank": False, "rank": 0},
"size": {"fetch": False, "use_custom_rank": False, "rank": 0},
"telecine": {"fetch": False, "use_custom_rank": False, "rank": 0},
"telesync": {"fetch": False, "use_custom_rank": False, "rank": 0},
},
},
}
rtn_settings = SettingsModel(**default_settings)
rtn_ranking = BestRanking()
# For use anywhere
rtn = RTN(settings=rtn_settings, ranking_model=rtn_ranking) # Web Config Initialization
# languages = [language for language in PTT.parse.LANGUAGES_TRANSLATION_TABLE.values()]
# languages.insert(0, "Unknown")
# languages.insert(1, "Multi")
web_config = {
# "languages": languages,
"resolutions": [resolution.value for resolution in RTN.models.Resolution],
"resultFormat": ["title", "metadata", "seeders", "size", "tracker", "languages"],
}
database_url = ( database_url = (
settings.DATABASE_PATH settings.DATABASE_PATH
@@ -267,44 +601,32 @@ database = Database(
) )
trackers = [ trackers = [
"tracker:https://tracker.gbitt.info:443/announce", "udp://tracker-udp.gbitt.info:80/announce",
"tracker:udp://discord.heihachi.pw:6969/announce", "udp://tracker.0x7c0.com:6969/announce",
"tracker:http://tracker.corpscorp.online:80/announce", "udp://opentracker.io:6969/announce",
"tracker:udp://tracker.leechers-paradise.org:6969/announce", "udp://leet-tracker.moe:1337/announce",
"tracker:https://tracker.renfei.net:443/announce", "udp://tracker.torrent.eu.org:451/announce",
"tracker:udp://exodus.desync.com:6969/announce", "udp://tracker.tiny-vps.com:6969/announce",
"tracker:http://tracker.xiaoduola.xyz:6969/announce", "udp://tracker.leechers-paradise.org:6969/announce",
"tracker:udp://ipv4.tracker.harry.lu:80/announce", "udp://tracker.pomf.se:80/announce",
"tracker:udp://tracker.torrent.eu.org:451/announce", "udp://9.rarbg.me:2710/announce",
"tracker:udp://tracker.coppersurfer.tk:6969/announce", "http://tracker.gbitt.info:80/announce",
"tracker:http://tracker.dmcomic.org:2710/announce", "udp://tracker.bittor.pw:1337/announce",
"tracker:http://www.genesis-sp.org:2710/announce", "udp://open.free-tracker.ga:6969/announce",
"tracker:http://t.jaekr.sh:6969/announce", "udp://open.stealth.si:80/announce",
"tracker:http://tracker.bt-hash.com:80/announce", "udp://retracker01-msk-virt.corbina.net:80/announce",
"tracker:https://tracker.tamersunion.org:443/announce", "udp://tracker.openbittorrent.com:80/announce",
"tracker:udp://open.stealth.si:80/announce", "udp://tracker.opentrackr.org:1337/announce",
"tracker:udp://tracker.opentrackr.org:1337/announce", "udp://isk.richardsw.club:6969/announce",
"tracker:udp://leet-tracker.moe:1337/announce", "https://tracker.gbitt.info:443/announce",
"tracker:udp://oh.fuuuuuck.com:6969/announce", "udp://tracker.coppersurfer.tk:6969/announce",
"tracker:udp://tracker.bittor.pw:1337/announce", "udp://oh.fuuuuuck.com:6969/announce",
"tracker:udp://explodie.org:6969/announce", "udp://ipv4.tracker.harry.lu:80/announce",
"tracker:http://finbytes.org:80/announce.php", "udp://open.demonii.com:1337/announce",
"tracker:udp://tracker.dump.cl:6969/announce", "https://tracker.tamersunion.org:443/announce",
"tracker:udp://open.free-tracker.ga:6969/announce", "https://tracker.renfei.net:443/announce",
"tracker:http://tracker.gbitt.info:80/announce", "udp://open.tracker.cl:1337/announce",
"tracker:udp://isk.richardsw.club:6969/announce", "udp://tracker.internetwarriors.net:1337/announce",
"tracker:http://bt1.xxxxbt.cc:6969/announce", "udp://exodus.desync.com:6969/announce",
"tracker:udp://tracker.qu.ax:6969/announce", "udp://tracker.dump.cl:6969/announce",
"tracker:udp://opentracker.io:6969/announce",
"tracker:udp://tracker.internetwarriors.net:1337/announce",
"tracker:udp://tracker.0x7c0.com:6969/announce",
"tracker:udp://9.rarbg.me:2710/announce",
"tracker:udp://tracker.pomf.se:80/announce",
"tracker:udp://tracker.openbittorrent.com:80/announce",
"tracker:udp://open.tracker.cl:1337/announce",
"tracker:http://www.torrentsnipe.info:2701/announce",
"tracker:udp://retracker01-msk-virt.corbina.net:80/announce",
"tracker:udp://open.demonii.com:1337/announce",
"tracker:udp://tracker-udp.gbitt.info:80/announce",
"tracker:udp://tracker.tiny-vps.com:6969/announce",
] ]
comet/utils/streaming.py
+95
View File
@@ -0,0 +1,95 @@
import time
import uuid
from starlette.background import BackgroundTask
from fastapi.responses import FileResponse
from comet.utils.models import settings, database
from comet.utils.logger import logger
import mediaflow_proxy.handlers
import mediaflow_proxy.utils.http_utils
async def on_stream_end(connection_id: str, ip: str):
try:
await database.execute(
"DELETE FROM active_connections WHERE id = :connection_id AND ip = :ip",
{"connection_id": connection_id, "ip": ip},
)
logger.log(
"STREAM", f"Stream ended - Connection: {connection_id} from IP: {ip}"
)
except Exception as e:
logger.warning(
f"Error handling stream end for connection {connection_id} from IP {ip}: {e}"
)
async def check_ip_connections(ip: str):
if settings.PROXY_DEBRID_STREAM_MAX_CONNECTIONS <= -1:
return True
try:
count = await database.fetch_val(
"SELECT COUNT(*) FROM active_connections WHERE ip = :ip",
{"ip": ip},
)
if count >= settings.PROXY_DEBRID_STREAM_MAX_CONNECTIONS:
logger.log(
"STREAM",
f"Connection limit reached for IP: {ip} ({count} active connections)",
)
return False
return True
except Exception as e:
logger.warning(f"Error checking IP connections for {ip}: {e}")
return False
async def add_active_connection(media_id: str, ip: str):
connection_id = str(uuid.uuid4())
await database.execute(
"INSERT INTO active_connections (id, ip, content, timestamp) VALUES (:connection_id, :ip, :content, :timestamp)",
{
"connection_id": connection_id,
"ip": ip,
"content": media_id,
"timestamp": time.time(),
},
)
logger.log(
"STREAM",
f"New stream connection - ID: {connection_id}, IP: {ip}, Content: {media_id}",
)
return connection_id
async def combined_background_tasks(
connection_id: str, ip: str, streamer_close_task: BackgroundTask
):
await streamer_close_task()
await on_stream_end(connection_id, ip)
async def custom_handle_stream_request(
method: str,
video_url: str,
proxy_headers: mediaflow_proxy.utils.http_utils.ProxyRequestHeaders,
media_id: str,
ip: str,
):
if not await check_ip_connections(ip):
return FileResponse("comet/assets/proxylimit.mp4")
connection_id = await add_active_connection(media_id, ip)
response = await mediaflow_proxy.handlers.handle_stream_request(
method, video_url, proxy_headers
)
original_background_task = response.background
response.background = BackgroundTask(
combined_background_tasks,
connection_id=connection_id,
ip=ip,
streamer_close_task=original_background_task,
)
return response
comet/utils/torrent.py
+490
View File
@@ -0,0 +1,490 @@
import hashlib
import re
import bencodepy
import aiohttp
import anyio
import asyncio
import orjson
import time
from urllib.parse import parse_qs, urlparse
from demagnetize.core import Demagnetizer
from torf import Magnet
from RTN import ParsedData, parse
from comet.utils.logger import logger
from comet.utils.models import settings, database
from comet.utils.general import is_video, default_dump
info_hash_pattern = re.compile(r"btih:([a-fA-F0-9]{40})")
def extract_trackers_from_magnet(magnet_uri: str):
try:
parsed = urlparse(magnet_uri)
params = parse_qs(parsed.query)
return params.get("tr", [])
except Exception as e:
logger.warning(f"Failed to extract trackers from magnet URI: {e}")
return []
async def download_torrent(session: aiohttp.ClientSession, url: str):
try:
timeout = aiohttp.ClientTimeout(total=settings.GET_TORRENT_TIMEOUT)
async with session.get(url, allow_redirects=False, timeout=timeout) as response:
if response.status == 200:
return (await response.read(), None, None)
location = response.headers.get("Location", "")
if location:
match = info_hash_pattern.search(location)
if match:
return (None, match.group(1), location)
return (None, None, None)
except Exception as e:
logger.warning(
f"Failed to download torrent from {url}: {e} (in most cases, you can ignore this error)"
)
return (None, None, None)
demagnetizer = Demagnetizer()
async def get_torrent_from_magnet(magnet_uri: str):
try:
magnet = Magnet.from_string(magnet_uri)
with anyio.fail_after(60):
torrent_data = await demagnetizer.demagnetize(magnet)
if torrent_data:
return torrent_data.dump()
except Exception as e:
logger.warning(f"Failed to get torrent from magnet: {e}")
return None
def extract_torrent_metadata(content: bytes):
try:
torrent_data = bencodepy.decode(content)
info = torrent_data[b"info"]
info_encoded = bencodepy.encode(info)
m = hashlib.sha1()
m.update(info_encoded)
info_hash = m.hexdigest()
announce_list = [
tracker[0].decode() for tracker in torrent_data.get(b"announce-list", [])
]
metadata = {"info_hash": info_hash, "announce_list": announce_list, "files": []}
files = info[b"files"] if b"files" in info else [info]
for idx, file in enumerate(files):
name = (
file[b"path"][-1].decode()
if b"path" in file
else file[b"name"].decode()
)
if not is_video(name) or "sample" in name.lower():
continue
size = file[b"length"]
metadata["files"].append({"index": idx, "name": name, "size": size})
return metadata
except Exception as e:
logger.warning(f"Failed to extract torrent metadata: {e}")
return {}
async def add_torrent(
info_hash: str,
seeders: int,
tracker: str,
media_id: str,
search_season: int,
sources: list,
file_index: int,
title: str,
size: int,
parsed: ParsedData,
):
try:
parsed_season = parsed.seasons[0] if parsed.seasons else search_season
parsed_episode = parsed.episodes[0] if parsed.episodes else None
if parsed_episode is not None:
await database.execute(
"""
DELETE FROM torrents
WHERE info_hash = :info_hash
AND season = :season
AND episode IS NULL
""",
{
"info_hash": info_hash,
"season": parsed_season,
},
)
logger.log(
"SCRAPER",
f"Deleted season-only entry for S{parsed_season:02d} of {info_hash}",
)
await database.execute(
f"""
INSERT {"OR IGNORE " if settings.DATABASE_TYPE == "sqlite" else ""}
INTO torrents
VALUES (:media_id, :info_hash, :file_index, :season, :episode, :title, :seeders, :size, :tracker, :sources, :parsed, :timestamp)
{" ON CONFLICT DO NOTHING" if settings.DATABASE_TYPE == "postgresql" else ""}
""",
{
"media_id": media_id,
"info_hash": info_hash,
"file_index": file_index,
"season": parsed_season,
"episode": parsed_episode,
"title": title,
"seeders": seeders,
"size": size,
"tracker": tracker,
"sources": orjson.dumps(sources).decode("utf-8"),
"parsed": orjson.dumps(parsed, default_dump).decode("utf-8"),
"timestamp": time.time(),
},
)
additional = ""
if parsed_season:
additional += f" - S{parsed_season:02d}"
additional += f"E{parsed_episode:02d}" if parsed_episode else ""
logger.log("SCRAPER", f"Added torrent for {media_id} - {title}{additional}")
except Exception as e:
logger.warning(f"Failed to add torrent for {info_hash}: {e}")
class AddTorrentQueue:
def __init__(self, max_concurrent: int = 10):
self.queue = asyncio.Queue()
self.max_concurrent = max_concurrent
self.is_running = False
self.semaphore = asyncio.Semaphore(max_concurrent)
async def add_torrent(
self,
magnet_url: str,
seeders: int,
tracker: str,
media_id: str,
search_season: int,
):
if not settings.DOWNLOAD_TORRENT_FILES:
return
await self.queue.put((magnet_url, seeders, tracker, media_id, search_season))
if not self.is_running:
self.is_running = True
asyncio.create_task(self._process_queue())
async def _process_queue(self):
while self.is_running:
try:
(
magnet_url,
seeders,
tracker,
media_id,
search_season,
) = await self.queue.get()
async with self.semaphore:
try:
content = await get_torrent_from_magnet(magnet_url)
if content:
metadata = extract_torrent_metadata(content)
for file in metadata["files"]:
parsed = parse(file["name"])
await add_torrent(
metadata["info_hash"],
seeders,
tracker,
media_id,
search_season,
metadata["announce_list"],
file["index"],
file["name"],
file["size"],
parsed,
)
finally:
self.queue.task_done()
except Exception:
await asyncio.sleep(1)
self.is_running = False
add_torrent_queue = AddTorrentQueue()
class TorrentUpdateQueue:
def __init__(self, batch_size: int = 100, flush_interval: float = 5.0):
self.queue = asyncio.Queue()
self.batch_size = batch_size
self.flush_interval = flush_interval
self.is_running = False
self.batches = {"to_check": [], "to_delete": [], "inserts": [], "updates": []}
self.last_error_time = 0
self.error_backoff = 1
async def add_torrent_info(self, file_info: dict, media_id: str = None):
await self.queue.put((file_info, media_id))
if not self.is_running:
self.is_running = True
asyncio.create_task(self._process_queue())
async def _process_queue(self):
last_flush_time = time.time()
while self.is_running:
try:
while not self.queue.empty():
try:
file_info, media_id = self.queue.get_nowait()
await self._process_file_info(file_info, media_id)
except asyncio.QueueEmpty:
break
current_time = time.time()
if (
current_time - last_flush_time >= self.flush_interval
or self.queue.empty()
) and any(len(batch) > 0 for batch in self.batches.values()):
await self._flush_batch()
last_flush_time = current_time
if self.queue.empty() and not any(
len(batch) > 0 for batch in self.batches.values()
):
self.is_running = False
break
await asyncio.sleep(0.1)
except asyncio.CancelledError:
break
except Exception as e:
logger.warning(f"Error in _process_queue: {e}")
await self._handle_error(e)
if any(len(batch) > 0 for batch in self.batches.values()):
await self._flush_batch()
self.is_running = False
async def _flush_batch(self):
try:
if self.batches["to_check"]:
sub_batch_size = 100
for i in range(0, len(self.batches["to_check"]), sub_batch_size):
sub_batch = self.batches["to_check"][i : i + sub_batch_size]
placeholders = []
params = {}
for idx, item in enumerate(sub_batch):
key = str(idx)
placeholders.append(
f"(CAST(:info_hash_{key} AS TEXT) = info_hash AND "
f"(CAST(:season_{key} AS INTEGER) IS NULL AND season IS NULL OR season = CAST(:season_{key} AS INTEGER)) AND "
f"(CAST(:episode_{key} AS INTEGER) IS NULL AND episode IS NULL OR episode = CAST(:episode_{key} AS INTEGER)))"
)
params[f"info_hash_{key}"] = item["info_hash"]
params[f"season_{key}"] = item["season"]
params[f"episode_{key}"] = item["episode"]
query = f"""
SELECT info_hash, season, episode
FROM torrents
WHERE {" OR ".join(placeholders)}
"""
async with database.transaction():
existing_rows = await database.fetch_all(query, params)
existing_set = {
(
row["info_hash"],
row["season"] if row["season"] is not None else None,
row["episode"] if row["episode"] is not None else None,
)
for row in existing_rows
}
for item in sub_batch:
key = (item["info_hash"], item["season"], item["episode"])
if key in existing_set:
self.batches["updates"].append(item["params"])
else:
self.batches["inserts"].append(item["params"])
self.batches["to_check"] = []
if self.batches["to_delete"]:
sub_batch_size = 100
for i in range(0, len(self.batches["to_delete"]), sub_batch_size):
sub_batch = self.batches["to_delete"][i : i + sub_batch_size]
placeholders = []
params = {}
for idx, item in enumerate(sub_batch):
key_suffix = f"_{idx}"
placeholders.append(
f"(CAST(:info_hash{key_suffix} AS TEXT), CAST(:season{key_suffix} AS INTEGER))"
)
params[f"info_hash{key_suffix}"] = item["info_hash"]
params[f"season{key_suffix}"] = item["season"]
async with database.transaction():
delete_query = f"""
DELETE FROM torrents
WHERE (info_hash, season) IN (
{",".join(placeholders)}
)
AND episode IS NULL
"""
await database.execute(delete_query, params)
self.batches["to_delete"] = []
if self.batches["inserts"]:
sub_batch_size = 100
for i in range(0, len(self.batches["inserts"]), sub_batch_size):
sub_batch = self.batches["inserts"][i : i + sub_batch_size]
async with database.transaction():
insert_query = f"""
INSERT {"OR IGNORE " if settings.DATABASE_TYPE == "sqlite" else ""}
INTO torrents
VALUES (
:media_id,
:info_hash,
:file_index,
:season,
:episode,
:title,
:seeders,
:size,
:tracker,
:sources,
:parsed,
:timestamp
)
{" ON CONFLICT DO NOTHING" if settings.DATABASE_TYPE == "postgresql" else ""}
"""
await database.execute_many(insert_query, sub_batch)
if len(self.batches["inserts"]) > 0:
logger.log(
"SCRAPER",
f"Inserted {len(self.batches['inserts'])} new torrents in batch",
)
self.batches["inserts"] = []
if self.batches["updates"]:
sub_batch_size = 100
for i in range(0, len(self.batches["updates"]), sub_batch_size):
sub_batch = self.batches["updates"][i : i + sub_batch_size]
async with database.transaction():
update_query = """
UPDATE torrents
SET title = CAST(:title AS TEXT),
file_index = CAST(:file_index AS INTEGER),
size = CAST(:size AS BIGINT),
seeders = CAST(:seeders AS INTEGER),
tracker = CAST(:tracker AS TEXT),
sources = CAST(:sources AS TEXT),
parsed = CAST(:parsed AS TEXT),
timestamp = CAST(:timestamp AS FLOAT),
media_id = CAST(:media_id AS TEXT)
WHERE info_hash = CAST(:info_hash AS TEXT)
AND (CAST(:season AS INTEGER) IS NULL AND season IS NULL OR season = CAST(:season AS INTEGER))
AND (CAST(:episode AS INTEGER) IS NULL AND episode IS NULL OR episode = CAST(:episode AS INTEGER))
"""
await database.execute_many(update_query, sub_batch)
if len(self.batches["updates"]) > 0:
logger.log(
"SCRAPER",
f"Updated {len(self.batches['updates'])} existing torrents in batch",
)
self.batches["updates"] = []
self.error_backoff = 1
except Exception as e:
await self._handle_error(e)
async def _process_file_info(self, file_info: dict, media_id: str = None):
try:
params = {
"info_hash": file_info["info_hash"],
"file_index": file_info["index"],
"season": file_info["season"],
"episode": file_info["episode"],
"title": file_info["title"],
"seeders": file_info["seeders"],
"size": file_info["size"],
"tracker": file_info["tracker"],
"sources": orjson.dumps(file_info["sources"]).decode("utf-8"),
"parsed": orjson.dumps(
file_info["parsed"], default=default_dump
).decode("utf-8"),
"timestamp": time.time(),
"media_id": media_id,
}
self.batches["to_check"].append(
{
"info_hash": file_info["info_hash"],
"season": file_info["season"],
"episode": file_info["episode"],
"params": params,
}
)
if file_info["episode"] is not None:
self.batches["to_delete"].append(
{"info_hash": file_info["info_hash"], "season": file_info["season"]}
)
await self._check_batch_size()
finally:
self.queue.task_done()
async def _check_batch_size(self):
if any(len(batch) >= self.batch_size for batch in self.batches.values()):
await self._flush_batch()
self.error_backoff = 1
async def _handle_error(self, e: Exception):
current_time = time.time()
if current_time - self.last_error_time < 5:
self.error_backoff = min(self.error_backoff * 2, 30)
else:
self.error_backoff = 1
self.last_error_time = current_time
logger.warning(f"Database error in torrent batch processing: {e}")
logger.warning(f"Waiting {self.error_backoff} seconds before retry")
await asyncio.sleep(self.error_backoff)
torrent_update_queue = TorrentUpdateQueue()
comet/utils/trackers.py
+18
View File
@@ -0,0 +1,18 @@
import aiohttp
from comet.utils.models import trackers
from comet.utils.logger import logger
async def download_best_trackers():
try:
async with aiohttp.ClientSession() as session:
response = await session.get(
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best.txt"
)
response = await response.text()
other_trackers = [tracker for tracker in response.split("\n") if tracker]
trackers.extend(other_trackers)
except Exception as e:
logger.warning(f"Failed to download best trackers: {e}")
deployment/docker-compose.yml
+17
View File
@@ -0,0 +1,17 @@
services:
comet:
container_name: comet
image: g0ldyy/comet:rewrite
restart: unless-stopped
ports:
- "8000:8000"
env_file:
- .env
volumes:
- ./data/comet:/app/data
healthcheck:
test: wget -qO- http://127.0.0.1:8000/health
interval: 30s
timeout: 10s
retries: 3
start_period: 20s
deployment/nginx.conf
+11
View File
@@ -0,0 +1,11 @@
server {
server_name example.com;
location / {
proxy_pass http://localhost:8000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
}
pyproject.toml
+27 -37
View File
@@ -1,37 +1,27 @@
[tool.poetry] [project]
name = "comet" name = "comet"
version = "0.1.0" version = "0.1.0"
description = "Stremio's fastest torrent/debrid search add-on." description = "Stremio's fastest torrent/debrid search add-on."
authors = ["Goldy"] authors = [
license = "MIT" { name = "Goldy", email = "153996346+g0ldyy@users.noreply.github.com" }
readme = "README.md" ]
repository = "https://github.com/g0ldyy/comet" requires-python = ">=3.11"
dependencies = [
[tool.poetry.dependencies] "aiohttp",
python = "^3.11" "aiosqlite",
uvicorn = "*" "asyncio",
fastapi = "*" "asyncpg",
aiohttp = "*" "bencode-py",
asyncio = "*" "curl-cffi",
loguru = "*" "databases",
databases = "*" "demagnetize",
pydantic-settings = "*" "fastapi",
bencode-py = "*" "gunicorn",
httpx = "*" "jinja2",
curl-cffi = "*" "loguru",
orjson = "*" "mediaflow-proxy",
asyncpg = "*" "orjson",
aiosqlite = "*" "pydantic-settings",
jinja2 = "*" "rank-torrent-name",
rank-torrent-name = "*" "uvicorn",
parsett = "*" ]
[tool.poetry.group.dev.dependencies]
isort = "*"
pyright = "*"
pytest = "*"
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"