g0ldyy/comet
1
0
Fork 0
mirror of https://github.com/g0ldyy/comet.git synced 2026-01-12 01:16:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #473 from g0ldyy/development

Browse Source 
Consolidate HTTP cache settings and restructure web UI
This commit is contained in:
Goldy
2026-01-11 16:03:26 +01:00
committed by GitHub
parent 73e383e42a c6f0790d68
commit f40a1b3a64
13 changed files with 1285 additions and 955 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env-sample
+12 -9
View File
@@ -277,16 +277,19 @@ STREMTHRU_URL=https://stremthru.13377001.xyz # needed to use debrid services
HTTP_CACHE_ENABLED=False # Master switch for HTTP cache headers
# PUBLIC streams TTL (for /stream/movie/tt1234567.json without user config)
# These can be cached at edge and shared between all users.
# Higher = less origin traffic, lower = fresher results
HTTP_CACHE_PUBLIC_STREAMS_TTL=300 # 5 minutes (recommended: 120-600)
# PRIVATE streams TTL (for /{config}/stream/... with user config)
# Only cached in user's browser (private cache).
# These contain user-specific debrid availability info.
HTTP_CACHE_PRIVATE_STREAMS_TTL=60 # 1 minute (recommended: 30-120)
# STREAMS streams TTL (for /stream/movie/tt1234567.json)
# These results are cached at edge and shared between all users.
# Higher = less traffic to origin, Lower = faster updates for new content
HTTP_CACHE_STREAMS_TTL=300 # 5 minutes
# Stale-While-Revalidate: serve stale content while fetching fresh in background
# Improves perceived performance - users get instant response with cached data
HTTP_CACHE_STALE_WHILE_REVALIDATE=60 # 1 minute
# MANIFEST TTL (for /manifest.json and /{config}/manifest.json)
# Manifest rarely changes, can be cached longer
HTTP_CACHE_MANIFEST_TTL=86400 # 24 hours (recommended: 3600-86400)
# CONFIGURE page TTL (for /configure and /{config}/configure)
# Static page, can be cached for a long time
HTTP_CACHE_CONFIGURE_TTL=86400 # 24 hours (recommended: 3600-86400)
comet/api/endpoints/chilllink.py
+1 -3
View File
@@ -1,5 +1,3 @@
import random
import string
from typing import Optional
from fastapi import APIRouter, BackgroundTasks, Query, Request
@@ -28,7 +26,7 @@ async def chilllink_manifest(request: Request, b64config: str = None):
config = config_check(b64config)
manifest = {
"id": f"{settings.ADDON_ID}.{''.join(random.choice(string.ascii_letters) for _ in range(4))}",
"id": settings.ADDON_ID,
"version": "2.0.0",
"description": "Chillio's fastest debrid search add-on.",
"supported_endpoints": {"feeds": None, "streams": "/streams"},
comet/api/endpoints/manifest.py
+2 -7
View File
@@ -1,6 +1,3 @@
import random
import string
from fastapi import APIRouter, Request
from comet.core.config_validation import config_check
@@ -27,7 +24,7 @@ router = APIRouter()
)
async def manifest(request: Request, b64config: str = None):
base_manifest = {
"id": f"{settings.ADDON_ID}.{''.join(random.choice(string.ascii_letters) for _ in range(4))}",
"id": settings.ADDON_ID,
"description": "Stremio's fastest torrent/debrid search add-on.",
"version": "2.0.0",
"catalogs": [],
@@ -58,9 +55,7 @@ async def manifest(request: Request, b64config: str = None):
)
if settings.HTTP_CACHE_ENABLED:
base_manifest_etag_data = base_manifest.copy()
base_manifest_etag_data.pop("id", None)
etag = generate_etag(base_manifest_etag_data)
etag = generate_etag(base_manifest)
if check_etag_match(request, etag):
return not_modified_response(etag)
comet/api/endpoints/stream.py
+56 -52
View File
@@ -30,7 +30,7 @@ streams = APIRouter()
def _build_stream_response(
request: Request,
content: dict,
is_public: bool = False,
is_empty: bool = False,
vary_headers: list = None,
):
if not settings.HTTP_CACHE_ENABLED:
@@ -41,12 +41,12 @@ def _build_stream_response(
if check_etag_match(request, etag):
return not_modified_response(etag)
if is_public:
cache_policy = CachePolicies.public_torrents()
if is_empty:
cache_policy = CachePolicies.empty_results()
vary = ["Accept", "Accept-Encoding"]
else:
cache_policy = CachePolicies.private_streams()
vary = ["Accept", "Accept-Encoding", "Authorization"]
cache_policy = CachePolicies.streams()
vary = ["Accept", "Accept-Encoding"]
if vary_headers:
vary.extend(vary_headers)
@@ -173,17 +173,11 @@ async def stream(
b64config: str = None,
chilllink: bool = False,
):
is_public_request = b64config is None
if media_type not in ["movie", "series"]:
return _build_stream_response(
request, {"streams": []}, is_public=is_public_request
)
return _build_stream_response(request, {"streams": []}, is_empty=True)
if "tmdb:" in media_id:
return _build_stream_response(
request, {"streams": []}, is_public=is_public_request
)
return _build_stream_response(request, {"streams": []}, is_empty=True)
media_id = media_id.replace("imdb_id:", "")
@@ -198,7 +192,7 @@ async def stream(
}
]
}
return error_response
return _build_stream_response(request, error_response, is_empty=True)
is_torrent = config["debridService"] == "torrent"
if settings.DISABLE_TORRENT_STREAMS and is_torrent:
@@ -209,9 +203,7 @@ async def stream(
if settings.TORRENT_DISABLED_STREAM_URL:
placeholder_stream["url"] = settings.TORRENT_DISABLED_STREAM_URL
return _build_stream_response(
request, {"streams": [placeholder_stream]}, is_public=is_public_request
)
return _build_stream_response(request, {"streams": [placeholder_stream]})
connector = aiohttp.TCPConnector(limit=0)
async with aiohttp.ClientSession(connector=connector) as session:
@@ -238,7 +230,7 @@ async def stream(
}
]
},
is_public=is_public_request,
is_empty=True,
)
# Check if metadata is already cached
@@ -335,15 +327,19 @@ async def stream(
if lock_acquired and scrape_lock:
await scrape_lock.release()
logger.log("SCRAPER", f"❌ Failed to fetch metadata for {media_id}")
return {
"streams": [
{
"name": "[⚠️] Comet",
"description": "Unable to get metadata.",
"url": "https://comet.feels.legal",
}
]
}
return _build_stream_response(
request,
{
"streams": [
{
"name": "[⚠️] Comet",
"description": "Unable to get metadata.",
"url": "https://comet.feels.legal",
}
]
},
is_empty=True,
)
title = metadata["title"]
year = metadata["year"]
@@ -449,15 +445,19 @@ async def stream(
)
if len(torrent_manager.torrents) == 0:
return {
"streams": [
{
"name": "[🔄] Comet",
"description": "Scraping in progress by another instance, please try again in a few seconds...",
"url": "https://comet.feels.legal",
}
]
}
return _build_stream_response(
request,
{
"streams": [
{
"name": "[🔄] Comet",
"description": "Scraping in progress by another instance, please try again in a few seconds...",
"url": "https://comet.feels.legal",
}
]
},
is_empty=True,
)
elif is_first or cache_is_stale:
# Background scrape if first search OR if cache is stale (needs refresh)
@@ -527,15 +527,19 @@ async def stream(
episode,
)
except DebridAuthError as e:
return {
"streams": [
{
"name": "[❌] Comet",
"description": e.display_message,
"url": "https://comet.feels.legal",
}
]
}
return _build_stream_response(
request,
{
"streams": [
{
"name": "[❌] Comet",
"description": e.display_message,
"url": "https://comet.feels.legal",
}
]
},
is_empty=True,
)
if debrid_service != "torrent":
cached_count = sum(
@@ -641,14 +645,14 @@ async def stream(
non_cached_results.append(the_stream)
if sort_mixed:
return _build_stream_response(
request,
{"streams": cached_results},
is_public=is_public_request,
)
final_streams = cached_results
else:
final_streams = cached_results + non_cached_results
has_results = len(final_streams) > 0
return _build_stream_response(
request,
{"streams": cached_results + non_cached_results},
is_public=is_public_request,
{"streams": final_streams},
is_empty=not has_results,
)
comet/core/logger.py
+1 -1
View File
@@ -436,7 +436,7 @@ def log_startup_info(settings):
logger.log("COMET", f"Custom Header HTML: {bool(settings.CUSTOM_HEADER_HTML)}")
http_cache_info = (
f" - Public Streams TTL: {settings.HTTP_CACHE_PUBLIC_STREAMS_TTL}s - Private Streams TTL: {settings.HTTP_CACHE_PRIVATE_STREAMS_TTL}s - Stale While Revalidate: {settings.HTTP_CACHE_STALE_WHILE_REVALIDATE}s"
f" - Streams TTL: {settings.HTTP_CACHE_STREAMS_TTL}s - Manifest TTL: {settings.HTTP_CACHE_MANIFEST_TTL}s - Configure TTL: {settings.HTTP_CACHE_CONFIGURE_TTL}s - SWR: {settings.HTTP_CACHE_STALE_WHILE_REVALIDATE}s"
if settings.HTTP_CACHE_ENABLED
else ""
)
comet/core/models.py
+3 -2
View File
@@ -138,9 +138,10 @@ class AppSettings(BaseSettings):
RATELIMIT_RETRY_BASE_DELAY: Optional[float] = 1.0
RTN_FILTER_DEBUG: Optional[bool] = False
HTTP_CACHE_ENABLED: Optional[bool] = False
HTTP_CACHE_PUBLIC_STREAMS_TTL: Optional[int] = 300
HTTP_CACHE_PRIVATE_STREAMS_TTL: Optional[int] = 60
HTTP_CACHE_STREAMS_TTL: Optional[int] = 300
HTTP_CACHE_STALE_WHILE_REVALIDATE: Optional[int] = 60
HTTP_CACHE_MANIFEST_TTL: Optional[int] = 86400
HTTP_CACHE_CONFIGURE_TTL: Optional[int] = 86400
@field_validator("INDEXER_MANAGER_TYPE")
def set_indexer_manager_type(cls, v, values):
comet/services/filtering.py
+28 -13
View File
@@ -17,13 +17,28 @@ def quick_alias_match(text_normalized: str, ez_aliases_normalized: list[str]):
return any(alias in text_normalized for alias in ez_aliases_normalized)
def filter_worker(torrents, title, year, year_end, aliases, remove_adult_content):
def filter_worker(
torrents, title, year, year_end, media_type, aliases, remove_adult_content
):
results = []
ez_aliases = aliases.get("ez", [])
if ez_aliases:
ez_aliases_normalized = [normalize_title(a) for a in ez_aliases]
min_year = 0
max_year = float("inf")
if year:
if year_end:
min_year = year
max_year = year_end
elif media_type == "series":
min_year = year - 1
else:
min_year = year - 1
max_year = year + 1
for torrent in torrents:
torrent_title = torrent["title"]
torrent_title_lower = torrent_title.lower()
@@ -53,18 +68,18 @@ def filter_worker(torrents, title, year, year_end, aliases, remove_adult_content
continue
if year and parsed.year:
if year_end is not None:
if not (year <= parsed.year <= year_end):
_log_exclusion(
f"📅 Rejected (Year Mismatch) | {torrent_title} | Year: {parsed.year} | Expected: {year}-{year_end}"
)
continue
else:
if year < (parsed.year - 1) or year > (parsed.year + 1):
_log_exclusion(
f"📅 Rejected (Year Mismatch) | {torrent_title} | Year: {parsed.year} | Expected: ~{year}"
)
continue
if not (min_year <= parsed.year <= max_year):
if year_end:
expected = f"{year}-{year_end}"
elif media_type == "series":
expected = f">{year}"
else:
expected = f"~{year}"
_log_exclusion(
f"📅 Rejected (Year Mismatch) | {torrent_title} | Year: {parsed.year} | Expected: {expected}"
)
continue
torrent["parsed"] = parsed
results.append(torrent)
comet/services/orchestration.py
+1
View File
@@ -225,6 +225,7 @@ class TorrentManager:
self.title,
self.year,
self.year_end,
self.media_type,
self.aliases,
self.remove_adult_content,
)
comet/templates/admin_dashboard.html
+54 -31
View File
@@ -1,4 +1,4 @@
<!DOCTYPE html>
<!doctype html>
<html class="sl-theme-dark">
<head>
<meta charset="UTF-8" />
@@ -44,9 +44,19 @@
#25292c 0%,
#0c0d13 100%
);
font-family: system-ui, -apple-system, "Segoe UI", Roboto,
"Helvetica Neue", "Noto Sans", "Liberation Sans", Arial, sans-serif,
"Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol",
font-family:
system-ui,
-apple-system,
"Segoe UI",
Roboto,
"Helvetica Neue",
"Noto Sans",
"Liberation Sans",
Arial,
sans-serif,
"Apple Color Emoji",
"Segoe UI Emoji",
"Segoe UI Symbol",
"Noto Color Emoji";
font-size: 1rem;
font-weight: 400;
@@ -308,7 +318,8 @@
border-radius: 50%;
filter: drop-shadow(0 0 6px currentColor);
transform: translate3d(104em, 0, 0);
animation: fall var(--fall-duration) var(--fall-delay) linear infinite,
animation:
fall var(--fall-duration) var(--fall-delay) linear infinite,
tail-fade var(--tail-fade-duration) var(--fall-delay) ease-out
infinite;
}
@@ -780,7 +791,7 @@
star.style.setProperty("--top-offset", `${randomTopOffset}vh`);
star.style.setProperty(
"--star-tail-length",
`${randomTailLength}em`
`${randomTailLength}em`,
);
star.style.setProperty("--fall-duration", `${randomFallDuration}s`);
star.style.setProperty("--fall-delay", "0s");
@@ -956,9 +967,21 @@
<h3><sl-icon name="download"></sl-icon> Torrents Analysis</h3>
<div class="metric-content">
<div class="metric-subsection">
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 12px;">
<h4 style="margin: 0;">Top Trackers</h4>
<sl-select id="tracker-limit" size="small" value="5" style="width: 110px;">
<div
style="
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 12px;
"
>
<h4 style="margin: 0">Top Trackers</h4>
<sl-select
id="tracker-limit"
size="small"
value="5"
style="width: 110px"
>
<sl-option value="5">Top 5</sl-option>
<sl-option value="10">Top 10</sl-option>
<sl-option value="20">Top 20</sl-option>
@@ -1133,7 +1156,7 @@
// Logs refresh based on selected interval
const interval = parseInt(
document.getElementById("refresh-interval").value
document.getElementById("refresh-interval").value,
);
logsInterval = setInterval(loadLogs, interval);
}
@@ -1151,7 +1174,7 @@
if (document.getElementById("auto-refresh").checked) {
clearInterval(logsInterval);
const interval = parseInt(
document.getElementById("refresh-interval").value
document.getElementById("refresh-interval").value,
);
logsInterval = setInterval(loadLogs, interval);
}
@@ -1189,7 +1212,7 @@
connections.length > 0
? Math.round(
connections.reduce((sum, c) => sum + c.duration, 0) /
connections.length
connections.length,
)
: 0;
@@ -1239,7 +1262,7 @@
<td>
<span class="connection-id">${conn.id.substring(
0,
8
8,
)}...</span>
</td>
<td>
@@ -1247,14 +1270,14 @@
</td>
<td>
<span title="${escapeHtml(
conn.content
conn.content,
)}">${
conn.content.length > 25
? escapeHtml(
conn.content.substring(0, 25)
) + "..."
: escapeHtml(conn.content)
}</span>
conn.content.length > 25
? escapeHtml(
conn.content.substring(0, 25),
) + "..."
: escapeHtml(conn.content)
}</span>
</td>
<td>
<span style="font-size: 0.875rem; color: #9ca3af;">${
@@ -1263,7 +1286,7 @@
</td>
<td>
<span class="duration-tag">${Math.round(
conn.duration
conn.duration,
)}s</span>
</td>
<td>
@@ -1284,7 +1307,7 @@
}</span>
</td>
</tr>
`
`,
)
.join("")}
</tbody>
@@ -1297,7 +1320,7 @@
async function loadLogs() {
try {
const response = await fetch(
`/admin/api/logs?since=${lastLogTimestamp}`
`/admin/api/logs?since=${lastLogTimestamp}`,
);
const data = await response.json();
@@ -1341,7 +1364,7 @@
} ${log.level}</span> <span class="log-module">${log.module}.${
log.function
}</span> - <span class="log-message">${escapeHtml(
log.message
log.message,
)}</span>`;
container.appendChild(logEntry);
@@ -1388,7 +1411,7 @@
const hideApiLogs = document.getElementById("hide-api-logs").checked;
const container = document.getElementById("logs-container");
const apiLogEntries = container.querySelectorAll(
'[data-api-log="true"]'
'[data-api-log="true"]',
);
apiLogEntries.forEach((entry) => {
@@ -1434,7 +1457,7 @@
const trackerContainer = document.getElementById("tracker-stats");
const limitSelect = document.getElementById("tracker-limit");
const limit = limitSelect ? limitSelect.value : "5";
let displayTrackers = torrentsData.by_tracker;
if (limit !== "all") {
displayTrackers = displayTrackers.slice(0, parseInt(limit));
@@ -1449,7 +1472,7 @@
(tracker) => `
<div class="tracker-item">
<div class="tracker-name">${escapeHtml(
tracker.tracker || "Unknown"
tracker.tracker || "Unknown",
)}</div>
<div class="tracker-stats">
<span class="tracker-count">${tracker.count.toLocaleString()} torrents</span>
@@ -1461,7 +1484,7 @@
} avg size</span>
</div>
</div>
`
`,
)
.join("");
}
@@ -1552,7 +1575,7 @@
function showMetricsError() {
const sections = document.querySelectorAll(
".metric-section .loading-state"
".metric-section .loading-state",
);
sections.forEach((section) => {
section.innerHTML =
@@ -1576,7 +1599,7 @@
(service) => `
<div class="debrid-service-item">
<div class="debrid-service-name">${escapeHtml(
service.service
service.service,
)}</div>
<div class="debrid-service-stats">
<span class="service-count">${service.count.toLocaleString()} files</span>
@@ -1588,7 +1611,7 @@
}</span>
</div>
</div>
`
`,
)
.join("");
}
comet/templates/admin_login.html
+17 -6
View File
@@ -1,4 +1,4 @@
<!DOCTYPE html>
<!doctype html>
<html class="sl-theme-dark">
<head>
<meta charset="UTF-8" />
@@ -44,9 +44,19 @@
#25292c 0%,
#0c0d13 100%
);
font-family: system-ui, -apple-system, "Segoe UI", Roboto,
"Helvetica Neue", "Noto Sans", "Liberation Sans", Arial, sans-serif,
"Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol",
font-family:
system-ui,
-apple-system,
"Segoe UI",
Roboto,
"Helvetica Neue",
"Noto Sans",
"Liberation Sans",
Arial,
sans-serif,
"Apple Color Emoji",
"Segoe UI Emoji",
"Segoe UI Symbol",
"Noto Color Emoji";
font-size: 1rem;
font-weight: 400;
@@ -125,7 +135,8 @@
border-radius: 50%;
filter: drop-shadow(0 0 6px currentColor);
transform: translate3d(104em, 0, 0);
animation: fall var(--fall-duration) var(--fall-delay) linear infinite,
animation:
fall var(--fall-duration) var(--fall-delay) linear infinite,
tail-fade var(--tail-fade-duration) var(--fall-delay) ease-out
infinite;
}
@@ -235,7 +246,7 @@
star.style.setProperty("--top-offset", `${randomTopOffset}vh`);
star.style.setProperty(
"--star-tail-length",
`${randomTailLength}em`
`${randomTailLength}em`,
);
star.style.setProperty("--fall-duration", `${randomFallDuration}s`);
star.style.setProperty("--fall-delay", "0s");
comet/templates/index.html
+1071 -808
View File
File diff suppressed because it is too large Load Diff
comet/utils/cache.py
+25 -20
View File
@@ -154,13 +154,13 @@ def not_modified_response(etag: str):
class CachePolicies:
@staticmethod
def public_torrents():
def streams():
"""
For public torrent lists (without user config).
For all stream results.
Cache for a short time at CDN, revalidate often.
"""
ttl = settings.HTTP_CACHE_PUBLIC_STREAMS_TTL
ttl = settings.HTTP_CACHE_STREAMS_TTL
swr = settings.HTTP_CACHE_STALE_WHILE_REVALIDATE
return (
@@ -172,36 +172,41 @@ class CachePolicies:
.stale_if_error(300)
)
@staticmethod
def private_streams():
"""
For user-specific stream results (with b64config).
Private cache only, short TTL.
"""
ttl = settings.HTTP_CACHE_PRIVATE_STREAMS_TTL
return CacheControl().private().max_age(ttl).must_revalidate()
@staticmethod
def manifest():
"""
For manifest.json responses.
Very short cache as it can change based on config.
Long cache as manifest rarely changes.
"""
return CacheControl().private().max_age(60).must_revalidate()
ttl = settings.HTTP_CACHE_MANIFEST_TTL
swr = settings.HTTP_CACHE_STALE_WHILE_REVALIDATE
return CacheControl().public().max_age(ttl).stale_while_revalidate(swr)
@staticmethod
def configure_page():
"""
For the /configure page.
Cacheable if no custom HTML, otherwise private.
Long cache as the page is mostly static.
"""
ttl = settings.HTTP_CACHE_CONFIGURE_TTL
swr = settings.HTTP_CACHE_STALE_WHILE_REVALIDATE
if settings.CUSTOM_HEADER_HTML:
return CacheControl().private().max_age(300)
return CacheControl().public().max_age(ttl).stale_while_revalidate(swr)
return CacheControl().public().max_age(300).s_maxage(3600)
@staticmethod
def empty_results():
"""
For empty/temporary responses (no torrents found, processing, errors).
Short public cache to prevent spam while allowing quick retries.
"""
return (
CacheControl()
.public()
.max_age(15) # Browser cache 15 seconds
.s_maxage(30) # CDN cache 30 seconds
.stale_if_error(60) # Serve stale on error for 1 minute
)
@staticmethod
def no_cache():
deployment/cloudflare-cache-rules.md
+14 -3
View File
@@ -3,7 +3,7 @@
To optimize performance, configure Cloudflare to offload traffic from your server. We use **Cache Rules** to cache responses while respecting the application's cache headers.
## 1. Streams (Cache Rule)
Cache all stream results (public and private). Comet controls the TTL via headers.
Cache all stream results. Comet controls the TTL via headers.
* **Rule Name**: Streams
* **Expression**: `(http.request.uri.path contains "/stream/")`
@@ -20,12 +20,23 @@ Cache the configuration page.
* **Action**: Eligible for Cache
* **Edge TTL**: Use cache-control header if present (first option)
* **Browser TTL**: Respect origin
* **Serve stale content while revalidating**: On
## 3. Tiered Cache
## 3. Manifest (Cache Rule)
Cache the add-on manifest.
* **Rule Name**: Manifest
* **Expression**: `(http.request.uri.path contains "/manifest.json")`
* **Action**: Eligible for Cache
* **Edge TTL**: Use cache-control header if present (first option)
* **Browser TTL**: Respect origin
* **Serve stale content while revalidating**: On
## 4. Tiered Cache
Enable **Tiered Cache** in **Caching > Tiered Cache**.
This minimizes requests to your origin by checking other Cloudflare datacenters first.
## 4. Network Optimizations
## 5. Network Optimizations
In **Speed > Protocol**:
* **HTTP/3 (QUIC)**: On (faster connections, especially on mobile)