mirror of
https://github.com/g0ldyy/comet.git
synced 2026-01-12 01:16:12 +01:00
feat: refactor live torrent caching to differentiate between displaying existing results and triggering new scrapes, and update default cache TTLs
This commit is contained in:
g0ldyy
+11
-2
@@ -54,8 +54,17 @@ DATABASE_STARTUP_CLEANUP_INTERVAL=3600 # Minimum seconds between heavy startup c
|
||||
# Cache Settings (Seconds) #
|
||||
# ============================== #
|
||||
METADATA_CACHE_TTL=2592000 # 30 days
|
||||
TORRENT_CACHE_TTL=1296000 # 15 days - How long before torrents in DB start to be removed. -1 to disable.
|
||||
LIVE_TORRENT_CACHE_TTL=1296000 # 15 days - How long before Live is re-queried for a search.
|
||||
|
||||
# TORRENT_CACHE_TTL: Controls when torrents are PERMANENTLY REMOVED from the database.
|
||||
# Set to -1 to disable automatic removal (torrents stay forever).
|
||||
TORRENT_CACHE_TTL=2592000 # 30 days
|
||||
|
||||
# LIVE_TORRENT_CACHE_TTL: Controls when a NEW LIVE SEARCH is triggered.
|
||||
# If the cache is older than this value, Comet performs a new search to find new torrents.
|
||||
# Important: Old cached torrents are ALWAYS included in results, this only controls refresh frequency.
|
||||
# Set to -1 to never trigger new searches (always use existing cache only).
|
||||
LIVE_TORRENT_CACHE_TTL=604800 # 7 days
|
||||
|
||||
DEBRID_CACHE_TTL=86400 # 1 day
|
||||
DEBRID_CACHE_CHECK_RATIO=0.05 # Minimum ratio (0.05 = 5%) of cached torrents/total torrents required to skip re-checking availability on the debrid service.
|
||||
METRICS_CACHE_TTL=60 # 1 minute
|
||||
|
||||
@@ -197,27 +197,50 @@ async def stream(
|
||||
id, season if "kitsu" not in media_id else 1, episode
|
||||
)
|
||||
|
||||
# Quick check for cached torrents
|
||||
cached_torrents_count = await database.fetch_val(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM torrents
|
||||
WHERE media_id = :media_id
|
||||
AND ((season IS NOT NULL AND season = CAST(:season as INTEGER)) OR (season IS NULL AND CAST(:season as INTEGER) IS NULL))
|
||||
AND (episode IS NULL OR episode = CAST(:episode as INTEGER))
|
||||
AND timestamp + :cache_ttl >= :current_time
|
||||
""",
|
||||
{
|
||||
"media_id": id,
|
||||
"season": season,
|
||||
"episode": episode,
|
||||
"cache_ttl": settings.LIVE_TORRENT_CACHE_TTL,
|
||||
"current_time": time.time(),
|
||||
},
|
||||
)
|
||||
# Quick check for "fresh" cached torrents to decide if we need to re-scrape.
|
||||
# This does NOT filter what torrents are shown, it only determines if a new search is triggered.
|
||||
# LIVE_TORRENT_CACHE_TTL controls when cache is considered "stale" and needs refreshing.
|
||||
# If -1, cache is never considered stale (all cached torrents are "fresh").
|
||||
if settings.LIVE_TORRENT_CACHE_TTL >= 0:
|
||||
fresh_cached_count = await database.fetch_val(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM torrents
|
||||
WHERE media_id = :media_id
|
||||
AND ((season IS NOT NULL AND season = CAST(:season as INTEGER)) OR (season IS NULL AND CAST(:season as INTEGER) IS NULL))
|
||||
AND (episode IS NULL OR episode = CAST(:episode as INTEGER))
|
||||
AND timestamp + :cache_ttl >= :current_time
|
||||
""",
|
||||
{
|
||||
"media_id": id,
|
||||
"season": season,
|
||||
"episode": episode,
|
||||
"cache_ttl": settings.LIVE_TORRENT_CACHE_TTL,
|
||||
"current_time": time.time(),
|
||||
},
|
||||
)
|
||||
else:
|
||||
# TTL=-1 means cache never expires, count all cached torrents as "fresh"
|
||||
fresh_cached_count = await database.fetch_val(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM torrents
|
||||
WHERE media_id = :media_id
|
||||
AND ((season IS NOT NULL AND season = CAST(:season as INTEGER)) OR (season IS NULL AND CAST(:season as INTEGER) IS NULL))
|
||||
AND (episode IS NULL OR episode = CAST(:episode as INTEGER))
|
||||
""",
|
||||
{
|
||||
"media_id": id,
|
||||
"season": season,
|
||||
"episode": episode,
|
||||
},
|
||||
)
|
||||
|
||||
# If both metadata and torrents are cached, skip lock entirely
|
||||
if cached_metadata is not None and cached_torrents_count > 0:
|
||||
# Track if cache is stale (no fresh torrents) for background refresh decision
|
||||
cache_is_stale = fresh_cached_count == 0
|
||||
|
||||
# If both metadata and fresh torrents are cached, skip lock entirely
|
||||
if cached_metadata is not None and fresh_cached_count > 0:
|
||||
logger.log("SCRAPER", f"🚀 Fast path: using cached data for {media_id}")
|
||||
metadata, aliases = cached_metadata[0], cached_metadata[1]
|
||||
# Variables for fast path
|
||||
@@ -363,24 +386,30 @@ async def stream(
|
||||
]
|
||||
}
|
||||
|
||||
elif is_first:
|
||||
logger.log(
|
||||
"SCRAPER",
|
||||
f"🔄 Starting background scrape + availability check for {log_title}",
|
||||
)
|
||||
elif is_first or cache_is_stale:
|
||||
# Background scrape if first search OR if cache is stale (needs refresh)
|
||||
if is_first:
|
||||
logger.log(
|
||||
"SCRAPER",
|
||||
f"🔄 First search - starting background scrape for {log_title}",
|
||||
)
|
||||
cached_results.append(
|
||||
{
|
||||
"name": "[🔄] Comet",
|
||||
"description": "First search for this media - More results will be available in a few seconds...",
|
||||
"url": "https://comet.fast",
|
||||
}
|
||||
)
|
||||
else:
|
||||
logger.log(
|
||||
"SCRAPER",
|
||||
f"🔄 Cache stale - starting background refresh for {log_title}",
|
||||
)
|
||||
|
||||
background_tasks.add_task(
|
||||
background_scrape, torrent_manager, media_id, debrid_service
|
||||
)
|
||||
|
||||
cached_results.append(
|
||||
{
|
||||
"name": "[🔄] Comet",
|
||||
"description": "First search for this media - More results will be available in a few seconds...",
|
||||
"url": "https://comet.fast",
|
||||
}
|
||||
)
|
||||
|
||||
# Perform scraping if lock acquired and needed
|
||||
if needs_scraping and scrape_lock:
|
||||
try:
|
||||
|
||||
@@ -39,8 +39,8 @@ class AppSettings(BaseSettings):
|
||||
DATABASE_READ_REPLICA_URLS: List[str] = Field(default_factory=list)
|
||||
DATABASE_STARTUP_CLEANUP_INTERVAL: Optional[int] = 3600
|
||||
METADATA_CACHE_TTL: Optional[int] = 2592000 # 30 days
|
||||
TORRENT_CACHE_TTL: Optional[int] = 1296000 # 15 days
|
||||
LIVE_TORRENT_CACHE_TTL: Optional[int] = 1296000 # 15 days
|
||||
TORRENT_CACHE_TTL: Optional[int] = 2592000 # 30 days
|
||||
LIVE_TORRENT_CACHE_TTL: Optional[int] = 604800 # 7 days
|
||||
DEBRID_CACHE_TTL: Optional[int] = 86400 # 1 day
|
||||
METRICS_CACHE_TTL: Optional[int] = 60 # 1 minute
|
||||
DEBRID_CACHE_CHECK_RATIO: Optional[float] = 0.0 # 0.0 to 1.0
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
import aiohttp
|
||||
import orjson
|
||||
@@ -7,7 +6,7 @@ from RTN import DefaultRanking, ParsedData
|
||||
|
||||
from comet.core.execution import get_executor
|
||||
from comet.core.logger import logger
|
||||
from comet.core.models import CometSettingsModel, database, settings
|
||||
from comet.core.models import CometSettingsModel, database
|
||||
from comet.scrapers.manager import scraper_manager
|
||||
from comet.services.filtering import filter_worker
|
||||
from comet.services.ranking import rank_worker
|
||||
@@ -105,14 +104,11 @@ class TorrentManager:
|
||||
WHERE media_id = :media_id
|
||||
AND ((season IS NOT NULL AND season = CAST(:season as INTEGER)) OR (season IS NULL AND CAST(:season as INTEGER) IS NULL))
|
||||
AND (episode IS NULL OR episode = CAST(:episode as INTEGER))
|
||||
AND timestamp + :cache_ttl >= :current_time
|
||||
""",
|
||||
{
|
||||
"media_id": self.media_only_id,
|
||||
"season": self.season,
|
||||
"episode": self.episode,
|
||||
"cache_ttl": settings.LIVE_TORRENT_CACHE_TTL,
|
||||
"current_time": time.time(),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user