diff --git a/.env-sample b/.env-sample index 8422879..a7afc95 100644 --- a/.env-sample +++ b/.env-sample @@ -54,8 +54,17 @@ DATABASE_STARTUP_CLEANUP_INTERVAL=3600 # Minimum seconds between heavy startup c # Cache Settings (Seconds) # # ============================== # METADATA_CACHE_TTL=2592000 # 30 days -TORRENT_CACHE_TTL=1296000 # 15 days - How long before torrents in DB start to be removed. -1 to disable. -LIVE_TORRENT_CACHE_TTL=1296000 # 15 days - How long before Live is re-queried for a search. + +# TORRENT_CACHE_TTL: Controls when torrents are PERMANENTLY REMOVED from the database. +# Set to -1 to disable automatic removal (torrents stay forever). +TORRENT_CACHE_TTL=2592000 # 30 days + +# LIVE_TORRENT_CACHE_TTL: Controls when a NEW LIVE SEARCH is triggered. +# If the cache is older than this value, Comet performs a new search to find new torrents. +# Important: Old cached torrents are ALWAYS included in results, this only controls refresh frequency. +# Set to -1 to never trigger new searches (always use existing cache only). +LIVE_TORRENT_CACHE_TTL=604800 # 7 days + DEBRID_CACHE_TTL=86400 # 1 day DEBRID_CACHE_CHECK_RATIO=0.05 # Minimum ratio (0.05 = 5%) of cached torrents/total torrents required to skip re-checking availability on the debrid service. METRICS_CACHE_TTL=60 # 1 minute diff --git a/comet/api/endpoints/stream.py b/comet/api/endpoints/stream.py index 417c571..3d5f7dc 100644 --- a/comet/api/endpoints/stream.py +++ b/comet/api/endpoints/stream.py @@ -197,27 +197,50 @@ async def stream( id, season if "kitsu" not in media_id else 1, episode ) - # Quick check for cached torrents - cached_torrents_count = await database.fetch_val( - """ - SELECT COUNT(*) - FROM torrents - WHERE media_id = :media_id - AND ((season IS NOT NULL AND season = CAST(:season as INTEGER)) OR (season IS NULL AND CAST(:season as INTEGER) IS NULL)) - AND (episode IS NULL OR episode = CAST(:episode as INTEGER)) - AND timestamp + :cache_ttl >= :current_time - """, - { - "media_id": id, - "season": season, - "episode": episode, - "cache_ttl": settings.LIVE_TORRENT_CACHE_TTL, - "current_time": time.time(), - }, - ) + # Quick check for "fresh" cached torrents to decide if we need to re-scrape. + # This does NOT filter what torrents are shown, it only determines if a new search is triggered. + # LIVE_TORRENT_CACHE_TTL controls when cache is considered "stale" and needs refreshing. + # If -1, cache is never considered stale (all cached torrents are "fresh"). + if settings.LIVE_TORRENT_CACHE_TTL >= 0: + fresh_cached_count = await database.fetch_val( + """ + SELECT COUNT(*) + FROM torrents + WHERE media_id = :media_id + AND ((season IS NOT NULL AND season = CAST(:season as INTEGER)) OR (season IS NULL AND CAST(:season as INTEGER) IS NULL)) + AND (episode IS NULL OR episode = CAST(:episode as INTEGER)) + AND timestamp + :cache_ttl >= :current_time + """, + { + "media_id": id, + "season": season, + "episode": episode, + "cache_ttl": settings.LIVE_TORRENT_CACHE_TTL, + "current_time": time.time(), + }, + ) + else: + # TTL=-1 means cache never expires, count all cached torrents as "fresh" + fresh_cached_count = await database.fetch_val( + """ + SELECT COUNT(*) + FROM torrents + WHERE media_id = :media_id + AND ((season IS NOT NULL AND season = CAST(:season as INTEGER)) OR (season IS NULL AND CAST(:season as INTEGER) IS NULL)) + AND (episode IS NULL OR episode = CAST(:episode as INTEGER)) + """, + { + "media_id": id, + "season": season, + "episode": episode, + }, + ) - # If both metadata and torrents are cached, skip lock entirely - if cached_metadata is not None and cached_torrents_count > 0: + # Track if cache is stale (no fresh torrents) for background refresh decision + cache_is_stale = fresh_cached_count == 0 + + # If both metadata and fresh torrents are cached, skip lock entirely + if cached_metadata is not None and fresh_cached_count > 0: logger.log("SCRAPER", f"🚀 Fast path: using cached data for {media_id}") metadata, aliases = cached_metadata[0], cached_metadata[1] # Variables for fast path @@ -363,24 +386,30 @@ async def stream( ] } - elif is_first: - logger.log( - "SCRAPER", - f"🔄 Starting background scrape + availability check for {log_title}", - ) + elif is_first or cache_is_stale: + # Background scrape if first search OR if cache is stale (needs refresh) + if is_first: + logger.log( + "SCRAPER", + f"🔄 First search - starting background scrape for {log_title}", + ) + cached_results.append( + { + "name": "[🔄] Comet", + "description": "First search for this media - More results will be available in a few seconds...", + "url": "https://comet.fast", + } + ) + else: + logger.log( + "SCRAPER", + f"🔄 Cache stale - starting background refresh for {log_title}", + ) background_tasks.add_task( background_scrape, torrent_manager, media_id, debrid_service ) - cached_results.append( - { - "name": "[🔄] Comet", - "description": "First search for this media - More results will be available in a few seconds...", - "url": "https://comet.fast", - } - ) - # Perform scraping if lock acquired and needed if needs_scraping and scrape_lock: try: diff --git a/comet/core/models.py b/comet/core/models.py index 26dbf6e..d85dd37 100644 --- a/comet/core/models.py +++ b/comet/core/models.py @@ -39,8 +39,8 @@ class AppSettings(BaseSettings): DATABASE_READ_REPLICA_URLS: List[str] = Field(default_factory=list) DATABASE_STARTUP_CLEANUP_INTERVAL: Optional[int] = 3600 METADATA_CACHE_TTL: Optional[int] = 2592000 # 30 days - TORRENT_CACHE_TTL: Optional[int] = 1296000 # 15 days - LIVE_TORRENT_CACHE_TTL: Optional[int] = 1296000 # 15 days + TORRENT_CACHE_TTL: Optional[int] = 2592000 # 30 days + LIVE_TORRENT_CACHE_TTL: Optional[int] = 604800 # 7 days DEBRID_CACHE_TTL: Optional[int] = 86400 # 1 day METRICS_CACHE_TTL: Optional[int] = 60 # 1 minute DEBRID_CACHE_CHECK_RATIO: Optional[float] = 0.0 # 0.0 to 1.0 diff --git a/comet/services/orchestration.py b/comet/services/orchestration.py index 9edb1fe..7999ec0 100644 --- a/comet/services/orchestration.py +++ b/comet/services/orchestration.py @@ -1,5 +1,4 @@ import asyncio -import time import aiohttp import orjson @@ -7,7 +6,7 @@ from RTN import DefaultRanking, ParsedData from comet.core.execution import get_executor from comet.core.logger import logger -from comet.core.models import CometSettingsModel, database, settings +from comet.core.models import CometSettingsModel, database from comet.scrapers.manager import scraper_manager from comet.services.filtering import filter_worker from comet.services.ranking import rank_worker @@ -105,14 +104,11 @@ class TorrentManager: WHERE media_id = :media_id AND ((season IS NOT NULL AND season = CAST(:season as INTEGER)) OR (season IS NULL AND CAST(:season as INTEGER) IS NULL)) AND (episode IS NULL OR episode = CAST(:episode as INTEGER)) - AND timestamp + :cache_ttl >= :current_time """, { "media_id": self.media_only_id, "season": self.season, "episode": self.episode, - "cache_ttl": settings.LIVE_TORRENT_CACHE_TTL, - "current_time": time.time(), }, )