diff --git a/.env-sample b/.env-sample index 533908e..5aff92d 100644 --- a/.env-sample +++ b/.env-sample @@ -40,7 +40,8 @@ DATABASE_BATCH_SIZE=20000 # The batch size for the database import and export op # Cache Settings (Seconds) # # ============================== # METADATA_CACHE_TTL=2592000 # 30 days -TORRENT_CACHE_TTL=1296000 # 15 days +TORRENT_CACHE_TTL=1296000 # 15 days - How long before torrents in DB start to be removed. -1 to disable. +LIVE_TORRENT_CACHE_TTL=1296000 # 15 days - How long before Live is re-queried for a search. DEBRID_CACHE_TTL=86400 # 1 day METRICS_CACHE_TTL=60 # 1 minute SCRAPE_LOCK_TTL=300 # 5 minutes - Duration for distributed scraping locks diff --git a/comet/api/endpoints/admin.py b/comet/api/endpoints/admin.py index 7df0206..95301af 100644 --- a/comet/api/endpoints/admin.py +++ b/comet/api/endpoints/admin.py @@ -223,25 +223,20 @@ async def admin_api_metrics(admin_session: str = Cookie(None)): # Torrents by tracker top_trackers = await database.fetch_all(""" - SELECT tracker, COUNT(*) as count + SELECT tracker, COUNT(*) as count, AVG(seeders) as avg_seeders, AVG(size) as avg_size FROM torrents GROUP BY tracker ORDER BY count DESC - LIMIT 5 """) tracker_stats = [] for row in top_trackers: - stats = await database.fetch_one( - "SELECT AVG(seeders) as avg_seeders, AVG(size) as avg_size FROM torrents WHERE tracker = :tracker", - {"tracker": row["tracker"]}, - ) tracker_stats.append( { "tracker": row["tracker"], "count": row["count"], - "avg_seeders": stats["avg_seeders"], - "avg_size": stats["avg_size"], + "avg_seeders": row["avg_seeders"], + "avg_size": row["avg_size"], } ) diff --git a/comet/api/endpoints/stream.py b/comet/api/endpoints/stream.py index 5193164..d91e9ef 100644 --- a/comet/api/endpoints/stream.py +++ b/comet/api/endpoints/stream.py @@ -109,6 +109,8 @@ async def stream( if "tmdb:" in media_id: return {"streams": []} + media_id = media_id.replace("imdb_id:", "") + config = config_check(b64config) if not config: return { @@ -145,7 +147,7 @@ async def stream( "media_id": id, "season": season, "episode": episode, - "cache_ttl": settings.TORRENT_CACHE_TTL, + "cache_ttl": settings.LIVE_TORRENT_CACHE_TTL, "current_time": time.time(), }, ) diff --git a/comet/core/database.py b/comet/core/database.py index 84475cf..a2eefad 100644 --- a/comet/core/database.py +++ b/comet/core/database.py @@ -610,13 +610,14 @@ async def setup_database(): {"cache_ttl": settings.METADATA_CACHE_TTL, "current_time": time.time()}, ) - await database.execute( - """ - DELETE FROM torrents - WHERE timestamp + :cache_ttl < :current_time; - """, - {"cache_ttl": settings.TORRENT_CACHE_TTL, "current_time": time.time()}, - ) + if settings.TORRENT_CACHE_TTL >= 0: + await database.execute( + """ + DELETE FROM torrents + WHERE timestamp + :cache_ttl < :current_time; + """, + {"cache_ttl": settings.TORRENT_CACHE_TTL, "current_time": time.time()}, + ) await database.execute( """ @@ -630,6 +631,8 @@ async def setup_database(): await database.execute("DELETE FROM active_connections") + await database.execute("DELETE FROM metrics_cache") + except Exception as e: logger.error(f"Error setting up the database: {e}") logger.exception(traceback.format_exc()) diff --git a/comet/core/logger.py b/comet/core/logger.py index 5b0a35e..58e7b5f 100644 --- a/comet/core/logger.py +++ b/comet/core/logger.py @@ -180,7 +180,7 @@ def log_startup_info(settings): ) logger.log( "COMET", - f"Database ({settings.DATABASE_TYPE}): {settings.DATABASE_PATH if settings.DATABASE_TYPE == 'sqlite' else settings.DATABASE_URL} - TTL: metadata={settings.METADATA_CACHE_TTL}s, torrents={settings.TORRENT_CACHE_TTL}s, debrid={settings.DEBRID_CACHE_TTL}s, metrics={settings.METRICS_CACHE_TTL}s", + f"Database ({settings.DATABASE_TYPE}): {settings.DATABASE_PATH if settings.DATABASE_TYPE == 'sqlite' else settings.DATABASE_URL} - TTL: metadata={settings.METADATA_CACHE_TTL}s, torrents={settings.TORRENT_CACHE_TTL}s, live_torrents={settings.LIVE_TORRENT_CACHE_TTL}s, debrid={settings.DEBRID_CACHE_TTL}s, metrics={settings.METRICS_CACHE_TTL}s", ) logger.log("COMET", f"Bypass Proxy: {settings.BYPASS_PROXY_URL}") diff --git a/comet/core/models.py b/comet/core/models.py index dae52f8..9036b4b 100644 --- a/comet/core/models.py +++ b/comet/core/models.py @@ -34,6 +34,7 @@ class AppSettings(BaseSettings): DATABASE_BATCH_SIZE: Optional[int] = 20000 METADATA_CACHE_TTL: Optional[int] = 2592000 # 30 days TORRENT_CACHE_TTL: Optional[int] = 1296000 # 15 days + LIVE_TORRENT_CACHE_TTL: Optional[int] = 1296000 # 15 days DEBRID_CACHE_TTL: Optional[int] = 86400 # 1 day METRICS_CACHE_TTL: Optional[int] = 60 # 1 minute SCRAPE_LOCK_TTL: Optional[int] = 300 # 5 minutes diff --git a/comet/services/orchestration.py b/comet/services/orchestration.py index 00317af..ba57094 100644 --- a/comet/services/orchestration.py +++ b/comet/services/orchestration.py @@ -110,7 +110,7 @@ class TorrentManager: "media_id": self.media_only_id, "season": self.season, "episode": self.episode, - "cache_ttl": settings.TORRENT_CACHE_TTL, + "cache_ttl": settings.LIVE_TORRENT_CACHE_TTL, "current_time": time.time(), }, ) @@ -156,7 +156,7 @@ class TorrentManager: ] query = f""" - INSERT {"OR IGNORE " if settings.DATABASE_TYPE == "sqlite" else ""} + INSERT {"OR REPLACE " if settings.DATABASE_TYPE == "sqlite" else ""} INTO torrents VALUES (:media_id, :info_hash, :file_index, :season, :episode, :title, :seeders, :size, :tracker, :sources, :parsed, :timestamp) {" ON CONFLICT DO NOTHING" if settings.DATABASE_TYPE == "postgresql" else ""} diff --git a/comet/services/torrent_manager.py b/comet/services/torrent_manager.py index 3365bb2..341a3e5 100644 --- a/comet/services/torrent_manager.py +++ b/comet/services/torrent_manager.py @@ -148,7 +148,7 @@ async def add_torrent( await database.execute( f""" - INSERT {"OR IGNORE " if settings.DATABASE_TYPE == "sqlite" else ""} + INSERT {"OR REPLACE " if settings.DATABASE_TYPE == "sqlite" else ""} INTO torrents VALUES (:media_id, :info_hash, :file_index, :season, :episode, :title, :seeders, :size, :tracker, :sources, :parsed, :timestamp) {" ON CONFLICT DO NOTHING" if settings.DATABASE_TYPE == "postgresql" else ""} @@ -402,7 +402,7 @@ class TorrentUpdateQueue: sub_batch = self.batches["inserts"][i : i + sub_batch_size] async with database.transaction(): insert_query = f""" - INSERT {"OR IGNORE " if settings.DATABASE_TYPE == "sqlite" else ""} + INSERT {"OR REPLACE " if settings.DATABASE_TYPE == "sqlite" else ""} INTO torrents VALUES ( :media_id, diff --git a/comet/templates/admin_dashboard.html b/comet/templates/admin_dashboard.html index fcbbf12..0720e53 100644 --- a/comet/templates/admin_dashboard.html +++ b/comet/templates/admin_dashboard.html @@ -956,7 +956,16 @@

Torrents Analysis

-

Top Trackers

+
+

Top Trackers

+ + Top 5 + Top 10 + Top 20 + Top 50 + All + +
@@ -1075,6 +1084,7 @@ initializeDashboard(); }); + let currentMetricsData = null; let connectionsInterval; let logsInterval; let lastLogTimestamp = 0; @@ -1103,6 +1113,13 @@ document .getElementById("hide-api-logs") .addEventListener("sl-change", toggleApiLogsVisibility); + document + .getElementById("tracker-limit") + .addEventListener("sl-change", () => { + if (currentMetricsData) { + updateTorrentsSection(currentMetricsData.torrents); + } + }); // Tab change handler document @@ -1390,6 +1407,7 @@ try { const response = await fetch("/admin/api/metrics"); const data = await response.json(); + currentMetricsData = data; updateMetricsOverview(data); updateTorrentsSection(data.torrents); @@ -1414,11 +1432,19 @@ function updateTorrentsSection(torrentsData) { // Update tracker statistics const trackerContainer = document.getElementById("tracker-stats"); - if (torrentsData.by_tracker.length === 0) { + const limitSelect = document.getElementById("tracker-limit"); + const limit = limitSelect ? limitSelect.value : "5"; + + let displayTrackers = torrentsData.by_tracker; + if (limit !== "all") { + displayTrackers = displayTrackers.slice(0, parseInt(limit)); + } + + if (displayTrackers.length === 0) { trackerContainer.innerHTML = '

No tracker data available

'; } else { - trackerContainer.innerHTML = torrentsData.by_tracker + trackerContainer.innerHTML = displayTrackers .map( (tracker) => `