From 356be2590c2a17299b17bdde4fa1087076ff9bb3 Mon Sep 17 00:00:00 2001 From: g0ldyy <153996346+g0ldyy@users.noreply.github.com> Date: Wed, 31 Dec 2025 12:28:37 +0100 Subject: [PATCH] refactor: optimize database indexes and introduce an index migration step --- comet/core/database.py | 177 +++++++++++++++------------------- deployment/docker-compose.yml | 2 +- 2 files changed, 78 insertions(+), 101 deletions(-) diff --git a/comet/core/database.py b/comet/core/database.py index c31afa7..51811ad 100644 --- a/comet/core/database.py +++ b/comet/core/database.py @@ -21,6 +21,8 @@ async def setup_database(): await database.connect() + await _migrate_indexes() + await database.execute( """ CREATE TABLE IF NOT EXISTS db_version ( @@ -159,7 +161,7 @@ async def setup_database(): await database.execute( """ - CREATE UNIQUE INDEX IF NOT EXISTS torrents_series_both_idx + CREATE UNIQUE INDEX IF NOT EXISTS unq_torrents_series ON torrents (media_id, info_hash, season, episode) WHERE season IS NOT NULL AND episode IS NOT NULL """ @@ -167,7 +169,7 @@ async def setup_database(): await database.execute( """ - CREATE UNIQUE INDEX IF NOT EXISTS torrents_season_only_idx + CREATE UNIQUE INDEX IF NOT EXISTS unq_torrents_season ON torrents (media_id, info_hash, season) WHERE season IS NOT NULL AND episode IS NULL """ @@ -175,7 +177,7 @@ async def setup_database(): await database.execute( """ - CREATE UNIQUE INDEX IF NOT EXISTS torrents_episode_only_idx + CREATE UNIQUE INDEX IF NOT EXISTS unq_torrents_episode ON torrents (media_id, info_hash, episode) WHERE season IS NULL AND episode IS NOT NULL """ @@ -183,7 +185,7 @@ async def setup_database(): await database.execute( """ - CREATE UNIQUE INDEX IF NOT EXISTS torrents_no_season_episode_idx + CREATE UNIQUE INDEX IF NOT EXISTS unq_torrents_movie ON torrents (media_id, info_hash) WHERE season IS NULL AND episode IS NULL """ @@ -207,7 +209,7 @@ async def setup_database(): await database.execute( """ - CREATE UNIQUE INDEX IF NOT EXISTS debrid_series_both_idx + CREATE UNIQUE INDEX IF NOT EXISTS unq_debrid_series ON debrid_availability (debrid_service, info_hash, season, episode) WHERE season IS NOT NULL AND episode IS NOT NULL """ @@ -215,7 +217,7 @@ async def setup_database(): await database.execute( """ - CREATE UNIQUE INDEX IF NOT EXISTS debrid_season_only_idx + CREATE UNIQUE INDEX IF NOT EXISTS unq_debrid_season ON debrid_availability (debrid_service, info_hash, season) WHERE season IS NOT NULL AND episode IS NULL """ @@ -223,7 +225,7 @@ async def setup_database(): await database.execute( """ - CREATE UNIQUE INDEX IF NOT EXISTS debrid_episode_only_idx + CREATE UNIQUE INDEX IF NOT EXISTS unq_debrid_episode ON debrid_availability (debrid_service, info_hash, episode) WHERE season IS NULL AND episode IS NOT NULL """ @@ -231,7 +233,7 @@ async def setup_database(): await database.execute( """ - CREATE UNIQUE INDEX IF NOT EXISTS debrid_no_season_episode_idx + CREATE UNIQUE INDEX IF NOT EXISTS unq_debrid_movie ON debrid_availability (debrid_service, info_hash) WHERE season IS NULL AND episode IS NULL """ @@ -372,87 +374,42 @@ async def setup_database(): ) # ============================================================================= - # TORRENTS TABLE INDEXES - Most critical for performance + # TORRENTS TABLE INDEXES # ============================================================================= - # Primary lookup index: media_id + season + episode + timestamp (cache TTL filter) + # Primary lookup index: media_id + season + episode (nullable) + timestamp + # Covers: get_cached_torrents, check_torrents_cache await database.execute( """ - CREATE INDEX IF NOT EXISTS idx_torrents_media_cache_lookup + CREATE INDEX IF NOT EXISTS idx_torrents_lookup ON torrents (media_id, season, episode, timestamp) """ ) - # Info hash lookup for playback (very frequent) + # Cleanup index: timestamp + # Covers: DELETE WHERE timestamp + ttl < current await database.execute( """ - CREATE INDEX IF NOT EXISTS idx_torrents_info_hash - ON torrents (info_hash) - """ - ) - - # Analytics queries: tracker-based aggregation - await database.execute( - """ - CREATE INDEX IF NOT EXISTS idx_torrents_tracker_analytics - ON torrents (tracker, seeders, size) - """ - ) - - # Size filtering for user preferences - await database.execute( - """ - CREATE INDEX IF NOT EXISTS idx_torrents_size_filter - ON torrents (size, timestamp) - """ - ) - - # Seeders ordering for quality ranking - await database.execute( - """ - CREATE INDEX IF NOT EXISTS idx_torrents_seeders_desc - ON torrents (seeders DESC, timestamp) + CREATE INDEX IF NOT EXISTS idx_torrents_timestamp + ON torrents (timestamp) """ ) # ============================================================================= - # DEBRID_AVAILABILITY TABLE INDEXES - Critical for cache performance + # DEBRID_AVAILABILITY TABLE INDEXES # ============================================================================= - # Primary cache lookup: service + info_hash list + timestamp + # Primary lookup index: service + info_hash + timestamp + # Covers: get_cached_availability (info_hash IN ...) await database.execute( """ - CREATE INDEX IF NOT EXISTS idx_debrid_service_hash_cache + CREATE INDEX IF NOT EXISTS idx_debrid_lookup ON debrid_availability (debrid_service, info_hash, timestamp) """ ) - # Season/episode filtering for series content - await database.execute( - """ - CREATE INDEX IF NOT EXISTS idx_debrid_season_episode_filter - ON debrid_availability (debrid_service, season, episode, timestamp) - """ - ) - - # Service-based analytics and cleanup - await database.execute( - """ - CREATE INDEX IF NOT EXISTS idx_debrid_service_timestamp - ON debrid_availability (debrid_service, timestamp) - """ - ) - - # Title filtering for OffCloud special case - await database.execute( - """ - CREATE INDEX IF NOT EXISTS idx_debrid_title_filter - ON debrid_availability (debrid_service, info_hash, title, timestamp) - """ - ) - # ============================================================================= - # DOWNLOAD_LINKS_CACHE TABLE INDEXES - Playback performance + # DOWNLOAD_LINKS_CACHE TABLE INDEXES # ============================================================================= # Primary playback lookup: debrid_key + info_hash + season + episode @@ -472,7 +429,7 @@ async def setup_database(): ) # ============================================================================= - # METADATA_CACHE TABLE INDEXES - Metadata performance + # METADATA_CACHE TABLE INDEXES # ============================================================================= # Primary cache lookup: media_id + timestamp @@ -492,7 +449,7 @@ async def setup_database(): ) # ============================================================================= - # FIRST_SEARCHES TABLE INDEXES - Search optimization + # FIRST_SEARCHES TABLE INDEXES # ============================================================================= # Primary search check: media_id (already PRIMARY KEY, but explicit for clarity) @@ -505,7 +462,7 @@ async def setup_database(): ) # ============================================================================= - # ACTIVE_CONNECTIONS TABLE INDEXES - Admin dashboard performance + # ACTIVE_CONNECTIONS TABLE INDEXES # ============================================================================= # Admin dashboard ordering: timestamp DESC (most recent first) @@ -533,7 +490,7 @@ async def setup_database(): ) # ============================================================================= - # SCRAPE_LOCKS TABLE INDEXES - Lock management + # SCRAPE_LOCKS TABLE INDEXES # ============================================================================= # Expired locks cleanup: expires_at < current_time @@ -553,7 +510,7 @@ async def setup_database(): ) # ============================================================================= - # ADMIN_SESSIONS TABLE INDEXES - Authentication performance + # ADMIN_SESSIONS TABLE INDEXES # ============================================================================= # Session cleanup: expires_at < current_time @@ -565,7 +522,7 @@ async def setup_database(): ) # ============================================================================= - # BACKGROUND_SCRAPER_STATE TABLE INDEXES - Scraper performance + # BACKGROUND_SCRAPER_STATE TABLE INDEXES # ============================================================================= # Media type filtering for scraper analytics @@ -592,34 +549,6 @@ async def setup_database(): """ ) - # ============================================================================= - # COMPOSITE INDEXES FOR COMPLEX QUERIES - # ============================================================================= - - # Torrents: media + quality filtering + cache validity - await database.execute( - """ - CREATE INDEX IF NOT EXISTS idx_torrents_quality_cache - ON torrents (media_id, seeders DESC, size DESC, timestamp) - """ - ) - - # Debrid: comprehensive availability lookup - await database.execute( - """ - CREATE INDEX IF NOT EXISTS idx_debrid_comprehensive - ON debrid_availability (debrid_service, info_hash, season, episode, size, timestamp) - """ - ) - - # Background scraper: comprehensive state tracking - await database.execute( - """ - CREATE INDEX IF NOT EXISTS idx_scraper_comprehensive - ON background_scraper_state (media_type, total_torrents_found, scraped_at) - """ - ) - await database.execute( """ CREATE INDEX IF NOT EXISTS idx_anime_mapping_imdb @@ -801,6 +730,54 @@ async def cleanup_expired_sessions(): await asyncio.sleep(5) # Clean up every 5 seconds +async def _migrate_indexes(): + try: + if settings.DATABASE_TYPE == "sqlite": + check_query = "SELECT name FROM sqlite_master WHERE type='index' AND name='torrents_series_both_idx'" + else: + check_query = "SELECT indexname FROM pg_indexes WHERE indexname='torrents_series_both_idx'" + + exists = await database.fetch_val(check_query) + + if not exists: + return + + logger.log("COMET", "Database: Migrating indexes (dropping legacy indexes)...") + + old_indexes = [ + "torrents_series_both_idx", + "torrents_season_only_idx", + "torrents_episode_only_idx", + "torrents_no_season_episode_idx", + "idx_torrents_media_cache_lookup", + "idx_torrents_info_hash", + "idx_torrents_tracker_analytics", + "idx_torrents_size_filter", + "idx_torrents_seeders_desc", + "idx_torrents_quality_cache", + "idx_torrents_media_season_episode", + "debrid_series_both_idx", + "debrid_season_only_idx", + "debrid_episode_only_idx", + "debrid_no_season_episode_idx", + "idx_debrid_service_hash_cache", + "idx_debrid_season_episode_filter", + "idx_debrid_service_timestamp", + "idx_debrid_title_filter", + "idx_debrid_comprehensive", + "idx_debrid_info_hash_season_episode", + "idx_debrid_timestamp", + ] + + for index_name in old_indexes: + await database.execute(f"DROP INDEX IF EXISTS {index_name}") + + logger.log("COMET", "Database: Legacy indexes dropped. New indexes will be created.") + + except Exception as e: + logger.warning(f"Error during index migration: {e}") + + async def teardown_database(): try: await database.disconnect() diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index 5eef29c..ee36e56 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -51,7 +51,7 @@ services: - "-c" - "max_connections=100" volumes: - - postgres_data:/var/lib/postgresql/18/docker + - postgres_data:/var/lib/postgresql/ healthcheck: test: ["CMD-SHELL", "pg_isready -U comet -d comet"] interval: 5s