g0ldyy/comet
1
0
Fork 0
mirror of https://github.com/g0ldyy/comet.git synced 2026-01-12 01:16:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: optimize database indexes and introduce an index migration step

Browse Source
This commit is contained in:
g0ldyy
2025-12-31 12:28:37 +01:00
parent 32e5127dd3
commit 356be2590c
2 changed files with 78 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
comet/core/database.py
+77 -100
View File
@@ -21,6 +21,8 @@ async def setup_database():
await database.connect()
await _migrate_indexes()
await database.execute(
"""
CREATE TABLE IF NOT EXISTS db_version (
@@ -159,7 +161,7 @@ async def setup_database():
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS torrents_series_both_idx
CREATE UNIQUE INDEX IF NOT EXISTS unq_torrents_series
ON torrents (media_id, info_hash, season, episode)
WHERE season IS NOT NULL AND episode IS NOT NULL
"""
@@ -167,7 +169,7 @@ async def setup_database():
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS torrents_season_only_idx
CREATE UNIQUE INDEX IF NOT EXISTS unq_torrents_season
ON torrents (media_id, info_hash, season)
WHERE season IS NOT NULL AND episode IS NULL
"""
@@ -175,7 +177,7 @@ async def setup_database():
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS torrents_episode_only_idx
CREATE UNIQUE INDEX IF NOT EXISTS unq_torrents_episode
ON torrents (media_id, info_hash, episode)
WHERE season IS NULL AND episode IS NOT NULL
"""
@@ -183,7 +185,7 @@ async def setup_database():
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS torrents_no_season_episode_idx
CREATE UNIQUE INDEX IF NOT EXISTS unq_torrents_movie
ON torrents (media_id, info_hash)
WHERE season IS NULL AND episode IS NULL
"""
@@ -207,7 +209,7 @@ async def setup_database():
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS debrid_series_both_idx
CREATE UNIQUE INDEX IF NOT EXISTS unq_debrid_series
ON debrid_availability (debrid_service, info_hash, season, episode)
WHERE season IS NOT NULL AND episode IS NOT NULL
"""
@@ -215,7 +217,7 @@ async def setup_database():
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS debrid_season_only_idx
CREATE UNIQUE INDEX IF NOT EXISTS unq_debrid_season
ON debrid_availability (debrid_service, info_hash, season)
WHERE season IS NOT NULL AND episode IS NULL
"""
@@ -223,7 +225,7 @@ async def setup_database():
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS debrid_episode_only_idx
CREATE UNIQUE INDEX IF NOT EXISTS unq_debrid_episode
ON debrid_availability (debrid_service, info_hash, episode)
WHERE season IS NULL AND episode IS NOT NULL
"""
@@ -231,7 +233,7 @@ async def setup_database():
await database.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS debrid_no_season_episode_idx
CREATE UNIQUE INDEX IF NOT EXISTS unq_debrid_movie
ON debrid_availability (debrid_service, info_hash)
WHERE season IS NULL AND episode IS NULL
"""
@@ -372,87 +374,42 @@ async def setup_database():
)
# =============================================================================
# TORRENTS TABLE INDEXES - Most critical for performance
# TORRENTS TABLE INDEXES
# =============================================================================
# Primary lookup index: media_id + season + episode + timestamp (cache TTL filter)
# Primary lookup index: media_id + season + episode (nullable) + timestamp
# Covers: get_cached_torrents, check_torrents_cache
await database.execute(
"""
CREATE INDEX IF NOT EXISTS idx_torrents_media_cache_lookup
CREATE INDEX IF NOT EXISTS idx_torrents_lookup
ON torrents (media_id, season, episode, timestamp)
"""
)
# Info hash lookup for playback (very frequent)
# Cleanup index: timestamp
# Covers: DELETE WHERE timestamp + ttl < current
await database.execute(
"""
CREATE INDEX IF NOT EXISTS idx_torrents_info_hash
ON torrents (info_hash)
"""
)
# Analytics queries: tracker-based aggregation
await database.execute(
"""
CREATE INDEX IF NOT EXISTS idx_torrents_tracker_analytics
ON torrents (tracker, seeders, size)
"""
)
# Size filtering for user preferences
await database.execute(
"""
CREATE INDEX IF NOT EXISTS idx_torrents_size_filter
ON torrents (size, timestamp)
"""
)
# Seeders ordering for quality ranking
await database.execute(
"""
CREATE INDEX IF NOT EXISTS idx_torrents_seeders_desc
ON torrents (seeders DESC, timestamp)
CREATE INDEX IF NOT EXISTS idx_torrents_timestamp
ON torrents (timestamp)
"""
)
# =============================================================================
# DEBRID_AVAILABILITY TABLE INDEXES - Critical for cache performance
# DEBRID_AVAILABILITY TABLE INDEXES
# =============================================================================
# Primary cache lookup: service + info_hash list + timestamp
# Primary lookup index: service + info_hash + timestamp
# Covers: get_cached_availability (info_hash IN ...)
await database.execute(
"""
CREATE INDEX IF NOT EXISTS idx_debrid_service_hash_cache
CREATE INDEX IF NOT EXISTS idx_debrid_lookup
ON debrid_availability (debrid_service, info_hash, timestamp)
"""
)
# Season/episode filtering for series content
await database.execute(
"""
CREATE INDEX IF NOT EXISTS idx_debrid_season_episode_filter
ON debrid_availability (debrid_service, season, episode, timestamp)
"""
)
# Service-based analytics and cleanup
await database.execute(
"""
CREATE INDEX IF NOT EXISTS idx_debrid_service_timestamp
ON debrid_availability (debrid_service, timestamp)
"""
)
# Title filtering for OffCloud special case
await database.execute(
"""
CREATE INDEX IF NOT EXISTS idx_debrid_title_filter
ON debrid_availability (debrid_service, info_hash, title, timestamp)
"""
)
# =============================================================================
# DOWNLOAD_LINKS_CACHE TABLE INDEXES - Playback performance
# DOWNLOAD_LINKS_CACHE TABLE INDEXES
# =============================================================================
# Primary playback lookup: debrid_key + info_hash + season + episode
@@ -472,7 +429,7 @@ async def setup_database():
)
# =============================================================================
# METADATA_CACHE TABLE INDEXES - Metadata performance
# METADATA_CACHE TABLE INDEXES
# =============================================================================
# Primary cache lookup: media_id + timestamp
@@ -492,7 +449,7 @@ async def setup_database():
)
# =============================================================================
# FIRST_SEARCHES TABLE INDEXES - Search optimization
# FIRST_SEARCHES TABLE INDEXES
# =============================================================================
# Primary search check: media_id (already PRIMARY KEY, but explicit for clarity)
@@ -505,7 +462,7 @@ async def setup_database():
)
# =============================================================================
# ACTIVE_CONNECTIONS TABLE INDEXES - Admin dashboard performance
# ACTIVE_CONNECTIONS TABLE INDEXES
# =============================================================================
# Admin dashboard ordering: timestamp DESC (most recent first)
@@ -533,7 +490,7 @@ async def setup_database():
)
# =============================================================================
# SCRAPE_LOCKS TABLE INDEXES - Lock management
# SCRAPE_LOCKS TABLE INDEXES
# =============================================================================
# Expired locks cleanup: expires_at < current_time
@@ -553,7 +510,7 @@ async def setup_database():
)
# =============================================================================
# ADMIN_SESSIONS TABLE INDEXES - Authentication performance
# ADMIN_SESSIONS TABLE INDEXES
# =============================================================================
# Session cleanup: expires_at < current_time
@@ -565,7 +522,7 @@ async def setup_database():
)
# =============================================================================
# BACKGROUND_SCRAPER_STATE TABLE INDEXES - Scraper performance
# BACKGROUND_SCRAPER_STATE TABLE INDEXES
# =============================================================================
# Media type filtering for scraper analytics
@@ -592,34 +549,6 @@ async def setup_database():
"""
)
# =============================================================================
# COMPOSITE INDEXES FOR COMPLEX QUERIES
# =============================================================================
# Torrents: media + quality filtering + cache validity
await database.execute(
"""
CREATE INDEX IF NOT EXISTS idx_torrents_quality_cache
ON torrents (media_id, seeders DESC, size DESC, timestamp)
"""
)
# Debrid: comprehensive availability lookup
await database.execute(
"""
CREATE INDEX IF NOT EXISTS idx_debrid_comprehensive
ON debrid_availability (debrid_service, info_hash, season, episode, size, timestamp)
"""
)
# Background scraper: comprehensive state tracking
await database.execute(
"""
CREATE INDEX IF NOT EXISTS idx_scraper_comprehensive
ON background_scraper_state (media_type, total_torrents_found, scraped_at)
"""
)
await database.execute(
"""
CREATE INDEX IF NOT EXISTS idx_anime_mapping_imdb
@@ -801,6 +730,54 @@ async def cleanup_expired_sessions():
await asyncio.sleep(5) # Clean up every 5 seconds
async def _migrate_indexes():
try:
if settings.DATABASE_TYPE == "sqlite":
check_query = "SELECT name FROM sqlite_master WHERE type='index' AND name='torrents_series_both_idx'"
else:
check_query = "SELECT indexname FROM pg_indexes WHERE indexname='torrents_series_both_idx'"
exists = await database.fetch_val(check_query)
if not exists:
return
logger.log("COMET", "Database: Migrating indexes (dropping legacy indexes)...")
old_indexes = [
"torrents_series_both_idx",
"torrents_season_only_idx",
"torrents_episode_only_idx",
"torrents_no_season_episode_idx",
"idx_torrents_media_cache_lookup",
"idx_torrents_info_hash",
"idx_torrents_tracker_analytics",
"idx_torrents_size_filter",
"idx_torrents_seeders_desc",
"idx_torrents_quality_cache",
"idx_torrents_media_season_episode",
"debrid_series_both_idx",
"debrid_season_only_idx",
"debrid_episode_only_idx",
"debrid_no_season_episode_idx",
"idx_debrid_service_hash_cache",
"idx_debrid_season_episode_filter",
"idx_debrid_service_timestamp",
"idx_debrid_title_filter",
"idx_debrid_comprehensive",
"idx_debrid_info_hash_season_episode",
"idx_debrid_timestamp",
]
for index_name in old_indexes:
await database.execute(f"DROP INDEX IF EXISTS {index_name}")
logger.log("COMET", "Database: Legacy indexes dropped. New indexes will be created.")
except Exception as e:
logger.warning(f"Error during index migration: {e}")
async def teardown_database():
try:
await database.disconnect()
deployment/docker-compose.yml
+1 -1
View File
@@ -51,7 +51,7 @@ services:
- "-c"
- "max_connections=100"
volumes:
- postgres_data:/var/lib/postgresql/18/docker
- postgres_data:/var/lib/postgresql/
healthcheck:
test: ["CMD-SHELL", "pg_isready -U comet -d comet"]
interval: 5s