Remove unoptimized IMDb and TMDB background tasks.

This commit eliminates unused Dramatiq background tasks and related scheduler jobs for processing and updating IMDb and TMDB data.

api/scheduler.py

@@ -3,7 +3,6 @@ from apscheduler.triggers.cron import CronTrigger
 
 from db.config import settings
 from mediafusion_scrapy.task import run_spider
-from scrapers.imdb_data import fetch_movie_ids_to_update
 from scrapers.background_scraper import run_background_search
 from scrapers.feed_scraper import run_prowlarr_feed_scraper, run_jackett_feed_scraper
 from scrapers.trackers import update_torrent_seeders
@@ -123,15 +122,6 @@ def setup_scheduler(scheduler: AsyncIOScheduler):
             },
         )
 
-    scheduler.add_job(
-        fetch_movie_ids_to_update.send,
-        CronTrigger.from_crontab(settings.update_imdb_data_crontab),
-        name="update_imdb_data",
-        kwargs={
-            "crontab_expression": settings.update_imdb_data_crontab,
-        },
-    )
-
     if not settings.disable_motogp_tgx_scheduler:
         scheduler.add_job(
             run_spider.send,

api/task.py

@@ -9,7 +9,6 @@ from utils import torrent
 from mediafusion_scrapy import task
 from scrapers import (
     tv,
-    imdb_data,
     trackers,
     helpers,
     prowlarr,

db/config.py

@@ -155,7 +155,6 @@ class Settings(BaseSettings):
     disable_sport_video_scheduler: bool = False
     dlhd_scheduler_crontab: str = "25 * * * *"
     disable_dlhd_scheduler: bool = False
-    update_imdb_data_crontab: str = "0 2 * * *"
     motogp_tgx_scheduler_crontab: str = "0 5 * * *"
     disable_motogp_tgx_scheduler: bool = False
     update_seeders_crontab: str = "0 0 * * *"

scrapers/imdb_data.py

@@ -1,9 +1,8 @@
 import logging
 import math
-from datetime import datetime, timedelta, date
+from datetime import datetime, date
 from typing import Optional
 
-import dramatiq
 import httpx
 from cinemagoerng import model, web, piculet
 from cinemagoerng.model import TVSeries, SearchFilters, RangeFilter
@@ -14,7 +13,6 @@ from db.config import settings
 from db.models import (
     MediaFusionMetaData,
 )
-from db.schemas import MetaIdProjection
 from utils.const import UA_HEADER
 from utils.network import CircuitBreaker, batch_process_with_circuit_breaker
 
@@ -231,6 +229,8 @@ async def process_imdb_data(imdb_ids: list[str], metadata_type: str):
             {
                 "$set": {
                     "genres": result.genres,
+                    "poster": result.primary_image,
+                    "description": result.plot.get("en-US"),
                     "imdb_rating": imdb_rating,
                     "parent_guide_nudity_status": result.advisories.nudity.status,
                     "parent_guide_certificates": list(
@@ -249,70 +249,6 @@ async def process_imdb_data(imdb_ids: list[str], metadata_type: str):
         logging.info(f"Updating metadata for movie {movie_id}")
 
 
-@dramatiq.actor(time_limit=3 * 60 * 60 * 1000, priority=8, max_retries=3)
-async def process_imdb_data_background(imdb_ids, metadata_type="movie"):
-    # Validate imdb_ids from the database
-    now = datetime.now()
-    seven_days_ago = now - timedelta(days=7)
-    valid_ids = await MediaFusionMetaData.find(
-        {
-            "_id": {"$in": imdb_ids},
-            "$or": [
-                {"last_updated_at": {"$lt": seven_days_ago}},
-                {"last_updated_at": None},
-            ],
-        },
-        projection_model=MetaIdProjection,
-        with_children=True,
-    ).to_list()
-    valid_ids = [doc.id for doc in valid_ids]
-
-    if valid_ids:
-        await process_imdb_data(valid_ids, metadata_type)
-
-
-@dramatiq.actor(
-    time_limit=60 * 1000, priority=10, max_retries=0, queue_name="scrapy"
-)  # Short time limit as this should be a fast operation
-async def fetch_movie_ids_to_update(*args, **kwargs):
-    now = datetime.now()
-    seven_days_ago = now - timedelta(days=7)
-
-    # Fetch only the IDs of movies that need updating
-    documents = await MediaFusionMetaData.find(
-        {
-            "_id": {"$regex": r"tt\d+"},
-            "type": {"$in": ["movie", "series"]},
-            "$or": [
-                {"last_updated_at": {"$lt": seven_days_ago}},
-                {"last_updated_at": None},
-            ],
-        },
-        projection_model=MetaIdProjection,
-        with_children=True,
-    ).to_list()
-    movie_ids = []
-    series_ids = []
-    for document in documents:
-        if document.type == "movie":
-            movie_ids.append(document.id)
-        else:
-            series_ids.append(document.id)
-    logging.info(
-        f"Fetched {len(movie_ids)} movie and {len(series_ids)} series IDs for updating"
-    )
-
-    # Divide the IDs into chunks and send them to another actor for processing
-    chunk_size = 25
-    for i in range(0, len(movie_ids), chunk_size):
-        chunk_ids = movie_ids[i : i + chunk_size]
-        process_imdb_data_background.send(chunk_ids, metadata_type="movie")
-
-    for i in range(0, len(series_ids), chunk_size):
-        chunk_ids = series_ids[i : i + chunk_size]
-        process_imdb_data_background.send(chunk_ids, metadata_type="series")
-
-
 async def get_episode_by_date(
     series_id: str, series_title: str, expected_date: date
 ) -> Optional[model.TVEpisode]:

scrapers/tmdb_data.py

@@ -1,14 +1,12 @@
 import logging
-from datetime import datetime, timedelta
+from datetime import datetime
 from typing import Optional, Dict, Any, List
-
-import dramatiq
-import httpx
 from urllib.parse import urljoin
 
+import httpx
+
 from db.config import settings
 from db.models import MediaFusionMetaData
-from db.schemas import MetaIdProjection
 from utils.const import UA_HEADER
 from utils.network import CircuitBreaker, batch_process_with_circuit_breaker
 
@@ -243,77 +241,3 @@ async def process_tmdb_data(tmdb_ids: List[str], metadata_type: str):
             }
         )
         logging.info(f"Updated TMDB metadata for {tmdb_id}")
-
-
-@dramatiq.actor(time_limit=3 * 60 * 60 * 1000, priority=8, max_retries=3)
-async def process_tmdb_data_background(
-    tmdb_ids: List[str], metadata_type: str = "movie"
-):
-    """
-    Background task to process TMDB data updates.
-    """
-    now = datetime.now()
-    seven_days_ago = now - timedelta(days=7)
-
-    # Validate tmdb_ids from the database
-    valid_ids = await MediaFusionMetaData.find(
-        {
-            "tmdb_id": {"$in": tmdb_ids},
-            "$or": [
-                {"last_updated_at": {"$lt": seven_days_ago}},
-                {"last_updated_at": None},
-            ],
-        },
-        projection_model=MetaIdProjection,
-        with_children=True,
-    ).to_list()
-
-    valid_ids = [doc.tmdb_id for doc in valid_ids]
-
-    if valid_ids:
-        await process_tmdb_data(valid_ids, metadata_type)
-
-
-@dramatiq.actor(time_limit=60 * 1000, priority=10, max_retries=0, queue_name="scrapy")
-async def fetch_tmdb_ids_to_update(*args, **kwargs):
-    """
-    Fetch TMDB IDs that need updating and queue them for processing.
-    """
-    now = datetime.now()
-    seven_days_ago = now - timedelta(days=7)
-
-    # Fetch IDs needing updates
-    documents = await MediaFusionMetaData.find(
-        {
-            "tmdb_id": {"$exists": True},
-            "type": {"$in": ["movie", "series"]},
-            "$or": [
-                {"last_updated_at": {"$lt": seven_days_ago}},
-                {"last_updated_at": None},
-            ],
-        },
-        projection_model=MetaIdProjection,
-        with_children=True,
-    ).to_list()
-
-    movie_ids = []
-    series_ids = []
-    for document in documents:
-        if document.type == "movie":
-            movie_ids.append(document.tmdb_id)
-        else:
-            series_ids.append(document.tmdb_id)
-
-    logging.info(
-        f"Fetched {len(movie_ids)} movie and {len(series_ids)} series TMDB IDs for updating"
-    )
-
-    # Process in chunks
-    chunk_size = 25
-    for i in range(0, len(movie_ids), chunk_size):
-        chunk_ids = movie_ids[i : i + chunk_size]
-        process_tmdb_data_background.send(chunk_ids, metadata_type="movie")
-
-    for i in range(0, len(series_ids), chunk_size):
-        chunk_ids = series_ids[i : i + chunk_size]
-        process_tmdb_data_background.send(chunk_ids, metadata_type="series")