diff --git a/api/scheduler.py b/api/scheduler.py index bb3836c..0953c00 100644 --- a/api/scheduler.py +++ b/api/scheduler.py @@ -3,7 +3,6 @@ from apscheduler.triggers.cron import CronTrigger from db.config import settings from mediafusion_scrapy.task import run_spider -from scrapers.imdb_data import fetch_movie_ids_to_update from scrapers.background_scraper import run_background_search from scrapers.feed_scraper import run_prowlarr_feed_scraper, run_jackett_feed_scraper from scrapers.trackers import update_torrent_seeders @@ -123,15 +122,6 @@ def setup_scheduler(scheduler: AsyncIOScheduler): }, ) - scheduler.add_job( - fetch_movie_ids_to_update.send, - CronTrigger.from_crontab(settings.update_imdb_data_crontab), - name="update_imdb_data", - kwargs={ - "crontab_expression": settings.update_imdb_data_crontab, - }, - ) - if not settings.disable_motogp_tgx_scheduler: scheduler.add_job( run_spider.send, diff --git a/api/task.py b/api/task.py index 9a0658a..0b5f2c4 100644 --- a/api/task.py +++ b/api/task.py @@ -9,7 +9,6 @@ from utils import torrent from mediafusion_scrapy import task from scrapers import ( tv, - imdb_data, trackers, helpers, prowlarr, diff --git a/db/config.py b/db/config.py index d207912..5a96d24 100644 --- a/db/config.py +++ b/db/config.py @@ -155,7 +155,6 @@ class Settings(BaseSettings): disable_sport_video_scheduler: bool = False dlhd_scheduler_crontab: str = "25 * * * *" disable_dlhd_scheduler: bool = False - update_imdb_data_crontab: str = "0 2 * * *" motogp_tgx_scheduler_crontab: str = "0 5 * * *" disable_motogp_tgx_scheduler: bool = False update_seeders_crontab: str = "0 0 * * *" diff --git a/scrapers/imdb_data.py b/scrapers/imdb_data.py index 08ca115..d11bb6d 100644 --- a/scrapers/imdb_data.py +++ b/scrapers/imdb_data.py @@ -1,9 +1,8 @@ import logging import math -from datetime import datetime, timedelta, date +from datetime import datetime, date from typing import Optional -import dramatiq import httpx from cinemagoerng import model, web, piculet from cinemagoerng.model import TVSeries, SearchFilters, RangeFilter @@ -14,7 +13,6 @@ from db.config import settings from db.models import ( MediaFusionMetaData, ) -from db.schemas import MetaIdProjection from utils.const import UA_HEADER from utils.network import CircuitBreaker, batch_process_with_circuit_breaker @@ -231,6 +229,8 @@ async def process_imdb_data(imdb_ids: list[str], metadata_type: str): { "$set": { "genres": result.genres, + "poster": result.primary_image, + "description": result.plot.get("en-US"), "imdb_rating": imdb_rating, "parent_guide_nudity_status": result.advisories.nudity.status, "parent_guide_certificates": list( @@ -249,70 +249,6 @@ async def process_imdb_data(imdb_ids: list[str], metadata_type: str): logging.info(f"Updating metadata for movie {movie_id}") -@dramatiq.actor(time_limit=3 * 60 * 60 * 1000, priority=8, max_retries=3) -async def process_imdb_data_background(imdb_ids, metadata_type="movie"): - # Validate imdb_ids from the database - now = datetime.now() - seven_days_ago = now - timedelta(days=7) - valid_ids = await MediaFusionMetaData.find( - { - "_id": {"$in": imdb_ids}, - "$or": [ - {"last_updated_at": {"$lt": seven_days_ago}}, - {"last_updated_at": None}, - ], - }, - projection_model=MetaIdProjection, - with_children=True, - ).to_list() - valid_ids = [doc.id for doc in valid_ids] - - if valid_ids: - await process_imdb_data(valid_ids, metadata_type) - - -@dramatiq.actor( - time_limit=60 * 1000, priority=10, max_retries=0, queue_name="scrapy" -) # Short time limit as this should be a fast operation -async def fetch_movie_ids_to_update(*args, **kwargs): - now = datetime.now() - seven_days_ago = now - timedelta(days=7) - - # Fetch only the IDs of movies that need updating - documents = await MediaFusionMetaData.find( - { - "_id": {"$regex": r"tt\d+"}, - "type": {"$in": ["movie", "series"]}, - "$or": [ - {"last_updated_at": {"$lt": seven_days_ago}}, - {"last_updated_at": None}, - ], - }, - projection_model=MetaIdProjection, - with_children=True, - ).to_list() - movie_ids = [] - series_ids = [] - for document in documents: - if document.type == "movie": - movie_ids.append(document.id) - else: - series_ids.append(document.id) - logging.info( - f"Fetched {len(movie_ids)} movie and {len(series_ids)} series IDs for updating" - ) - - # Divide the IDs into chunks and send them to another actor for processing - chunk_size = 25 - for i in range(0, len(movie_ids), chunk_size): - chunk_ids = movie_ids[i : i + chunk_size] - process_imdb_data_background.send(chunk_ids, metadata_type="movie") - - for i in range(0, len(series_ids), chunk_size): - chunk_ids = series_ids[i : i + chunk_size] - process_imdb_data_background.send(chunk_ids, metadata_type="series") - - async def get_episode_by_date( series_id: str, series_title: str, expected_date: date ) -> Optional[model.TVEpisode]: diff --git a/scrapers/tmdb_data.py b/scrapers/tmdb_data.py index 0a79899..51c4897 100644 --- a/scrapers/tmdb_data.py +++ b/scrapers/tmdb_data.py @@ -1,14 +1,12 @@ import logging -from datetime import datetime, timedelta +from datetime import datetime from typing import Optional, Dict, Any, List - -import dramatiq -import httpx from urllib.parse import urljoin +import httpx + from db.config import settings from db.models import MediaFusionMetaData -from db.schemas import MetaIdProjection from utils.const import UA_HEADER from utils.network import CircuitBreaker, batch_process_with_circuit_breaker @@ -243,77 +241,3 @@ async def process_tmdb_data(tmdb_ids: List[str], metadata_type: str): } ) logging.info(f"Updated TMDB metadata for {tmdb_id}") - - -@dramatiq.actor(time_limit=3 * 60 * 60 * 1000, priority=8, max_retries=3) -async def process_tmdb_data_background( - tmdb_ids: List[str], metadata_type: str = "movie" -): - """ - Background task to process TMDB data updates. - """ - now = datetime.now() - seven_days_ago = now - timedelta(days=7) - - # Validate tmdb_ids from the database - valid_ids = await MediaFusionMetaData.find( - { - "tmdb_id": {"$in": tmdb_ids}, - "$or": [ - {"last_updated_at": {"$lt": seven_days_ago}}, - {"last_updated_at": None}, - ], - }, - projection_model=MetaIdProjection, - with_children=True, - ).to_list() - - valid_ids = [doc.tmdb_id for doc in valid_ids] - - if valid_ids: - await process_tmdb_data(valid_ids, metadata_type) - - -@dramatiq.actor(time_limit=60 * 1000, priority=10, max_retries=0, queue_name="scrapy") -async def fetch_tmdb_ids_to_update(*args, **kwargs): - """ - Fetch TMDB IDs that need updating and queue them for processing. - """ - now = datetime.now() - seven_days_ago = now - timedelta(days=7) - - # Fetch IDs needing updates - documents = await MediaFusionMetaData.find( - { - "tmdb_id": {"$exists": True}, - "type": {"$in": ["movie", "series"]}, - "$or": [ - {"last_updated_at": {"$lt": seven_days_ago}}, - {"last_updated_at": None}, - ], - }, - projection_model=MetaIdProjection, - with_children=True, - ).to_list() - - movie_ids = [] - series_ids = [] - for document in documents: - if document.type == "movie": - movie_ids.append(document.tmdb_id) - else: - series_ids.append(document.tmdb_id) - - logging.info( - f"Fetched {len(movie_ids)} movie and {len(series_ids)} series TMDB IDs for updating" - ) - - # Process in chunks - chunk_size = 25 - for i in range(0, len(movie_ids), chunk_size): - chunk_ids = movie_ids[i : i + chunk_size] - process_tmdb_data_background.send(chunk_ids, metadata_type="movie") - - for i in range(0, len(series_ids), chunk_size): - chunk_ids = series_ids[i : i + chunk_size] - process_tmdb_data_background.send(chunk_ids, metadata_type="series")