From ff4be3865eb87643de37a3e95ae2aeb5d1e8eda3 Mon Sep 17 00:00:00 2001 From: mhdzumair Date: Sat, 14 Dec 2024 21:33:53 +0530 Subject: [PATCH] Replaced `get_imdb_movie_data` with `get_imdb_title` for better naming consistency and added `get_imdb_title_data` to streamline IMDb metadata processing. Updated related code to accommodate these changes and ensure proper functionality of IMDb interactions. --- db/crud.py | 6 +- .../pipelines/sport_video_parser_pipeline.py | 2 +- .../pipelines/sports_parser_pipeline.py | 6 +- scrapers/imdb_data.py | 63 +++++++++++++++---- scrapers/prowlarr_feed.py | 4 +- 5 files changed, 60 insertions(+), 21 deletions(-) diff --git a/db/crud.py b/db/crud.py index cc400f7..a03abda 100644 --- a/db/crud.py +++ b/db/crud.py @@ -29,7 +29,7 @@ from db.models import ( from db.schemas import Stream, TorrentStreamsList from scrapers.tmdb_data import search_tmdb from scrapers.utils import run_scrapers -from scrapers.imdb_data import get_imdb_movie_data, search_imdb +from scrapers.imdb_data import get_imdb_title, search_imdb from streaming_providers.cache_helpers import store_cached_info_hashes from utils import crypto from utils.lock import acquire_redis_lock, release_redis_lock @@ -168,7 +168,7 @@ async def get_movie_data_by_id(movie_id: str) -> Optional[MediaFusionMovieMetaDa movie_data = await MediaFusionMovieMetaData.get(movie_id) # store it in the db for feature reference. if not movie_data and movie_id.startswith("tt"): - movie = await get_imdb_movie_data(movie_id, "movie") + movie = await get_imdb_title(movie_id, "movie") if not movie: return None @@ -236,7 +236,7 @@ async def get_series_data_by_id( ) if not series_data and series_id.startswith("tt"): - series = await get_imdb_movie_data(series_id, "series") + series = await get_imdb_title(series_id, "series") if not series: return None diff --git a/mediafusion_scrapy/pipelines/sport_video_parser_pipeline.py b/mediafusion_scrapy/pipelines/sport_video_parser_pipeline.py index 02a8401..27b4827 100644 --- a/mediafusion_scrapy/pipelines/sport_video_parser_pipeline.py +++ b/mediafusion_scrapy/pipelines/sport_video_parser_pipeline.py @@ -51,7 +51,7 @@ class SportVideoParserPipeline: "created_at": date_obj.strftime("%Y-%m-%d"), "year": date_obj.year, "languages": [re.sub(r"[ .]", "", item["language"])], - "is_imdb": False, + "is_search_imdb_title": False, "resolution": resolution, } ) diff --git a/mediafusion_scrapy/pipelines/sports_parser_pipeline.py b/mediafusion_scrapy/pipelines/sports_parser_pipeline.py index 6c02f28..2b73b49 100644 --- a/mediafusion_scrapy/pipelines/sports_parser_pipeline.py +++ b/mediafusion_scrapy/pipelines/sports_parser_pipeline.py @@ -46,7 +46,7 @@ class BaseParserPipeline: item.update( dict( type="movie", - is_imdb=False, + is_search_imdb_title=False, genres=[self.event_name.upper()], is_add_title_to_poster=True, ) @@ -212,10 +212,10 @@ class UFCParserPipeline(BaseParserPipeline): def __init__(self): super().__init__("ufc") - def update_imdb_data(self, torrent_data: dict): + async def update_imdb_data(self, torrent_data: dict): year = torrent_data.get("date").year title = torrent_data.get("event") - tmdb_data = search_tmdb(title, year) + tmdb_data = await search_tmdb(title, year) if not tmdb_data: if not torrent_data["poster"]: torrent_data["poster"] = random.choice( diff --git a/scrapers/imdb_data.py b/scrapers/imdb_data.py index ebd0207..3df9103 100644 --- a/scrapers/imdb_data.py +++ b/scrapers/imdb_data.py @@ -19,7 +19,7 @@ from utils.const import UA_HEADER from utils.network import CircuitBreaker, batch_process_with_circuit_breaker -async def get_imdb_movie_data(imdb_id: str, media_type: str) -> Optional[model.Title]: +async def get_imdb_title(imdb_id: str, media_type: str) -> Optional[model.Title]: try: title = await web.get_title_async( imdb_id, page="main", httpx_kwargs={"proxy": settings.requests_proxy_url} @@ -49,6 +49,42 @@ async def get_imdb_movie_data(imdb_id: str, media_type: str) -> Optional[model.T return title +async def get_imdb_title_data(imdb_id: str, media_type: str) -> Optional[dict]: + imdb_title = await get_imdb_title(imdb_id, media_type) + if not imdb_title: + return None + + poster_image, background_image = await get_poster_urls(imdb_title.imdb_id) + if not poster_image: + poster_image = imdb_title.primary_image + background_image = poster_image + + end_year = imdb_title.end_year if imdb_title.type_id == "tvSeries" else None + + return { + "imdb_id": imdb_title.imdb_id, + "poster": poster_image, + "background": background_image, + "title": imdb_title.title, + "year": imdb_title.year, + "end_year": end_year, + "description": imdb_title.plot.get("en-US"), + "genres": imdb_title.genres, + "imdb_rating": imdb_title.rating, + "aka_titles": list(set(aka.title for aka in imdb_title.akas)), + "type_id": imdb_title.type_id, + "parent_guide_nudity_status": imdb_title.advisories.nudity.status, + "parent_guide_certificates": list( + set( + rating + for cert in imdb_title.certification.certificates + for rating in cert.ratings + ) + ), + "runtime": imdb_title.runtime, + } + + async def get_imdb_rating(movie_id: str) -> Optional[float]: try: movie = await web.get_title_async( @@ -59,20 +95,21 @@ async def get_imdb_rating(movie_id: str) -> Optional[float]: return movie.rating +async def get_poster_urls(imdb_id: str) -> tuple: + poster = f"https://live.metahub.space/poster/medium/{imdb_id}/img" + try: + async with httpx.AsyncClient(proxy=settings.requests_proxy_url) as client: + response = await client.head(poster, timeout=10) + if response.status_code == 200: + return poster, poster + except httpx.RequestError: + pass + return None, None + + async def search_imdb( title: str, year: int, media_type: str = None, max_retries: int = 3 ) -> dict: - async def get_poster_urls(imdb_id: str) -> tuple: - poster = f"https://live.metahub.space/poster/medium/{imdb_id}/img" - try: - async with httpx.AsyncClient(proxy=settings.requests_proxy_url) as client: - response = await client.head(poster, timeout=10) - if response.status_code == 200: - return poster, poster - except httpx.RequestError: - pass - return None, None - async def process_movie(imdb_title: model.Movie | model.TVSeries) -> dict: try: if (imdb_title.type_id != "tvSeries" and imdb_title.year != year) or ( @@ -168,7 +205,7 @@ async def process_imdb_data(imdb_ids: list[str], metadata_type: str): ) async for result in batch_process_with_circuit_breaker( - get_imdb_movie_data, + get_imdb_title, imdb_ids, 5, rate_limit_delay=3, diff --git a/scrapers/prowlarr_feed.py b/scrapers/prowlarr_feed.py index 7d1dde8..b34cc13 100644 --- a/scrapers/prowlarr_feed.py +++ b/scrapers/prowlarr_feed.py @@ -214,7 +214,9 @@ async def get_metadata_by_id(imdb_id: str, media_type: str): async def search_and_create_metadata(metadata: dict, media_type: str): - metadata = await get_or_create_metadata(metadata, media_type, is_imdb=True) + metadata = await get_or_create_metadata( + metadata, media_type, is_search_imdb_title=True + ) if not metadata["id"].startswith("tt"): # Only create metadata for IMDb movies and series return None