From d169b28aec2cdd51c4b04687e71a8335e2e03280 Mon Sep 17 00:00:00 2001 From: Mohamed Zumair Date: Thu, 12 Sep 2024 12:17:33 +0530 Subject: [PATCH] Add support for RPDB posters & Support fallback posters via MediaFusion posters (#284) --- api/main.py | 79 ++++++++++++++++++++++------- db/crud.py | 14 ++---- db/schemas.py | 9 ++++ resources/html/configure.html | 41 +++++++++++++++ resources/js/config_script.js | 17 ++++++- scrapers/rpdb.py | 95 +++++++++++++++++++++++++++++++++++ utils/network.py | 2 +- 7 files changed, 227 insertions(+), 30 deletions(-) create mode 100644 scrapers/rpdb.py diff --git a/api/main.py b/api/main.py index a66fbe5..0f9d104 100644 --- a/api/main.py +++ b/api/main.py @@ -25,6 +25,7 @@ from db.config import settings from kodi.routes import kodi_router from metrics.routes import metrics_router from scrapers.routes import router as scrapers_router +from scrapers.rpdb import update_rpdb_posters, update_rpdb_poster from streaming_providers import mapper from streaming_providers.routes import router as streaming_provider_router from utils import const, crypto, get_json_data, poster, torrent, wrappers @@ -261,33 +262,78 @@ async def get_catalog( genre: str = None, user_data: schemas.UserData = Depends(get_user_data), ): + skip, genre = parse_genre_and_skip(genre, skip) + cache_key, is_watchlist_catalog = get_cache_key( + catalog_type, catalog_id, skip, genre, user_data + ) + + if cache_key: + response.headers.update(const.CACHE_HEADERS) + if cached_data := await REDIS_ASYNC_CLIENT.get(cache_key): + return await update_rpdb_posters( + schemas.Metas.model_validate_json(cached_data), user_data, catalog_type + ) + else: + response.headers.update(const.NO_CACHE_HEADERS) + + metas = await fetch_metas( + catalog_type, catalog_id, genre, skip, user_data, request, is_watchlist_catalog + ) + + if cache_key: + await REDIS_ASYNC_CLIENT.set( + cache_key, + metas.model_dump_json(exclude_none=True, by_alias=True), + ex=settings.meta_cache_ttl, + ) + + return await update_rpdb_posters(metas, user_data, catalog_type) + + +def parse_genre_and_skip(genre: str, skip: int) -> tuple[int, str]: if genre and "&" in genre: genre, skip = genre.split("&") skip = skip.split("=")[1] if "=" in skip else "0" skip = int(skip) if skip and skip.isdigit() else 0 + return skip, genre + +def get_cache_key( + catalog_type: str, + catalog_id: str, + skip: int, + genre: str, + user_data: schemas.UserData, +) -> tuple[str, bool]: cache_key = f"{catalog_type}_{catalog_id}_{skip}_{genre}_catalog" is_watchlist_catalog = False + if user_data.streaming_provider and catalog_id.startswith( user_data.streaming_provider.service ): - response.headers.update(const.NO_CACHE_HEADERS) cache_key = None is_watchlist_catalog = True elif catalog_type == "events": - response.headers.update(const.NO_CACHE_HEADERS) cache_key = None elif catalog_type in ["movie", "series"]: cache_key += "_" + "_".join( user_data.nudity_filter + user_data.certification_filter ) - # Try retrieving the cached data - if cache_key: - if cached_data := await REDIS_ASYNC_CLIENT.get(cache_key): - return json.loads(cached_data) + return cache_key, is_watchlist_catalog + +async def fetch_metas( + catalog_type: str, + catalog_id: str, + genre: str, + skip: int, + user_data: schemas.UserData, + request: Request, + is_watchlist_catalog: bool, +) -> schemas.Metas: metas = schemas.Metas() + if catalog_type == "tv": metas.metas.extend( await crud.get_tv_meta_list( @@ -309,6 +355,7 @@ async def get_catalog( genre=genre, ) ) + if ( is_watchlist_catalog and catalog_type == "movie" @@ -323,13 +370,6 @@ async def get_catalog( ) metas.metas.insert(0, delete_all_meta) - if cache_key: - await REDIS_ASYNC_CLIENT.set( - cache_key, - metas.model_dump_json(exclude_none=True, by_alias=True), - ex=settings.meta_cache_ttl, - ) - return metas @@ -366,7 +406,10 @@ async def search_meta( search_query, namespace=get_request_namespace(request) ) - return await crud.process_search_query(search_query, catalog_type, user_data) + metadata = await crud.process_search_query(search_query, catalog_type, user_data) + return await update_rpdb_posters( + schemas.Metas.model_validate(metadata), user_data, catalog_type + ) @app.get( @@ -399,10 +442,10 @@ async def get_meta( # Try retrieving the cached data cached_data = await REDIS_ASYNC_CLIENT.get(cache_key) if cached_data: - meta_data = json.loads(cached_data) + meta_data = schemas.MetaItem.model_validate_json(cached_data) if not meta_data: raise HTTPException(status_code=404, detail="Meta ID not found.") - return meta_data + return await update_rpdb_poster(meta_data, user_data, catalog_type) if catalog_type == "movie": if meta_id.startswith("dl"): @@ -425,7 +468,9 @@ async def get_meta( if not data: raise HTTPException(status_code=404, detail="Meta ID not found.") - return data + return await update_rpdb_poster( + schemas.MetaItem.model_validate(data), user_data, catalog_type + ) @app.get( diff --git a/db/crud.py b/db/crud.py index 6ad898e..d679056 100644 --- a/db/crud.py +++ b/db/crud.py @@ -461,6 +461,8 @@ async def get_series_meta(meta_id: str, user_data: schemas.UserData): "$nin": get_filter_certification_values(user_data) } + poster_path = f"{settings.poster_host_url}/poster/series/{meta_id}.jpg" + # Define the aggregation pipeline pipeline = [ {"$match": match_filter}, @@ -561,7 +563,6 @@ async def get_series_meta(meta_id: str, user_data: schemas.UserData): "id": "$videos.id", "meta_id": "$_id", "series_title": "$series_title", # Store series title - "poster": "$poster", "background": "$background", }, "video": {"$first": "$videos"}, @@ -575,7 +576,6 @@ async def get_series_meta(meta_id: str, user_data: schemas.UserData): "episode": "$video.episode", "meta_id": "$_id.meta_id", "series_title": "$_id.series_title", - "poster": "$_id.poster", "background": "$_id.background", "video": "$video", } @@ -586,7 +586,6 @@ async def get_series_meta(meta_id: str, user_data: schemas.UserData): "$group": { "_id": "$meta_id", "title": {"$first": "$series_title"}, - "poster": {"$first": "$poster"}, "background": {"$first": "$background"}, "videos": {"$push": "$video"}, } @@ -598,14 +597,7 @@ async def get_series_meta(meta_id: str, user_data: schemas.UserData): "_id": "$_id", "type": {"$literal": "series"}, "title": "$title", - "poster": { - "$concat": [ - settings.poster_host_url, - "/poster/series/", - {"$toString": "$_id"}, - ".jpg", - ] - }, + "poster": {"$literal": poster_path}, "background": {"$ifNull": ["$background", "$poster"]}, "videos": "$videos", }, diff --git a/db/schemas.py b/db/schemas.py index cbf2156..9d95235 100644 --- a/db/schemas.py +++ b/db/schemas.py @@ -117,6 +117,14 @@ class MediaFlowConfig(BaseModel): populate_by_name = True +class RPDBConfig(BaseModel): + api_key: str = Field(alias="ak") + + class Config: + extra = "ignore" + populate_by_name = True + + class StreamingProvider(BaseModel): service: Literal[ "realdebrid", @@ -186,6 +194,7 @@ class UserData(BaseModel): default=list(const.QUALITY_GROUPS.keys()), alias="qf" ) mediaflow_config: MediaFlowConfig | None = Field(default=None, alias="mfc") + rpdb_config: RPDBConfig | None = Field(default=None, alias="rpc") @field_validator("selected_resolutions", mode="after") def validate_selected_resolutions(cls, v): diff --git a/resources/html/configure.html b/resources/html/configure.html index d53b884..09fb64a 100644 --- a/resources/html/configure.html +++ b/resources/html/configure.html @@ -477,6 +477,11 @@
+
+ + MediaFlow Setup Guide + +
+ +
+
RPDB (RatingPosterDB) Configuration
+
+ + +
+
+
+ Get RPDB API Key + | + Configure RPDB Posters Settings +
+

+ RPDB is an optional Freemium third-party service for enhanced rating posters. By default, MediaFusion generates rating posters using IMDb ratings. +

+
+ +
+ + +
+
+ Please enter a valid RPDB API Key. +
+
+
+
{% if authentication_required %} diff --git a/resources/js/config_script.js b/resources/js/config_script.js index 41fd7cd..22a15fc 100644 --- a/resources/js/config_script.js +++ b/resources/js/config_script.js @@ -317,6 +317,14 @@ function getUserData() { validateInput('mediaflow_api_password', mediaflowConfig.api_password.trim() !== ''); } + let rpdbConfig = null; + if (document.getElementById('enable_rpdb').checked) { + rpdbConfig = { + api_key: document.getElementById('rpdb_api_key').value, + }; + validateInput('rpdb_api_key', rpdbConfig.api_key.trim() !== ''); + } + // Collect and validate other user data const maxSizeSlider = document.getElementById('max_size_slider'); const maxSizeValue = maxSizeSlider.value; @@ -367,6 +375,7 @@ function getUserData() { quality_filter: selectedQualityFilters, api_password: apiPassword, mediaflow_config: mediaflowConfig, + rpdb_config: rpdbConfig, }; } @@ -456,6 +465,11 @@ document.getElementById('enable_mediaflow').addEventListener('change', function setElementDisplay('mediaflow_config', this.checked ? 'block' : 'none'); }); +document.getElementById('enable_rpdb').addEventListener('change', function () { + setElementDisplay('rpdb_config', this.checked ? 'block' : 'none'); +}); + + // Event listener for the slider document.getElementById('max_size_slider').addEventListener('input', updateSizeOutput); @@ -544,6 +558,7 @@ document.addEventListener('DOMContentLoaded', function () { setupPasswordToggle('qbittorrent_password', 'toggleQbittorrentPassword', 'toggleQbittorrentPasswordIcon'); setupPasswordToggle('webdav_password', 'toggleWebdavPassword', 'toggleWebdavPasswordIcon'); setupPasswordToggle('mediaflow_api_password', 'toggleMediaFlowPassword', 'toggleMediaFlowPasswordIcon'); + setupPasswordToggle('rpdb_api_key', 'toggleRPDBApiKey', 'toggleRPDBApiKeyIcon'); }); @@ -583,7 +598,7 @@ document.addEventListener('DOMContentLoaded', function () { }); -document.addEventListener('DOMContentLoaded', function() { +document.addEventListener('DOMContentLoaded', function () { const kodiSetupBtn = document.getElementById('kodiSetupBtn'); if (kodiSetupBtn) { kodiSetupBtn.addEventListener('click', initiateKodiSetup); diff --git a/scrapers/rpdb.py b/scrapers/rpdb.py new file mode 100644 index 0000000..ce5ed6f --- /dev/null +++ b/scrapers/rpdb.py @@ -0,0 +1,95 @@ +import asyncio +import time + +import httpx +from sqlalchemy.testing.plugin.plugin_base import logging + +from db import schemas +from utils.runtime_const import REDIS_ASYNC_CLIENT + +RPDB_SUPPORTED_SET = "rpdb_supported_ids" +RPDB_UNSUPPORTED_HASH = "rpdb_unsupported_ids" +RPDB_UNSUPPORTED_EXPIRY = 60 * 60 * 24 * 7 # 7 days in seconds + + +async def check_rpdb_poster_availability(rpdb_poster_url: str) -> bool: + try: + async with httpx.AsyncClient(timeout=10.0) as client: + response = await client.head(rpdb_poster_url) + return response.status_code == 200 + except httpx.HTTPError as exc: + logging.error(f"HTTP Exception for {exc.request.url} - {exc}") + return False + except Exception as exc: + logging.error(f"Exception for {rpdb_poster_url} - {exc}") + return False + + +async def update_single_rpdb_poster(imdb_id: str, rpdb_poster_base: str) -> str | None: + if not imdb_id.startswith("tt"): + return None + + rpdb_poster_url = f"{rpdb_poster_base}{imdb_id}.jpg" + current_time = int(time.time()) + + # Check if the IMDB ID is in the supported set + if await REDIS_ASYNC_CLIENT.sismember(RPDB_SUPPORTED_SET, imdb_id): + return rpdb_poster_url + + # Check if the IMDB ID is in the unsupported hash + expiry_time = await REDIS_ASYNC_CLIENT.hget(RPDB_UNSUPPORTED_HASH, imdb_id) + + if expiry_time: + expiry_time = int(expiry_time) + if expiry_time > current_time: + # Still within expiry period, return None + return None + else: + # Expired, remove from unsupported hash + await REDIS_ASYNC_CLIENT.hdel(RPDB_UNSUPPORTED_HASH, imdb_id) + + # Check availability (either not in unsupported hash or expired) + if await check_rpdb_poster_availability(rpdb_poster_url): + await REDIS_ASYNC_CLIENT.sadd(RPDB_SUPPORTED_SET, imdb_id) + return rpdb_poster_url + else: + new_expiry_time = current_time + RPDB_UNSUPPORTED_EXPIRY + await REDIS_ASYNC_CLIENT.hset( + RPDB_UNSUPPORTED_HASH, imdb_id, str(new_expiry_time) + ) + return None + + +async def update_rpdb_poster( + meta_item: schemas.MetaItem, user_data: schemas.UserData, catalog_type: str +) -> schemas.MetaItem: + if not user_data.rpdb_config or catalog_type not in ["movie", "series"]: + return meta_item + + rpdb_poster_base = f"https://api.ratingposterdb.com/{user_data.rpdb_config.api_key}/imdb/poster-default/" + updated_poster = await update_single_rpdb_poster( + meta_item.meta.id, rpdb_poster_base + ) + + if updated_poster: + meta_item.meta.poster = updated_poster + + return meta_item + + +async def update_rpdb_posters( + metas: schemas.Metas, user_data: schemas.UserData, catalog_type: str +) -> schemas.Metas: + if not user_data.rpdb_config or catalog_type not in ["movie", "series"]: + return metas + + rpdb_poster_base = f"https://api.ratingposterdb.com/{user_data.rpdb_config.api_key}/imdb/poster-default/" + + async def update_poster(meta): + updated_poster = await update_single_rpdb_poster(meta.id, rpdb_poster_base) + if updated_poster: + meta.poster = updated_poster + + await asyncio.gather(*(update_poster(meta) for meta in metas.metas)) + + return metas diff --git a/utils/network.py b/utils/network.py index 40390cf..9681fc3 100644 --- a/utils/network.py +++ b/utils/network.py @@ -235,7 +235,7 @@ def get_request_namespace(request: Request) -> str: return namespace -def get_user_data(request: Request) -> UserData: +def get_user_data(request: Request, secret_str: str | None = None) -> UserData: return request.user