From e2d1e07086d3746313ef7608d606fcd61ce1cce7 Mon Sep 17 00:00:00 2001 From: mhdzumair Date: Sun, 5 Jan 2025 22:01:13 +0530 Subject: [PATCH] Add support for MDBList Catalog support --- api/main.py | 34 +- db/crud.py | 170 ++++++ db/models.py | 2 +- db/schemas.py | 25 + resources/html/configure.html | 223 ++++++- resources/js/config_script.js | 20 +- resources/js/mdblist.js | 833 +++++++++++++++++++++++++++ resources/templates/manifest.json.j2 | 15 +- scrapers/mdblist.py | 185 ++++++ scrapers/rpdb.py | 3 + scrapers/scraper_tasks.py | 2 +- utils/parser.py | 10 + utils/validation_helper.py | 13 + 13 files changed, 1526 insertions(+), 9 deletions(-) create mode 100644 resources/js/mdblist.js create mode 100644 scrapers/mdblist.py diff --git a/api/main.py b/api/main.py index 452e922..ab40251 100644 --- a/api/main.py +++ b/api/main.py @@ -51,6 +51,7 @@ from utils.runtime_const import ( from utils.validation_helper import ( validate_mediaflow_proxy_credentials, validate_rpdb_token, + validate_mdblist_token, ) logging.basicConfig( @@ -160,6 +161,8 @@ async def configure( response.headers.update(const.NO_CACHE_HEADERS) configured_fields = [] + mdblist_configured_lists = [] + catalogs_data = const.CATALOG_DATA.copy() # Remove the password from the streaming provider if user_data.streaming_provider: user_data.streaming_provider.password = "••••••••" @@ -183,11 +186,21 @@ async def configure( user_data.rpdb_config.api_key = "••••••••" configured_fields.append("rpdb_api_key") + # Check MDBList configuration + if user_data.mdblist_config: + # user_data.mdblist_config.api_key = "••••••••" + # configured_fields.append("mdblist_api_key") + for mdblist in user_data.mdblist_config.lists: + mdblist_configured_lists.append(mdblist.model_dump()) + catalogs_data[f"mdblist_{mdblist.catalog_type}_{mdblist.id}"] = ( + f"MDBList: {mdblist.title}" + ) + user_data.api_password = None # Prepare catalogs based on user preferences or default order sorted_catalogs = sorted( - const.CATALOG_DATA.items(), + catalogs_data.items(), key=lambda x: ( user_data.selected_catalogs.index(x[0]) if x[0] in user_data.selected_catalogs @@ -226,6 +239,7 @@ async def configure( "disabled_providers": settings.disabled_providers, "configured_fields": configured_fields, "secret_str": secret_str, + "mdblist_configured_lists": mdblist_configured_lists, }, ) @@ -316,6 +330,7 @@ async def get_catalog( skip: int = 0, genre: str = None, user_data: schemas.UserData = Depends(get_user_data), + background_tasks: BackgroundTasks = BackgroundTasks(), ): cache_key, is_watchlist_catalog = get_cache_key( catalog_type, catalog_id, skip, genre, user_data @@ -333,7 +348,14 @@ async def get_catalog( response.headers.update(const.NO_CACHE_HEADERS) metas = await fetch_metas( - catalog_type, catalog_id, genre, skip, user_data, request, is_watchlist_catalog + catalog_type, + catalog_id, + genre, + skip, + user_data, + request, + is_watchlist_catalog, + background_tasks, ) if cache_key: @@ -379,6 +401,7 @@ async def fetch_metas( user_data: schemas.UserData, request: Request, is_watchlist_catalog: bool, + background_tasks: BackgroundTasks, ) -> schemas.Metas: metas = schemas.Metas() @@ -390,6 +413,12 @@ async def fetch_metas( ) elif catalog_type == "events": metas.metas.extend(await crud.get_events_meta_list(genre, skip)) + elif catalog_id.startswith("mdblist"): + metas.metas.extend( + await crud.get_mdblist_meta_list( + user_data, background_tasks, catalog_id, catalog_type, genre, skip + ) + ) else: user_ip = await get_user_public_ip(request, user_data) metas.metas.extend( @@ -655,6 +684,7 @@ async def encrypt_user_data( validate_provider_credentials(request, user_data), validate_mediaflow_proxy_credentials(user_data), validate_rpdb_token(user_data), + validate_mdblist_token(user_data), ] results = await asyncio.gather(*validation_tasks, return_exceptions=True) diff --git a/db/crud.py b/db/crud.py index 5432f68..b3406bd 100644 --- a/db/crud.py +++ b/db/crud.py @@ -30,6 +30,7 @@ from db.models import ( ) from db.redis_database import REDIS_ASYNC_CLIENT from db.schemas import Stream, TorrentStreamsList +from scrapers.mdblist import initialize_mdblist_scraper from scrapers.scraper_tasks import run_scrapers, meta_fetcher from streaming_providers.cache_helpers import store_cached_info_hashes from utils import crypto @@ -142,6 +143,157 @@ async def get_meta_list( return meta_list +async def get_mdblist_meta_list( + user_data: schemas.UserData, + background_tasks: BackgroundTasks, + catalog: str, + catalog_type: str, + genre: Optional[str] = None, + skip: int = 0, + limit: int = 25, +) -> list[schemas.Meta]: + """Get a list of metadata entries from MDBList with improved pagination handling""" + if not user_data.mdblist_config: + return [] + + # Extract list info from catalog ID + _, media_type, list_id = catalog.split("_", 2) + list_config = next( + ( + list_item + for list_item in user_data.mdblist_config.lists + if str(list_item.id) == list_id and list_item.catalog_type == media_type + ), + None, + ) + + if not list_config: + return [] + + meta_class = ( + MediaFusionMovieMetaData + if catalog_type == "movie" + else MediaFusionSeriesMetaData + ) + + # Initialize MDBList scraper + mdblist_scraper = await initialize_mdblist_scraper(user_data.mdblist_config.api_key) + try: + if not list_config.use_filters: + return await mdblist_scraper.get_list_items( + list_id=list_id, + media_type=media_type, + skip=skip, + limit=limit, + genre=genre, + use_filters=False, + ) + + # For filtered results, we need to maintain a window of filtered results + # that's larger than the requested page to handle pagination properly + WINDOW_SIZE = max(500, skip + limit * 2) # Maintain a larger window of results + + # Get a window of IMDb IDs from MDBList + imdb_ids = await mdblist_scraper.get_list_items( + list_id=list_id, + media_type=media_type, + skip=0, # Always start from beginning for filtered results + limit=WINDOW_SIZE, + genre=genre, + use_filters=True, + ) + + if not imdb_ids: + return [] + + # Prepare the filter pipeline + match_filter = { + "_id": {"$in": imdb_ids}, + "type": catalog_type, + "total_streams": {"$gt": 0}, + } + + # Handle nudity filter + if "Disable" not in user_data.nudity_filter: + if "Unknown" in user_data.nudity_filter: + match_filter["parent_guide_nudity_status"] = {"$exists": True} + elif user_data.nudity_filter: + match_filter["parent_guide_nudity_status"] = { + "$nin": user_data.nudity_filter + } + + # Handle certification filter + if "Disable" not in user_data.certification_filter: + cert_filters = [] + if "Unknown" in user_data.certification_filter: + cert_filters.append( + {"parent_guide_certificates": {"$exists": True, "$ne": []}} + ) + filter_values = get_filter_certification_values(user_data) + if filter_values: + cert_filters.append( + {"parent_guide_certificates": {"$nin": filter_values}} + ) + if cert_filters: + match_filter["$or"] = cert_filters + + # Use facet to get both filtered results and total count efficiently + pipeline = [ + {"$match": match_filter}, + {"$sort": {"last_stream_added": -1}}, + { + "$facet": { + "results": [{"$skip": skip}, {"$limit": limit}], + "total": [{"$count": "count"}], + }, + }, + ] + + results = await meta_class.get_motor_collection().aggregate(pipeline).to_list() + if not results: + return [] + + filtered_results = [ + schemas.Meta.model_validate(result) for result in results[0]["results"] + ] + total_count = results[0]["total"][0]["count"] if results[0]["total"] else 0 + + # If we're close to the end of our window and there might be more results, + # trigger background fetch of next batch of metadata + if total_count >= WINDOW_SIZE - limit: + # Get the next batch of IMDb IDs for background processing + next_batch_ids = await mdblist_scraper.get_list_items( + list_id=list_id, + media_type=media_type, + skip=WINDOW_SIZE, + limit=100, # Fetch next batch + genre=genre, + use_filters=True, + ) + + if next_batch_ids: + # Check which IDs are missing from our database + existing_meta_ids = set( + doc["_id"] + for doc in await meta_class.get_motor_collection() + .find({"_id": {"$in": next_batch_ids}}, {"_id": 1}) + .to_list(None) + ) + missing_ids = list(set(next_batch_ids) - existing_meta_ids) + + if missing_ids: + background_tasks.add_task( + fetch_metadata, + missing_ids, + catalog_type, + ) + + return filtered_results + + finally: + await mdblist_scraper.close() + + async def get_tv_meta_list( namespace: str, genre: Optional[str] = None, skip: int = 0, limit: int = 25 ) -> list[schemas.Meta]: @@ -1502,3 +1654,21 @@ async def update_metadata(imdb_ids: list[str], metadata_type: str): cache_keys = await REDIS_ASYNC_CLIENT.keys(f"{metadata_type}_{meta_id}_meta*") cache_keys.append(f"{metadata_type}_data:{meta_id}") await REDIS_ASYNC_CLIENT.delete(*cache_keys) + + +async def fetch_metadata(imdb_ids: list[str], metadata_type: str): + circuit_breaker = CircuitBreaker( + failure_threshold=2, recovery_timeout=5, half_open_attempts=2 + ) + + async for result in batch_process_with_circuit_breaker( + get_movie_data_by_id if metadata_type == "movie" else get_series_data_by_id, + imdb_ids, + 5, + rate_limit_delay=1, + cb=circuit_breaker, + ): + if not result: + continue + + logging.info(f"Stored metadata for {metadata_type} {result.id}") diff --git a/db/models.py b/db/models.py index c863950..fb8c6b2 100644 --- a/db/models.py +++ b/db/models.py @@ -282,10 +282,10 @@ class TorrentStreams(Document): class TVStreams(Document): meta_id: str name: str + source: str url: str | None = None ytId: str | None = None externalUrl: str | None = None - source: str behaviorHints: dict[str, Any] | None = None created_at: datetime = Field(default_factory=datetime.now) country: str | None = None diff --git a/db/schemas.py b/db/schemas.py index eb6b27f..798d6a1 100644 --- a/db/schemas.py +++ b/db/schemas.py @@ -196,6 +196,30 @@ class SortingOption(BaseModel): populate_by_name = True +class MDBListItem(BaseModel): + id: int = Field(alias="i") + title: str = Field(alias="t") + catalog_type: Literal["movie", "series"] = Field(alias="ct") + use_filters: bool = Field(default=False, alias="uf") + + @property + def catalog_id(self) -> str: + return f"mdblist_{self.catalog_type}_{self.id}" + + class Config: + extra = "ignore" + populate_by_name = True + + +class MDBListConfig(BaseModel): + api_key: str = Field(alias="ak") + lists: list[MDBListItem] = Field(default_factory=list, alias="l") + + class Config: + extra = "ignore" + populate_by_name = True + + class UserData(BaseModel): streaming_provider: StreamingProvider | None = Field(default=None, alias="sp") selected_catalogs: list[str] = Field(alias="sc", default_factory=list) @@ -237,6 +261,7 @@ class UserData(BaseModel): live_search_streams: bool = Field(default=False, alias="lss") contribution_streams: bool = Field(default=False, alias="cs") show_language_country_flag: bool = Field(default=False, alias="slcf") + mdblist_config: MDBListConfig | None = Field(default=None, alias="mdb") @field_validator("selected_resolutions", mode="after") def validate_selected_resolutions(cls, v): diff --git a/resources/html/configure.html b/resources/html/configure.html index 5ceb230..854af22 100644 --- a/resources/html/configure.html +++ b/resources/html/configure.html @@ -625,7 +625,8 @@ class="btn btn-outline-primary sort-direction-toggle px-3" data-sort-id="{{ sorting_option.key }}" data-bs-toggle="tooltip" - title="{{ tooltip_text[sorting_option.key].split('|')[1 if active_sort.direction == 'asc' else 0] }}" sorting_info="{{ tooltip_text[sorting_option.key] }}" + title="{{ tooltip_text[sorting_option.key].split('|')[1 if active_sort.direction == 'asc' else 0] }}" + sorting_info="{{ tooltip_text[sorting_option.key] }}" {% if not active_sort.found %}disabled{% endif %}> {% if active_sort.direction == 'asc' %} @@ -840,6 +841,133 @@ + +
+
MDBList Configuration
+
+ + +
+
+ +
+ +
+ + + +
+
+ Please enter a valid MDBList API Key. +
+ + To get your API key: +
    +
  1. Sign up/Login to MDBList
    2. +
  2. Go to API Key Preferences
    3. +
  3. Create a new API key if you don't have one
    4. +
+ +
+ + + +
+
+ {% if authentication_required %} @@ -921,8 +1049,100 @@ + + + + + + + + + + + @@ -930,6 +1150,7 @@ + diff --git a/resources/js/config_script.js b/resources/js/config_script.js index 456ced7..29563cc 100644 --- a/resources/js/config_script.js +++ b/resources/js/config_script.js @@ -1,6 +1,7 @@ // ---- Variables ---- const oAuthBtn = document.getElementById('oauth_btn'); let currentAuthorizationToken = null; +let mdbListUI; const servicesRequiringCredentials = ['pikpak',]; const servicesRequiringUrl = ['stremthru']; const servicesNotNeedingDebridProxy = ['stremthru']; @@ -425,6 +426,14 @@ function getUserData() { validateInput('rpdb_api_key', rpdbConfig.api_key.trim() !== ''); } + let mdblistConfig = null; + if (document.getElementById('enable_mdblist').checked) { + mdblistConfig = { + api_key: document.getElementById('mdblist_api_key').value, + lists: mdbListUI.getSelectedListsData() + }; + } + // Collect and validate other user data const maxSizeSlider = document.getElementById('max_size_slider'); const maxSizeValue = maxSizeSlider.value; @@ -489,6 +498,7 @@ function getUserData() { rpdb_config: rpdbConfig, live_search_streams: document.getElementById('liveSearchStreams').checked, contribution_streams: document.getElementById('contributionStreams').checked, + mdblist_config: mdblistConfig, }; } @@ -663,6 +673,10 @@ document.getElementById('enable_rpdb').addEventListener('change', function () { setElementDisplay('rpdb_config', this.checked ? 'block' : 'none'); }); +document.getElementById('enable_mdblist').addEventListener('change', function() { + setElementDisplay('mdblist_config', this.checked ? 'block' : 'none'); +}); + // Event listener for the slider document.getElementById('max_size_slider').addEventListener('input', updateSizeOutput); @@ -730,13 +744,14 @@ document.getElementById('copyBtn').addEventListener('click', async function (eve // ---- Initial Setup ---- document.addEventListener('DOMContentLoaded', function () { + mdbListUI = new MDBListUI(); updateProviderFields(); updateSizeOutput(); }); document.addEventListener('DOMContentLoaded', function () { - var tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]')) - var tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) { + let tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]')) + let tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) { return new bootstrap.Tooltip(tooltipTriggerEl) }); @@ -755,6 +770,7 @@ document.addEventListener('DOMContentLoaded', function () { setupPasswordToggle('webdav_password', 'toggleWebdavPassword', 'toggleWebdavPasswordIcon'); setupPasswordToggle('mediaflow_api_password', 'toggleMediaFlowPassword', 'toggleMediaFlowPasswordIcon'); setupPasswordToggle('rpdb_api_key', 'toggleRPDBApiKey', 'toggleRPDBApiKeyIcon'); + setupPasswordToggle('mdblist_api_key', 'toggleMDBListApiKey', 'toggleMDBListApiKeyIcon'); }); document.addEventListener('DOMContentLoaded', function () { diff --git a/resources/js/mdblist.js b/resources/js/mdblist.js new file mode 100644 index 0000000..2fb541f --- /dev/null +++ b/resources/js/mdblist.js @@ -0,0 +1,833 @@ +const MDBList = { + BASE_URL: 'https://api.mdblist.com', + + /** + * Makes an API request with error handling + */ + async makeRequest(endpoint, apiKey, options = {}) { + const url = endpoint.startsWith('http') ? endpoint : `${MDBList.BASE_URL}${endpoint}`; + const finalUrl = new URL(url); + finalUrl.searchParams.append('apikey', apiKey); + + try { + const response = await fetch(finalUrl, options); + + // Handle different response statuses + switch (response.status) { + case 200: + return await response.json(); + case 403: + throw new Error('Invalid API key'); + case 404: + throw new Error('Resource not found'); + case 429: + throw new Error('Rate limit exceeded'); + default: + if (!response.ok) { + throw new Error(`API request failed with status ${response.status}`); + } + } + } catch (error) { + // Convert fetch network errors to user-friendly messages + if (error.name === 'TypeError' && error.message === 'Failed to fetch') { + throw new Error('Network error - please check your connection'); + } + throw error; + } + }, + + async verifyApiKey(apiKey) { + if (!apiKey?.trim()) { + return false; + } + + try { + await MDBList.makeRequest('/user', apiKey); + return true; + } catch (error) { + console.error('API key verification failed:', error); + return false; + } + }, + + async getUserLists(apiKey) { + try { + return await MDBList.makeRequest('/lists/user', apiKey); + } catch (error) { + console.error('Error fetching user lists:', error); + throw MDBList.handleApiError(error); + } + }, + + async getTopLists(apiKey) { + try { + return await MDBList.makeRequest('/lists/top', apiKey); + } catch (error) { + console.error('Error fetching top lists:', error); + throw MDBList.handleApiError(error); + } + }, + + async searchLists(apiKey, query) { + if (!query?.trim()) { + return []; + } + + try { + const url = '/lists/search?query=' + encodeURIComponent(query.trim()); + return await MDBList.makeRequest(url, apiKey); + } catch (error) { + console.error('Error searching lists:', error); + throw MDBList.handleApiError(error); + } + }, + + async getListDetails(apiKey, listId) { + if (!listId) { + throw new Error('List ID is required'); + } + + try { + return await MDBList.makeRequest(`/lists/${listId}`, apiKey); + } catch (error) { + console.error('Error fetching list details:', error); + throw MDBList.handleApiError(error); + } + }, + + handleApiError(error) { + const errorMessages = { + 'Invalid API key': 'Your API key is invalid or has expired. Please verify your API key.', + 'Rate limit exceeded': 'You have made too many requests. Please try again later.', + 'Network error': 'Unable to connect to MDBList. Please check your internet connection.', + 'Resource not found': 'The requested list could not be found.' + }; + + const message = errorMessages[error.message] || 'An error occurred while fetching data from MDBList'; + return new Error(message); + } +}; + +// MDBList UI handler +class MDBListUI { + constructor() { + this.selectedLists = new Map(); + this.editModal = new bootstrap.Modal(document.getElementById('editListModal')); + this.apiKey = document.getElementById('mdblist_api_key').value; + this.setupEventListeners(); + this.initializeFromConfig(); + setElementDisplay('mdblist_management', this.apiKey ? 'block' : 'none'); + } + + // Helper functions for consistent ID management + generateListId(baseId, mediaType) { + return `${baseId}_${mediaType}`; + } + + parseListId(listId) { + const [baseId, mediaType] = String(listId).split('_'); + return {baseId: parseInt(baseId), mediaType}; + } + + isListAdded(baseId) { + // Check if the list is added in either movie or show format + const movieId = this.generateListId(baseId, 'movie'); + const showId = this.generateListId(baseId, 'show'); + + return this.selectedLists.has(movieId) || this.selectedLists.has(showId); + } + + initializeFromConfig() { + const configElement = document.getElementById('mdblist_configured_lists'); + if (!configElement?.value) return; + + try { + const config = JSON.parse(configElement.value); + config.forEach(list => { + const mediaType = list.catalog_type === 'series' ? 'show' : 'movie'; + const listId = this.generateListId(list.id, mediaType); + + this.selectedLists.set(listId, { + id: listId, + baseId: list.id, + title: list.title, + catalogType: list.catalog_type, + useFilters: list.use_filters, + // Preserve other properties if available + items: list.items || 0, + owner: list.owner || '', + slug: list.slug || '' + }); + }); + + this.renderSelectedLists(); + this.updateCatalogs(); + } catch (error) { + console.error('Error initializing from config:', error); + showNotification('Error loading configured lists', 'error'); + } + } + + setupEventListeners() { + // API key verification + document.getElementById('verifyMDBListApiKey')?.addEventListener('click', + () => this.verifyApiKey()); + document.getElementById('mdblist_api_key')?.addEventListener('change', + (e) => this.handleApiKeyChange(e)); + + // Tab navigation + document.querySelectorAll('button[data-bs-toggle="tab"]').forEach(tab => { + tab.addEventListener('shown.bs.tab', (e) => this.handleTabChange(e.target.id)); + }); + + // Search functionality + document.getElementById('search-lists-btn')?.addEventListener('click', + () => this.handleSearch()); + document.getElementById('list-search-input')?.addEventListener('keypress', + (e) => e.key === 'Enter' && this.handleSearch()); + + // Manual list addition + document.getElementById('add-list-btn')?.addEventListener('click', + () => this.handleManualAdd()); + + // Edit modal + document.getElementById('save-list-edit')?.addEventListener('click', + () => this.saveListEdit()); + } + + async verifyApiKey() { + const apiKeyInput = document.getElementById('mdblist_api_key'); + const apiKey = apiKeyInput.value.trim(); + + if (!apiKey) { + showNotification('Please enter an API key', 'error'); + return; + } + + showLoadingWidget('Verifying API key...'); + try { + const isValid = await MDBList.verifyApiKey(apiKey); + if (isValid) { + this.apiKey = apiKey; + setElementDisplay('mdblist_management', 'block'); + showNotification('API key verified successfully', 'success'); + await this.loadUserLists(); + } else { + showNotification('Invalid API key', 'error'); + } + } catch (error) { + showNotification(error.message, 'error'); + } finally { + hideLoadingWidget(); + } + } + + handleApiKeyChange(e) { + // Update the API key + this.apiKey = e.target?.value?.trim() || ''; + + // Toggle management section visibility based on API key presence + setElementDisplay('mdblist_management', this.apiKey ? 'block' : 'none'); + + // Clear lists when API key is removed + if (!this.apiKey) { + this.selectedLists.clear(); + this.renderSelectedLists(); + this.updateCatalogs(); + } + } + + async handleTabChange(tabId) { + if (!this.apiKey) return; + + const tabContent = { + 'my-lists-tab': () => this.loadUserLists(), + 'top-lists-tab': () => this.loadTopLists() + }; + + await (tabContent[tabId] || (() => { + }))(); + } + + async loadLists(fetchFunction, containerId, loadingId) { + const container = document.getElementById(containerId); + document.getElementById(loadingId); + if (!container) return; + + setElementDisplay(loadingId, 'block'); + container.innerHTML = ''; + + try { + const lists = await fetchFunction(this.apiKey); + setElementDisplay(loadingId, 'none'); + + if (lists?.length > 0) { + lists.forEach(list => this.renderListItem(container, list)); + } else { + container.innerHTML = '
No lists found
'; + } + } catch (error) { + console.error('Error loading lists:', error); + container.innerHTML = '
Error loading lists
'; + setElementDisplay(loadingId, 'none'); + } + } + + async loadUserLists() { + await this.loadLists( + MDBList.getUserLists, + 'my-lists-content', + 'my-lists-loading' + ); + } + + async loadTopLists() { + await this.loadLists( + MDBList.getTopLists, + 'top-lists-content', + 'top-lists-loading' + ); + } + + async handleSearch() { + const query = document.getElementById('list-search-input')?.value.trim(); + if (!query) return; + + const container = document.getElementById('search-lists-content'); + if (!container) return; + + container.innerHTML = '
'; + + try { + const results = await MDBList.searchLists(this.apiKey, query); + container.innerHTML = ''; + + if (results?.length > 0) { + results.forEach(list => this.renderListItem(container, list)); + } else { + container.innerHTML = '
No lists found
'; + } + } catch (error) { + console.error('Error searching lists:', error); + container.innerHTML = '
Error searching lists
'; + } + } + + renderListItem(container, list) { + const template = document.getElementById('list-item-template'); + if (!template) return; + + const listItem = template.content.cloneNode(true); + const listElement = listItem.querySelector('.list-group-item'); + + // Set basic information + listItem.querySelector('.list-title').textContent = list.name; + listElement.dataset.listId = list.id; + + // Set counts and badges + this.setItemBadges(listItem, list); + + // Set link and owner info + this.setListLinks(listItem, list); + + // Setup add button + const addBtn = listItem.querySelector('.add-list-btn'); + if (addBtn) { + addBtn.addEventListener('click', () => this.handleAddList(list)); + if (this.isListAdded(list.id)) { + this.markListAsAdded(addBtn); + } + } + + container.appendChild(listItem); + } + + setItemBadges(listItem, list) { + // Item count badge + listItem.querySelector('.item-count').innerHTML = ` + + ${list.items || 0} + `; + + // Likes badge + listItem.querySelector('.like-count').innerHTML = ` + + ${list.likes || 0} + `; + + // Media type badge + const mediaTypeEl = listItem.querySelector('.media-type'); + const mediaIcon = list.mediatype === 'show' ? 'tv' : + list.mediatype === 'movie' ? 'film' : 'collection-play'; + const mediaText = list.mediatype === 'show' ? 'Series' : + list.mediatype === 'movie' ? 'Movies' : 'Mixed'; + + mediaTypeEl.innerHTML = ` + + ${mediaText} + `; + } + + setListLinks(listItem, list) { + const linkView = listItem.querySelector('.list-link'); + const ownerEl = listItem.querySelector('.list-owner'); + + if (list.user_name && list.slug) { + ownerEl.textContent = `by ${list.user_name}`; + linkView.href = `https://mdblist.com/lists/${list.user_name}/${list.slug}`; + } else { + ownerEl.textContent = 'My List'; + linkView.href = 'https://mdblist.com/mylists/'; + } + + linkView.title = 'View on MDBList'; + } + + markListAsAdded(button) { + button.classList.replace('btn-outline-primary', 'btn-success'); + button.innerHTML = ' Added'; + button.disabled = true; + } + + async handleManualAdd() { + const input = document.getElementById('list-url-input'); + if (!input?.value) return; + + const urlOrId = input.value.trim(); + showLoadingWidget('Fetching list details...'); + + try { + // Extract list ID from URL if needed + const listId = urlOrId.includes('mdblist.com/lists/') ? + urlOrId.split('mdblist.com/lists/')[1].replace(/\/$/, '') : + urlOrId; + + const listDetails = await MDBList.getListDetails(this.apiKey, listId); + if (listDetails) { + this.handleAddList(listDetails); + } else { + throw new Error('Failed to fetch list details'); + } + } catch (error) { + console.error('Error adding list:', error); + showNotification('Failed to fetch list details', 'error'); + } finally { + hideLoadingWidget(); + } + } + + editList(list) { + const modal = document.getElementById('editListModal'); + if (!modal) return; + + // Populate form fields + const fields = { + 'edit-list-title': list.title || list.name, + 'edit-list-id': list.id, + 'edit-list-original-id': list.baseId || list.id + }; + + Object.entries(fields).forEach(([id, value]) => { + const element = modal.querySelector(`#${id}`); + if (element) element.value = value; + }); + + // Handle checkboxes + const movieCheck = modal.querySelector('#edit-list-type-movie'); + const showCheck = modal.querySelector('#edit-list-type-show'); + const useFiltersCheck = modal.querySelector('#edit-list-use-filters'); + + if (movieCheck && showCheck) { + const isMovie = list.catalogType === 'movie'; + const isShow = list.catalogType === 'series'; + + movieCheck.checked = isMovie || list.allowBothTypes; + showCheck.checked = isShow || list.allowBothTypes; + + // Handle disabled states + const shouldDisable = !list.allowBothTypes; + movieCheck.disabled = shouldDisable && !isMovie; + showCheck.disabled = shouldDisable && !isShow; + } + + if (useFiltersCheck) { + useFiltersCheck.checked = list.useFilters || false; + } + + this.editModal.show(); + } + + + handleAddList(list) { + const lists = Array.isArray(list) ? list : [list]; + let listId; + let originalId; + + // Group lists by mediatype + const movieList = lists.find(l => l.mediatype === 'movie'); + const showList = lists.find(l => l.mediatype === 'show'); + + if (!movieList && !showList) { + // If no specific mediatype, show edit modal to choose + listId = lists[0].id; + originalId = lists[0].id; + this.editList({ + id: lists[0].id, + name: lists[0].name, + originalTitle: lists[0].name, + items: lists[0].items, + owner: lists[0].user_name, + slug: lists[0].slug, + likes: lists[0].likes, + allowBothTypes: true + }); + } else { + // Add specific media type lists + if (movieList) { + listId = this.generateListId(movieList.id, 'movie'); + originalId = movieList.id; + this.addListToSelected(listId, { + id: listId, + baseId: movieList.id, + title: movieList.name, + catalogType: 'movie', + items: movieList.items, + owner: movieList.user_name, + slug: movieList.slug, + likes: movieList.likes, + useFilters: false // default value + }); + } + if (showList) { + listId = this.generateListId(showList.id, 'show'); + originalId = showList.id; + this.addListToSelected(listId, { + id: listId, + baseId: showList.id, + title: showList.name, + catalogType: 'series', + items: showList.items, + owner: showList.user_name, + slug: showList.slug, + likes: showList.likes, + useFilters: false // default value + }); + } + } + + // Update UI immediately + const addBtn = document.querySelector(`[data-list-id="${originalId}"] .add-list-btn`); + if (addBtn) { + this.markListAsAdded(addBtn); + } + } + + addListToSelected(listId, listData) { + // Validate required data + if (!listId || !listData.title || !listData.catalogType) { + console.error('Invalid list data:', listData); + return; + } + + // Store with consistent format + this.selectedLists.set(listId, { + ...listData, + id: listId, + baseId: this.parseListId(listId).baseId, + useFilters: listData.useFilters || false, + items: listData.items || 0 + }); + + // Update UI + this.renderSelectedLists(); + this.updateCatalogs(); + } + + saveListEdit() { + const modal = document.getElementById('editListModal'); + if (!modal) return; + + // Get form values + const baseId = parseInt(modal.querySelector('#edit-list-original-id').value); + const title = modal.querySelector('#edit-list-title').value; + const movieSelected = modal.querySelector('#edit-list-type-movie').checked; + const showSelected = modal.querySelector('#edit-list-type-show').checked; + const useFilters = modal.querySelector('#edit-list-use-filters').checked; + + // Store current catalog positions + const currentPositions = this.getCurrentCatalogPositions(); + + // Remove existing entries for this base ID + this.removeList(baseId); + + // Add new entries based on selection + if (movieSelected) { + const movieId = this.generateListId(baseId, 'movie'); + this.addListToSelected(movieId, { + baseId, + title, + catalogType: 'movie', + useFilters + }); + } + + if (showSelected) { + const showId = this.generateListId(baseId, 'show'); + this.addListToSelected(showId, { + baseId, + title, + catalogType: 'series', + useFilters + }); + } + + // Restore catalog positions + this.restoreCatalogPositions(currentPositions); + this.editModal.hide(); + } + + removeList(listId) { + this.selectedLists.delete(listId); + this.renderSelectedLists(); + this.updateCatalogs(); + + // Update the add button state in the list view + const addBtn = document.querySelector(`[data-list-id="${this.parseListId(listId).baseId}"] .add-list-btn`); + if (addBtn) { + addBtn.classList.replace('btn-success', 'btn-outline-primary'); + addBtn.innerHTML = ' Add'; + addBtn.disabled = false; + } + } + + getCurrentCatalogPositions() { + const positions = new Map(); + const catalogs = document.querySelectorAll('.draggable-catalog'); + + catalogs.forEach((catalog, index) => { + const checkbox = catalog.querySelector('input[type="checkbox"]'); + positions.set(catalog.dataset.id, { + index, + checked: checkbox?.checked ?? true, + isGeneral: !catalog.dataset.id.startsWith('mdblist_') + }); + }); + + return positions; + } + + restoreCatalogPositions(positions) { + const container = document.getElementById('catalogs'); + if (!container) return; + + const currentCatalogs = Array.from(container.querySelectorAll('.draggable-catalog')); + + // Sort catalogs based on their stored positions + currentCatalogs.sort((a, b) => { + const posA = positions.get(a.dataset.id); + const posB = positions.get(b.dataset.id); + + // Keep non-MDBList catalogs in their original position + if (posA?.isGeneral && !posB?.isGeneral) return -1; + if (!posA?.isGeneral && posB?.isGeneral) return 1; + + // Sort based on stored positions + if (!posA) return 1; + if (!posB) return -1; + return posA.index - posB.index; + }); + + // Reorder catalogs + currentCatalogs.forEach(catalog => { + container.appendChild(catalog); + + // Restore checkbox state + const position = positions.get(catalog.dataset.id); + if (position) { + const checkbox = catalog.querySelector('input[type="checkbox"]'); + if (checkbox) checkbox.checked = position.checked; + } + }); + } + + updateCatalogs() { + const catalogsContainer = document.getElementById('catalogs'); + if (!catalogsContainer) return; + + // Get current positions and states + const currentState = this.getCurrentCatalogPositions(); + const currentCatalogs = Array.from(catalogsContainer.children); + + // Store all existing catalogs with their data + const existingCatalogs = new Map( + currentCatalogs.map(catalog => [ + catalog.dataset.id, + { + element: catalog, + isMDBList: catalog.dataset.id.startsWith('mdblist_') + } + ]) + ); + + // Clear container + catalogsContainer.innerHTML = ''; + + // Create new MDBList catalogs + this.selectedLists.forEach(list => { + const catalogId = `mdblist_${list.catalogType}_${list.baseId}`; + if (!existingCatalogs.has(catalogId)) { + existingCatalogs.set(catalogId, { + element: this.createCatalogElement(catalogId, list.title), + isMDBList: true + }); + } + }); + + // Sort catalogs based on their previous positions + const sortedCatalogIds = Array.from(currentState.keys()) + .sort((a, b) => { + const posA = currentState.get(a); + const posB = currentState.get(b); + return posA.index - posB.index; + }); + + // Add catalogs in the correct order + sortedCatalogIds.forEach(catalogId => { + const catalogInfo = existingCatalogs.get(catalogId); + if (!catalogInfo) return; // Skip if catalog no longer exists + + // Skip MDBList catalogs that aren't in selectedLists + if (catalogInfo.isMDBList) { + const [, type, baseId] = catalogId.split('_'); + const listId = this.generateListId(baseId, type === 'series' ? 'show' : 'movie'); + if (!this.selectedLists.has(listId)) return; + } + + // Clone the element to avoid reference issues + const newElement = catalogInfo.element.cloneNode(true); + + // Restore checkbox state + const state = currentState.get(catalogId); + if (state) { + const checkbox = newElement.querySelector('input[type="checkbox"]'); + if (checkbox) checkbox.checked = state.checked; + } + + catalogsContainer.appendChild(newElement); + }); + + // Add any new MDBList catalogs that weren't in the previous state + this.selectedLists.forEach(list => { + const catalogId = `mdblist_${list.catalogType}_${list.baseId}`; + if (!currentState.has(catalogId)) { + const catalogDiv = this.createCatalogElement(catalogId, list.title); + catalogsContainer.appendChild(catalogDiv); + } + }); + } + + createCatalogElement(catalogId, title) { + const div = document.createElement('div'); + div.className = 'col-12 col-md-6 col-lg-4 draggable-catalog'; + div.dataset.id = catalogId; + + div.innerHTML = ` +
+ + +
+ `; + + return div; + } + + renderSelectedLists() { + const container = document.getElementById('selected-lists'); + if (!container) return; + + container.innerHTML = ''; + + // Render each list separately + this.selectedLists.forEach(list => { + const template = document.getElementById('selected-list-item-template'); + if (!template) return; + + const item = template.content.cloneNode(true); + + // Set list title + item.querySelector('.list-title').textContent = list.title; + + // Set media type + const mediaType = item.querySelector('.media-type'); + mediaType.textContent = list.catalogType === 'series' ? 'Series' : 'Movies'; + + // Set filters status + const useFilters = item.querySelector('.use-filters'); + useFilters.textContent = list.useFilters ? 'Filters enabled' : 'No filters'; + + // Setup buttons with individual list data + const editBtn = item.querySelector('.edit-btn'); + const removeBtn = item.querySelector('.remove-btn'); + + editBtn?.addEventListener('click', () => this.editList({ + ...list, + allowBothTypes: false // Since we're managing them separately + })); + + removeBtn?.addEventListener('click', () => this.removeList(list.id)); + + container.appendChild(item); + }); + + + } + + getSelectedListsData() { + // Group by baseId to combine movie/show pairs + const groupedLists = new Map(); + + this.selectedLists.forEach(list => { + const {baseId} = this.parseListId(list.id); + if (!groupedLists.has(baseId)) { + groupedLists.set(baseId, { + id: baseId, + title: list.title, + catalog_types: new Set(), + use_filters: list.useFilters + }); + } + + const group = groupedLists.get(baseId); + group.catalog_types.add(list.catalogType); + group.use_filters = group.use_filters || list.useFilters; + }); + + // Convert to array format + return Array.from(groupedLists.values()).flatMap(group => { + const types = Array.from(group.catalog_types); + if (types.length === 1) { + // Single type list + return [{ + id: group.id, + title: group.title, + catalog_type: types[0], + use_filters: group.use_filters + }]; + } else { + // Split into separate entries for each type + return types.map(type => ({ + id: group.id, + title: group.title, + catalog_type: type, + use_filters: group.use_filters + })); + } + }); + } +} \ No newline at end of file diff --git a/resources/templates/manifest.json.j2 b/resources/templates/manifest.json.j2 index 133d15e..ac40d54 100644 --- a/resources/templates/manifest.json.j2 +++ b/resources/templates/manifest.json.j2 @@ -94,8 +94,19 @@ "arabic_series": {"type": "series", "name": "Arabic Series"} } %} {% for catalog_id in selected_catalogs %} - {% set catalog = catalog_definitions[catalog_id] %} - {% if catalog %} + {% set catalog = catalog_definitions[catalog_id] | default(mdblist_data[catalog_id]) %} + {% if "mdblist" in catalog_id %} + {{ comma() }} + { + "id": "{{ catalog_id }}", + "type": "{{ catalog.catalog_type }}", + "name": "{{ catalog.title }}", + "extra": [ + {"name": "skip", "isRequired": false}, + {"name": "genre", "isRequired": false, "options": ["action","anime","comedy","crime","documentary","drama","family","fantasy","history","holiday","horror","music","musical","mystery","science-fiction","short","sporting-event","superhero","suspense","thriller","war","western","animation","adventure","romance","reality","soap","news","talk-show","biography","sci-fi","sport","adult","film-noir","reality-tv","game-show","special-interest","children","home-and-garden","tv-movie","sports","eastern","disaster","donghua","sci-fi-fantasy","action-adventure","talk","war-politics","kids"]} + ] + } + {% elif catalog %} {% set genres = catalog.genres | default(genres[catalog.type]) | default([]) %} {{ comma() }} { diff --git a/scrapers/mdblist.py b/scrapers/mdblist.py new file mode 100644 index 0000000..52ff28f --- /dev/null +++ b/scrapers/mdblist.py @@ -0,0 +1,185 @@ +import json +import logging +from typing import Optional, Dict, List + +import httpx + +from db import schemas +from db.config import settings +from db.redis_database import REDIS_ASYNC_CLIENT + + +class MDBListScraper: + def __init__(self, api_key: str): + self.api_key = api_key + self.base_url = "https://api.mdblist.com" + self.client = httpx.AsyncClient(proxy=settings.requests_proxy_url, timeout=30.0) + + async def close(self): + await self.client.aclose() + + async def _fetch_list( + self, + list_id: str, + limit: int = 100, + offset: int = 0, + genre: Optional[str] = None, + ) -> Optional[Dict]: + """Fetch a list from MDBList API""" + params = { + "apikey": self.api_key, + "limit": limit, + "offset": offset, + "append_to_response": "genre", + } + if genre: + params["filter_genre"] = genre + + cache_key = ( + f"mdblist:list:{list_id}:limit_{limit}:offset_{offset}:{genre or 'all'}" + ) + cached_data = await REDIS_ASYNC_CLIENT.get(cache_key) + + if cached_data: + return json.loads(cached_data) + + try: + response = await self.client.get( + f"{self.base_url}/lists/{list_id}/items", params=params + ) + if response.status_code == 200: + # Cache for 15 minutes + await REDIS_ASYNC_CLIENT.set(cache_key, response.text, ex=900) + return response.json() + logging.error(f"Failed to fetch MDBList data: {response.status_code}") + return None + except Exception as e: + logging.error(f"Error fetching MDBList data: {e}") + return None + + def _convert_to_meta(self, item: Dict, media_type: str) -> schemas.Meta: + """Convert MDBList item to Meta object""" + genres = item.get("genre", []) + if genres and genres[0] is None: + # clean up genre data + genres = None + return schemas.Meta.model_validate( + { + "_id": item["imdb_id"], + "type": media_type, + "title": item["title"], + "year": item["release_year"], + "genres": genres, + "poster": f"{settings.poster_host_url}/poster/{media_type}/{item['imdb_id']}.jpg", + } + ) + + async def _fetch_and_process_batch( + self, + list_id: str, + media_type: str, + offset: int, + limit: int, + skip: int, + genre: Optional[str], + use_filters: bool = False, + ) -> List[schemas.Meta] | List[str]: + """Helper method to fetch and process a batch of results""" + data = await self._fetch_list(list_id, 100, offset, genre) + if not data: + return [] + + items = data.get("movies" if media_type == "movie" else "shows", []) + + if not use_filters: + # Convert directly to Meta objects + meta_list = [ + self._convert_to_meta(item, media_type) + for item in items + if item.get("imdb_id", "").startswith("tt") + ] + # Calculate the slice we need from this batch + start_idx = skip % 100 + end_idx = start_idx + limit + return meta_list[start_idx:end_idx] + + return [ + item["imdb_id"] + for item in items + if item.get("imdb_id", "").startswith("tt") + ] + + async def get_list_items( + self, + list_id: str, + media_type: str, + skip: int = 0, + limit: int = 25, + genre: Optional[str] = None, + use_filters: bool = True, + ) -> List[schemas.Meta] | List[str]: + """ + Get items from a MDBList list with pagination support. + For filtered results, keeps fetching until we have enough items after filtering. + """ + if not use_filters: + # Direct return for unfiltered results + fetch_limit = 100 + offset = (skip // fetch_limit) * fetch_limit + return await self._fetch_and_process_batch( + list_id, media_type, offset, limit, skip, genre, use_filters=False + ) + + # For filtered results, we need to handle pagination differently + cache_key = f"mdblist:filtered:{list_id}:genre_{genre or 'all'}" + cached_filtered_ids = await REDIS_ASYNC_CLIENT.lrange(cache_key, 0, -1) + + if cached_filtered_ids: + # Use cached filtered results + start_idx = skip + end_idx = skip + limit + return [ + cached_id.decode() + for cached_id in cached_filtered_ids[start_idx:end_idx] + ] + + # No cache - need to fetch and filter + all_imdb_ids = [] + offset = 0 + batch_size = 100 + + while len(all_imdb_ids) < skip + limit: + batch = await self._fetch_list(list_id, batch_size, offset, genre) + if not batch: + break + + items = batch.get("movies" if media_type == "movie" else "shows", []) + if not items: + break + + new_ids = [ + item["imdb_id"] + for item in items + if item.get("imdb_id", "").startswith("tt") + ] + all_imdb_ids.extend(new_ids) + + if len(items) < batch_size: # No more results available + break + + offset += batch_size + + # Cache the full result for 15 minutes + if all_imdb_ids: + pipeline = await REDIS_ASYNC_CLIENT.pipeline() + pipeline.delete(cache_key) + pipeline.rpush(cache_key, *all_imdb_ids) + pipeline.expire(cache_key, 900) # 15 minutes + await pipeline.execute() + + return all_imdb_ids[skip : skip + limit] + + +async def initialize_mdblist_scraper(api_key: str) -> MDBListScraper: + """Initialize MDBList scraper with API key""" + return MDBListScraper(api_key) diff --git a/scrapers/rpdb.py b/scrapers/rpdb.py index 7e055c5..879c976 100644 --- a/scrapers/rpdb.py +++ b/scrapers/rpdb.py @@ -20,6 +20,9 @@ async def check_rpdb_poster_availability(rpdb_poster_url: str) -> bool: ) as client: response = await client.head(rpdb_poster_url) return response.status_code == 200 + except httpx.TimeoutException as exc: + logging.error(f"Timeout for {rpdb_poster_url} - {exc}") + return False except httpx.HTTPError as exc: logging.error(f"HTTP Exception for {exc.request.url} - {exc}") return False diff --git a/scrapers/scraper_tasks.py b/scrapers/scraper_tasks.py index 45bcb80..e0ac2da 100644 --- a/scrapers/scraper_tasks.py +++ b/scrapers/scraper_tasks.py @@ -256,7 +256,7 @@ class MetadataFetcher: break except Exception as e: - logger.error(f"Error fetching from {source.value}: {e}") + logger.exception(f"Error fetching from {source.value}: {e}") continue return metadata diff --git a/utils/parser.py b/utils/parser.py index e177119..39ec76d 100644 --- a/utils/parser.py +++ b/utils/parser.py @@ -581,6 +581,15 @@ async def generate_manifest(user_data: UserData, genres: dict) -> dict: if user_data.mediaflow_config: addon_name += " 🕵🏼‍♂️" + mdblist_data = {} + if user_data.mdblist_config: + mdblist_data = { + f"mdblist_{mdblist.catalog_type}_{mdblist.id}": mdblist.model_dump( + include={"title", "catalog_type"} + ) + for mdblist in user_data.mdblist_config.lists + } + manifest_data = { "addon_name": addon_name, "version": settings.version, @@ -594,6 +603,7 @@ async def generate_manifest(user_data: UserData, genres: dict) -> dict: "enable_watchlist_catalogs": enable_watchlist_catalogs, "selected_catalogs": user_data.selected_catalogs, "genres": genres, + "mdblist_data": mdblist_data, } manifest_json = MANIFEST_TEMPLATE.render(manifest_data) diff --git a/utils/validation_helper.py b/utils/validation_helper.py index d2f1cb4..cff062b 100644 --- a/utils/validation_helper.py +++ b/utils/validation_helper.py @@ -346,3 +346,16 @@ async def validate_rpdb_token(user_data: schemas.UserData) -> dict: url=validation_url, invalid_creds_message="Invalid RPDB API Key. Please check your RPDB API Key.", ) + + +async def validate_mdblist_token(user_data: schemas.UserData) -> dict: + if not user_data.mdblist_config: + return {"status": "success", "message": "MDBList is not enabled."} + + validation_url = ( + f"https://api.mdblist.com/user?apikey={user_data.mdblist_config.api_key}" + ) + return await validate_service( + url=validation_url, + invalid_creds_message="Invalid MDBList API Key. Please check your MDBList API Key.", + )