From 95db6bf20790c576de3ed02892e7659af0124d4c Mon Sep 17 00:00:00 2001 From: mhdzumair Date: Thu, 8 Feb 2024 12:55:11 +0530 Subject: [PATCH] refactoring and bugfixes --- api/main.py | 15 +++----------- db/config.py | 4 ++-- db/crud.py | 33 +++++++++++++++++++------------ scrappers/helpers.py | 6 ++++++ scrappers/prowlarr.py | 16 +++++++++++---- scrappers/torrentio.py | 45 ++++++++++++++++++++++++------------------ 6 files changed, 69 insertions(+), 50 deletions(-) diff --git a/api/main.py b/api/main.py index a38bbf9..438ad6e 100644 --- a/api/main.py +++ b/api/main.py @@ -85,24 +85,15 @@ async def start_scheduler(): scheduler.add_job( crud.delete_search_history, CronTrigger(day="*/1"), name="delete_search_history" ) - - if settings.enable_scrapper: - scheduler.add_job( - tamil_blasters.run_schedule_scrape, - CronTrigger(hour="*/6"), - name="tamil_blasters", - ) - scheduler.add_job( - tamilmv.run_schedule_scrape, CronTrigger(hour="*/3"), name="tamilmv" - ) - scheduler.start() app.state.scheduler = scheduler @app.on_event("shutdown") async def stop_scheduler(): - if settings.enable_scrapper: + try: app.state.scheduler.shutdown(wait=False) + except AttributeError: + pass @app.on_event("shutdown") diff --git a/db/config.py b/db/config.py index b86da38..f98a2df 100644 --- a/db/config.py +++ b/db/config.py @@ -8,11 +8,11 @@ class Settings(BaseSettings): secret_key: str host_url: str = "https://mediafusion.fun" logging_level: str = "INFO" - enable_scrapper: bool = False - enable_search_scrapper: bool = False + enable_tamilmv_search_scrapper: bool = False scrapper_proxy_url: str | None = None torrentio_url: str = "https://torrentio.strem.fun" is_scrap_from_torrentio: bool = False + torrentio_search_interval_days: int = 3 premiumize_oauth_client_id: str | None = None premiumize_oauth_client_secret: str | None = None prowlarr_url: str = "http://prowlarr-service:9696" diff --git a/db/crud.py b/db/crud.py index 6325b01..ea53d54 100644 --- a/db/crud.py +++ b/db/crud.py @@ -129,13 +129,16 @@ async def get_movie_streams( .to_list() ) - if settings.is_scrap_from_torrentio: - streams = await get_streams_from_torrentio( - user_data, streams, video_id, "movie" - ) - if settings.prowlarr_api_key: + if video_id.startswith("tt"): if ( - video_id.startswith("tt") + settings.is_scrap_from_torrentio + and "torrentio_streams" in user_data.selected_catalogs + ): + streams = await get_streams_from_torrentio( + redis, streams, video_id, "movie" + ) + if ( + settings.prowlarr_api_key and "prowlarr_streams" in user_data.selected_catalogs ): movie_metadata = await get_movie_data_by_id(video_id, False) @@ -168,13 +171,17 @@ async def get_series_streams( .to_list() ) - if settings.is_scrap_from_torrentio: - streams = await get_streams_from_torrentio( - user_data, streams, video_id, "series", season, episode - ) - if settings.prowlarr_api_key: + if video_id.startswith("tt"): if ( - video_id.startswith("tt") + settings.is_scrap_from_torrentio + and "torrentio_streams" in user_data.selected_catalogs + ): + streams = await get_streams_from_torrentio( + redis, streams, video_id, "series", season, episode + ) + + if ( + settings.prowlarr_api_key and "prowlarr_streams" in user_data.selected_catalogs ): series_metadata = await get_series_data_by_id(video_id, False) @@ -464,7 +471,7 @@ async def save_series_metadata(metadata: dict): async def process_search_query(search_query: str, catalog_type: str) -> dict: - if settings.enable_search_scrapper and catalog_type in ["movie", "series"]: + if settings.enable_tamilmv_search_scrapper and catalog_type in ["movie", "series"]: # check if the search query is already searched in for last 24 hours or not last_search = await models.SearchHistory.find_one( { diff --git a/scrappers/helpers.py b/scrappers/helpers.py index f64cded..cc1921a 100644 --- a/scrappers/helpers.py +++ b/scrappers/helpers.py @@ -130,6 +130,9 @@ def get_scrapper_config(site_name: str, get_key: str) -> dict: @dramatiq.actor(time_limit=30 * 60 * 1000) async def update_torrent_movie_streams_metadata(info_hashes: list[str]): """Update torrent streams metadata.""" + if not info_hashes: + return + streams_metadata = await info_hashes_to_torrent_metadata(info_hashes, []) for stream_metadata in streams_metadata: @@ -151,6 +154,9 @@ async def update_torrent_movie_streams_metadata(info_hashes: list[str]): @dramatiq.actor(time_limit=30 * 60 * 1000) async def update_torrent_series_streams_metadata(info_hashes: list[str]): """Update torrent streams metadata.""" + if not info_hashes: + return + streams_metadata = await info_hashes_to_torrent_metadata(info_hashes, []) for stream_metadata in streams_metadata: diff --git a/scrappers/prowlarr.py b/scrappers/prowlarr.py index a45b9b5..05537c3 100644 --- a/scrappers/prowlarr.py +++ b/scrappers/prowlarr.py @@ -7,7 +7,7 @@ import dramatiq import httpx from redis.asyncio import Redis from thefuzz import fuzz -from torf import Magnet +from torf import Magnet, MagnetError from db.config import settings from db.models import TorrentStreams, Season, Episode @@ -240,7 +240,13 @@ async def prowlarr_data_parser(meta_data: dict) -> tuple[dict, bool]: ) except Exception as e: if meta_data.get("magnetUrl"): - magnet = Magnet.from_string(meta_data.get("magnetUrl")) + try: + magnet = Magnet.from_string(meta_data.get("magnetUrl")) + except MagnetError: + logging.error( + f"Error parsing {meta_data.get('indexer')} magnet link: {meta_data.get('magnetUrl')}", + ) + return {}, False torrent_data = { "info_hash": magnet.infohash, "announce_list": magnet.tr, @@ -518,7 +524,8 @@ async def parse_and_store_movie_stream_data( ] # Continue to use background tasks for demagnetization - update_torrent_movie_streams_metadata.send(info_hashes) + if info_hashes: + update_torrent_movie_streams_metadata.send(info_hashes) return streams @@ -563,7 +570,8 @@ async def parse_and_store_series_stream_data( ] # Use background tasks for metadata updates - update_torrent_series_streams_metadata.send(info_hashes) + if info_hashes: + update_torrent_series_streams_metadata.send(info_hashes) return streams diff --git a/scrappers/torrentio.py b/scrappers/torrentio.py index e40f53f..c9d2e89 100644 --- a/scrappers/torrentio.py +++ b/scrappers/torrentio.py @@ -5,6 +5,7 @@ from os import path import PTN import httpx +from redis.asyncio import Redis from db.config import settings from db.models import TorrentStreams, Season, Episode @@ -19,30 +20,34 @@ from utils.validation_helper import is_video_file async def get_streams_from_torrentio( - user_data: UserData, + redis: Redis, streams: list[TorrentStreams], video_id: str, catalog_type: str, season: int = None, episode: int = None, ): - if video_id.startswith("tt") and "torrentio_streams" in user_data.selected_catalogs: - last_stream = next( - (stream for stream in streams if stream.source == "Torrentio"), None + cache_key = f"{catalog_type}_{video_id}_{season}_{episode}_torrentio_streams" + cached_data = await redis.get(cache_key) + if cached_data: + return streams + + if catalog_type == "movie": + streams.extend(await scrap_movie_streams_from_torrentio(video_id, catalog_type)) + elif catalog_type == "series": + streams.extend( + await scrap_series_streams_from_torrentio( + video_id, catalog_type, season, episode + ) ) - if last_stream is None or last_stream.updated_at < datetime.now() - timedelta( - days=3 - ): - if catalog_type == "movie": - streams.extend( - await scrap_movie_streams_from_torrentio(video_id, catalog_type) - ) - elif catalog_type == "series": - streams.extend( - await scrap_series_streams_from_torrentio( - video_id, catalog_type, season, episode - ) - ) + + # Cache the data for 24 hours + await redis.set( + cache_key, + "True", + ex=int(timedelta(days=settings.torrentio_search_interval_days).total_seconds()), + ) + return streams @@ -156,7 +161,8 @@ async def store_and_parse_movie_stream_data( if torrent_stream.filename is None: info_hashes.append(stream["infoHash"]) - update_torrent_movie_streams_metadata.send(info_hashes) + if info_hashes: + update_torrent_movie_streams_metadata.send(info_hashes) return streams @@ -255,7 +261,8 @@ async def store_and_parse_series_stream_data( if episode_item and episode_item.size is None: info_hashes.append(stream["infoHash"]) - update_torrent_series_streams_metadata.send(info_hashes) + if info_hashes: + update_torrent_series_streams_metadata.send(info_hashes) return streams