From 7df0bdd23f1971ebc20703fb91629d7069c8e078 Mon Sep 17 00:00:00 2001 From: mhdzumair Date: Sat, 14 Dec 2024 21:40:45 +0530 Subject: [PATCH] Add Movies/TV TGX spider and scheduling support Introduced `MoviesTVTgxSpider` for parsing TGx movie/TV torrents and added relevant pipelines for processing. Enhanced scheduler, constants, and configurations to support the new functionality. Made updates to improve torrent detail parsing and handle dynamic content loading using Playwright. --- api/scheduler.py | 12 ++ db/config.py | 2 + mediafusion_scrapy/pipelines/__init__.py | 3 +- .../pipelines/movie_tv_parser_pipeline.py | 62 +++++++ .../pipelines/torrent_parser_pipeline.py | 5 +- mediafusion_scrapy/spiders/tgx.py | 152 +++++++++++++++--- resources/templates/manifest.json.j2 | 2 + utils/const.py | 2 + 8 files changed, 216 insertions(+), 24 deletions(-) create mode 100644 mediafusion_scrapy/pipelines/movie_tv_parser_pipeline.py diff --git a/api/scheduler.py b/api/scheduler.py index c0ed1f9..4e168fe 100644 --- a/api/scheduler.py +++ b/api/scheduler.py @@ -187,6 +187,18 @@ def setup_scheduler(scheduler: AsyncIOScheduler): }, ) + if not settings.disable_movies_tv_tgx_scheduler: + scheduler.add_job( + run_spider.send, + CronTrigger.from_crontab(settings.movies_tv_tgx_scheduler_crontab), + name="movies_tv_tgx", + kwargs={ + "spider_name": "movies_tv_tgx", + "crontab_expression": settings.movies_tv_tgx_scheduler_crontab, + "scrape_all": "false", + }, + ) + # Schedule the feed scraper if not settings.disable_prowlarr_feed_scraper: scheduler.add_job( diff --git a/db/config.py b/db/config.py index c530874..05cf153 100644 --- a/db/config.py +++ b/db/config.py @@ -139,6 +139,8 @@ class Settings(BaseSettings): disable_wwe_tgx_scheduler: bool = False ufc_tgx_scheduler_crontab: str = "30 */3 * * *" disable_ufc_tgx_scheduler: bool = False + movies_tv_tgx_scheduler_crontab: str = "0 * * * *" + disable_movies_tv_tgx_scheduler: bool = False prowlarr_feed_scraper_crontab: str = "0 */3 * * *" disable_prowlarr_feed_scraper: bool = False cleanup_expired_scraper_task_crontab: str = "0 * * * *" diff --git a/mediafusion_scrapy/pipelines/__init__.py b/mediafusion_scrapy/pipelines/__init__.py index bb6ec6f..ae6be31 100644 --- a/mediafusion_scrapy/pipelines/__init__.py +++ b/mediafusion_scrapy/pipelines/__init__.py @@ -3,6 +3,7 @@ from .duplicates_pipeline import TorrentDuplicatesPipeline from .formula_parser_pipeline import FormulaParserPipeline from .live_stream_resolver_pipeline import LiveStreamResolverPipeline from .moto_gp_parser_pipeline import MotoGPParserPipeline +from .movie_tv_parser_pipeline import MovieTVParserPipeline from .redis_cache_pipeline import RedisCacheURLPipeline from .sport_video_parser_pipeline import SportVideoParserPipeline from .sports_parser_pipeline import UFCParserPipeline, WWEParserPipeline @@ -19,7 +20,6 @@ from .torrent_parser_pipeline import ( TorrentDownloadAndParsePipeline, ) - __all__ = [ "TorrentDuplicatesPipeline", "FormulaParserPipeline", @@ -38,4 +38,5 @@ __all__ = [ "WWEParserPipeline", "UFCParserPipeline", "CatalogParsePipeline", + "MovieTVParserPipeline", ] diff --git a/mediafusion_scrapy/pipelines/movie_tv_parser_pipeline.py b/mediafusion_scrapy/pipelines/movie_tv_parser_pipeline.py new file mode 100644 index 0000000..ebdc37d --- /dev/null +++ b/mediafusion_scrapy/pipelines/movie_tv_parser_pipeline.py @@ -0,0 +1,62 @@ +import PTT +from scrapy.exceptions import DropItem + +from scrapers.imdb_data import search_imdb, get_imdb_title_data +from scrapers.tmdb_data import search_tmdb +from utils.const import QUALITY_GROUPS + + +class MovieTVParserPipeline: + + async def process_item(self, item, spider): + data = item.copy() + title = data["torrent_title"] + if "title" not in data: + data.update(PTT.parse_title(title, True)) + + if not data.get("title"): + raise DropItem(f"Title not parsed: {title}") + + if "file_data" not in data: + raise DropItem(f"File data not found in item: {data}") + + if data["type"] == "movie": + data["type"] = "series" if data["seasons"] else "movie" + + if data["type"] == "series" and len(data.get("season", [])) > 1: + raise DropItem(f"Multiple seasons found in title: {title}") + + title = data["title"] + if data.get("imdb_id"): + imdb_data = await get_imdb_title_data(data["imdb_id"], data["type"]) + else: + imdb_data = await search_imdb(title, data.get("year"), data["type"]) + if not imdb_data: + imdb_data = await search_tmdb(title, data.get("year"), data["type"]) + + if not imdb_data: + raise DropItem(f"IMDb data not found for title: {title}") + + data.update(imdb_data) + data["is_search_imdb_title"] = False + data["id"] = data["imdb_id"] + data["catalog"] = [ + data["catalog_source"], + f"{data['catalog_source']}_{data['type']}", + ] + + if data["type"] == "series": + data["video_type"] = "series" + elif data.get("quality") in QUALITY_GROUPS["CAM/Screener"]: + data["video_type"] = "tcrip" + else: + data["video_type"] = "hdrip" + + # Handle file data + if data["type"] == "movie": + largest_file = max(data["file_data"], key=lambda x: x["size"]) + data["largest_file"] = { + "index": data["file_data"].index(largest_file), + "filename": largest_file["filename"], + } + return data diff --git a/mediafusion_scrapy/pipelines/torrent_parser_pipeline.py b/mediafusion_scrapy/pipelines/torrent_parser_pipeline.py index 176f545..41e9ada 100644 --- a/mediafusion_scrapy/pipelines/torrent_parser_pipeline.py +++ b/mediafusion_scrapy/pipelines/torrent_parser_pipeline.py @@ -58,9 +58,6 @@ class TorrentDownloadAndParsePipeline: class MagnetDownloadAndParsePipeline: async def process_item(self, item, spider): - if item.get("file_data"): - return item - magnet_link = item.get("magnet_link") if not magnet_link: @@ -77,6 +74,8 @@ class MagnetDownloadAndParsePipeline: ) if not torrent_metadata: + if item.get("file_data"): + return item raise DropItem(f"Failed to extract torrent metadata: {item}") item.update(torrent_metadata[0]) diff --git a/mediafusion_scrapy/spiders/tgx.py b/mediafusion_scrapy/spiders/tgx.py index 6ff8794..7780ce7 100644 --- a/mediafusion_scrapy/spiders/tgx.py +++ b/mediafusion_scrapy/spiders/tgx.py @@ -1,9 +1,10 @@ import random import re +from copy import deepcopy from datetime import datetime +import PTT import scrapy -from scrapy_playwright.page import PageMethod from db.config import settings from db.models import TorrentStreams @@ -25,9 +26,12 @@ class TgxSpider(scrapy.Spider): keyword_patterns: re.Pattern scraped_info_hash_key: str - def __init__(self, scrape_all: str = "False", *args, **kwargs): + def __init__( + self, scrape_all: str = "False", total_pages: int = None, *args, **kwargs + ): super(TgxSpider, self).__init__(*args, **kwargs) self.scrape_all = scrape_all.lower() == "true" + self.total_pages = total_pages self.redis = REDIS_ASYNC_CLIENT async def __aexit__(self, exc_type, exc_val, exc_tb): @@ -46,6 +50,7 @@ class TgxSpider(scrapy.Spider): "wait_until": "domcontentloaded", "timeout": 60000, }, + "playwright_include_page": True, "is_search_query": False, "parse_url": parse_url, }, @@ -61,19 +66,40 @@ class TgxSpider(scrapy.Spider): "wait_until": "domcontentloaded", "timeout": 60000, }, + "playwright_include_page": True, "is_search_query": True, "parse_url": parse_url, }, ) async def parse(self, response, **kwargs): - if "galaxyfence.php" in response.url: - self.logger.warning("Encountered galaxyfence.php. Retrying") - parse_url = response.meta.get("parse_url") - yield scrapy.Request( - parse_url, self.parse, meta=response.meta, dont_filter=True, priority=10 - ) - return + # First try waiting for the content to load + page = response.meta["playwright_page"] + if not response.css("div.tgxtablerow.txlight"): + try: + await page.wait_for_selector("div.tgxtablerow.txlight", timeout=10000) + # Get the updated content after waiting + content = await page.content() + await page.close() + response = response.replace(body=content) + except Exception: + # If timeout occurs, check for galaxyfence + await page.close() + if "galaxyfence.php" in response.url: + self.logger.warning("Encountered galaxyfence.php. Retrying") + parse_url = response.meta.get("parse_url") + yield scrapy.Request( + parse_url, + self.parse, + meta=response.meta, + dont_filter=True, + priority=10, + ) + return + else: + self.logger.warning(f"Content not loaded for URL: {response.url}") + return + await page.close() uploader_profile_name = response.meta.get("uploader_profile") is_search_query = response.meta.get("is_search_query") @@ -88,6 +114,9 @@ class TgxSpider(scrapy.Spider): int(last_page_number) if last_page_number.isdigit() else 0 ) + if self.total_pages: + last_page_number = min(last_page_number, self.total_pages) + # Generate requests for all pages for page_number in range(1, last_page_number + 1): next_page_url = ( @@ -104,6 +133,9 @@ class TgxSpider(scrapy.Spider): int(last_page_number) if last_page_number.isdigit() else 0 ) + if self.total_pages: + last_page_number = min(last_page_number, self.total_pages) + # Generate requests for all pages for page_number in range(1, last_page_number + 1): next_page_url = ( @@ -164,6 +196,7 @@ class TgxSpider(scrapy.Spider): "website": torrent_page_link, "unique_id": tgx_unique_id, "source": "TorrentGalaxy", + "catalog_source": "tgx", "uploader": uploader_profile_name, "announce_list": announce_list, "catalog": self.catalog, @@ -187,19 +220,39 @@ class TgxSpider(scrapy.Spider): "referer": response.url, "timeout": 60000, }, + "playwright_include_page": True, "torrent_data": torrent_data, + "torrent_page_link": torrent_page_link, }, ) - def parse_torrent_details(self, response): - torrent_data = response.meta["torrent_data"].copy() + async def parse_torrent_details(self, response): + # First try waiting for the content to load + page = response.meta["playwright_page"] + if not response.xpath('//button[contains(@class, "flist")]//em/text()'): + try: + await page.wait_for_selector("button.flist em", timeout=10000) + # Get the updated content after waiting + content = await page.content() + response = response.replace(body=content) + except Exception: + # If timeout occurs, check for galaxyfence + await page.close() + if ( + response.xpath("//blockquote[contains(., 'GALAXY CHECKPOINT')]") + or "galaxyfence.php" in response.url + ): + self.logger.warning( + f"Encountered GALAXY CHECKPOINT. Retrying: {response.url}" + ) + yield self.retry_request(response) + return + else: + self.logger.warning(f"Content not loaded for URL: {response.url}") + return + await page.close() - if response.xpath("//blockquote[contains(., 'GALAXY CHECKPOINT')]"): - self.logger.warning( - f"Encountered GALAXY CHECKPOINT. Retrying: {response.url}" - ) - yield self.retry_request(response) - return + torrent_data = deepcopy(response.meta["torrent_data"]) # Extracting file details and sizes if available file_data = [] @@ -211,7 +264,17 @@ class TgxSpider(scrapy.Spider): file_name = row.xpath('td[@class="table_col1"]/text()').get() file_size = row.xpath('td[@class="table_col2"]/text()').get() if file_name and file_size: - file_data.append({"filename": file_name, "size": file_size}) + parsed_data = PTT.parse_title(file_name) + file_data.append( + { + "filename": file_name, + "size": convert_size_to_bytes(file_size), + "index": len(file_data) + 1, + "seasons": parsed_data.get("seasons"), + "episodes": parsed_data.get("episodes"), + "title": parsed_data.get("title"), + } + ) if file_count == len(file_data): torrent_data["file_data"] = file_data @@ -262,6 +325,14 @@ class TgxSpider(scrapy.Spider): if language: torrent_data["languages"] = [language.strip()] + # Extracting type + title_type = response.xpath( + "//div[b='Category:']/following-sibling::div/a/text()" + ).get() + if title_type: + title_type = title_type.strip().lower() + torrent_data["type"] = "series" if title_type == "tv" else "movie" + yield torrent_data def handle_failure(self, failure): @@ -270,8 +341,9 @@ class TgxSpider(scrapy.Spider): def retry_request(self, request): retry_count = request.meta.get("retry_count", 0) + torrent_page_link = request.meta.get("torrent_page_link") if retry_count < 3: # Set your retry limit - new_url = request.url.replace( + new_url = torrent_page_link.replace( request.url.split("/")[2], random.choice(self.allowed_domains) ) self.logger.info(f"Retrying with new URL: {new_url}") @@ -286,8 +358,10 @@ class TgxSpider(scrapy.Spider): "referer": request.url, "timeout": 60000, }, + "playwright_include_page": True, "torrent_data": request.meta["torrent_data"], "retry_count": retry_count + 1, + "torrent_page_link": torrent_page_link, }, ) else: @@ -299,7 +373,6 @@ class FormulaTgxSpider(TgxSpider): name = "formula_tgx" uploader_profiles = [ "egortech", - "F1Carreras", "smcgill1969", ] catalog = ["formula_racing"] @@ -426,3 +499,42 @@ class UFCTGXSpider(BaseEventSpider): *args, **kwargs, ) + + +class MoviesTVTgxSpider(TgxSpider): + name = "movies_tv_tgx" + search_queries = [ + # Movies + # "c3=1&c46=1&c45=1&c42=1&c4=1&c1=1&search=&lang=0&nox=2&nox=1&page=0", + # # TV Series + # "c41=1&c5=1&c11=1&c6=1&search=&lang=0&nox=2&nox=1&page=0", + "c3=1&c46=1&c45=1&c42=1&c4=1&c1=1&search=&lang=8&nox=2&page=0" + ] + catalog = [] + background_image = None # Will be fetched from TMDB/IMDB + logo_image = None + + # Pattern to exclude unwanted content + keyword_patterns = re.compile( + r"^(?!.*(?:WWE|UFC|Formula|MotoGP)).*$", # Excludes sports content + re.IGNORECASE, + ) + scraped_info_hash_key = "movies_tv_tgx_scraped_info_hash" + + custom_settings = { + "ITEM_PIPELINES": { + "mediafusion_scrapy.pipelines.MagnetDownloadAndParsePipeline": 100, + "mediafusion_scrapy.pipelines.MovieTVParserPipeline": 200, + "mediafusion_scrapy.pipelines.CatalogParsePipeline": 300, + "mediafusion_scrapy.pipelines.MovieStorePipeline": 400, + "mediafusion_scrapy.pipelines.SeriesStorePipeline": 500, + }, + "PLAYWRIGHT_BROWSER_TYPE": "chromium", + "PLAYWRIGHT_CDP_URL": settings.playwright_cdp_url, + "DOWNLOAD_HANDLERS": { + "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", + "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", + }, + "PLAYWRIGHT_MAX_CONTEXTS": 2, + "PLAYWRIGHT_MAX_PAGES_PER_CONTEXT": 3, + } diff --git a/resources/templates/manifest.json.j2 b/resources/templates/manifest.json.j2 index 299a0c4..0db1437 100644 --- a/resources/templates/manifest.json.j2 +++ b/resources/templates/manifest.json.j2 @@ -84,6 +84,8 @@ "hockey": {"type": "movie", "name": "Hockey", "genres": []}, "rugby": {"type": "movie", "name": "Rugby/AFL", "genres": []}, "fighting": {"type": "movie", "name": "Fighting (WWE, UFC)", "genres": ["WWE", "UFC"]}, + "tgx_movie": {"type": "movie", "name": "TGx Movies"}, + "tgx_series": {"type": "series", "name": "TGx Series"}, "other_sports": {"type": "movie", "name": "Other Sports", "genres": []}, "live_sport_events": {"type": "events", "name": "Live Sport Events", "genres": ["Afl", "American Football", "Baseball", "Basketball", "Billiards", "Cricket", "Dart", "Fighting", "Football", "Golf", "Hockey", "Motor Sport", "Other Sports", "Rugby", "Tennis"]}, "prowlarr_movies": {"type": "movie", "name": "Prowlarr Scraped Movies"}, diff --git a/utils/const.py b/utils/const.py index f31c7ab..927308a 100644 --- a/utils/const.py +++ b/utils/const.py @@ -46,6 +46,8 @@ CATALOG_DATA = { "telugu_series": "Telugu Series", "telugu_tcrip": "Telugu TCRip Movies", "fighting": "Fighting (WWE, UFC)", + "tgx_movie": "TGx Movies", + "tgx_series": "TGx Series", } USER_UPLOAD_SUPPORTED_MOVIE_CATALOG_IDS = [