diff --git a/api/main.py b/api/main.py index 03a678c..f15ced4 100644 --- a/api/main.py +++ b/api/main.py @@ -26,13 +26,13 @@ from metrics.routes import metrics_router from scrapers.routes import router as scrapers_router from streaming_providers import mapper from streaming_providers.routes import router as streaming_provider_router -from utils import crypto, torrent, poster, const, wrappers +from utils import crypto, torrent, poster, const, wrappers, get_json_data from utils.lock import ( acquire_scheduler_lock, maintain_heartbeat, release_scheduler_lock, ) -from utils.parser import generate_manifest, get_json_data +from utils.parser import generate_manifest logging.basicConfig( format="%(levelname)s::%(asctime)s - %(message)s", diff --git a/mediafusion_scrapy/pipelines.py b/mediafusion_scrapy/pipelines.py index 54a2fbc..7f94444 100644 --- a/mediafusion_scrapy/pipelines.py +++ b/mediafusion_scrapy/pipelines.py @@ -1,5 +1,6 @@ import asyncio import logging +import random import re from datetime import datetime from uuid import uuid4 @@ -24,6 +25,7 @@ from db.models import ( from db.schemas import TVMetaData from utils import torrent, const from utils.parser import convert_size_to_bytes +from utils.runtime_const import SPORTS_ARTIFACTS class TorrentDuplicatesPipeline: @@ -470,6 +472,130 @@ class FormulaParserPipeline: torrent_data["episodes"] = episodes +class MotoGPParserPipeline: + def __init__(self): + self.name_parser_patterns = { + "smcgill1969": [ + re.compile( + r"MotoGP" # Series name with flexible space or dot + r"\.(?P\d{4})" # Year + r"x(?P\d{2})" # Round + r"\.(?P.+?)" # Event and Episode name + r"\.(?PBTSportHD|TNTSportsHD)" # Broadcaster + r"\.(?P4K|SD|1080p)" # Resolution and Quality + ), + re.compile( + r"MotoGP" # Series name with flexible space or dot + r"\.(?P\d{4})" # Year + r"(?:-\d{2}-\d{2})?" # ignore Date part starting with a hyphen + r"\.(?P.+?)" # Event and Episode name + r"(?:\.(?PWEB-DL|HDTV))?" # Optional Quality + r"(?:\.(?PBTSportHD|TNTSportsHD))?" # Optional Broadcaster + r"\.(?P4K|SD|1080p)" # Resolution + ), + ], + } + self.default_poster = random.choice(SPORTS_ARTIFACTS["MotoGP"]["poster"]) + + self.smcgill1969_resolutions = { + "4K": "4K", + "SD": "576p", + "1080p": "1080p", + } + self.known_countries_first_words = ["San", "Great"] + + self.title_parser_functions = { + "smcgill1969": self.parse_smcgill1969_title, + } + self.description_parser_functions = { + "smcgill1969": self.parse_smcgill1969_description, + } + + def process_item(self, item, spider): + uploader = item["uploader"] + title = re.sub(r"\.\.+", ".", item["torrent_name"]) + self.title_parser_functions[uploader](title, item) + if not item.get("title"): + raise DropItem(f"Title not parsed: {title}") + self.description_parser_functions[uploader](item) + if not item.get("episodes"): + raise DropItem(f"Episodes not parsed: {item!r}") + return item + + def event_and_episode_name_parser(self, event_and_episode_name): + parts = event_and_episode_name.split(".") + if parts[0] in self.known_countries_first_words and len(parts) > 1: + event = f"{parts[0]} {parts[1]}" + episode_name = " ".join(parts[2:]) + else: + event = parts[0] + episode_name = " ".join(parts[1:]) + return event, episode_name + + def parse_smcgill1969_title(self, title, torrent_data: dict): + for pattern in self.name_parser_patterns.get("smcgill1969"): + match = pattern.match(title) + if match: + data = match.groupdict() + series = "MotoGP" + event, episode_name = self.event_and_episode_name_parser( + data["EventAndEpisodeName"] + ) + + torrent_data.update( + { + "title": " ".join( + filter( + None, + [ + series, + data.get("Year"), + event, + "(smcgill1969)", # add the uploader to the title for uniqueness + ], + ) + ), + "year": int(data["Year"]), + "resolution": self.smcgill1969_resolutions.get( + data["Resolution"] + ), + "event": event, + "episode_name": episode_name, + } + ) + return + logging.warning(f"Failed to parse title: {title}") + + def parse_smcgill1969_description(self, torrent_data: dict): + torrent_description = torrent_data.get("description") + + codec_matches = re.findall(r"Codec ID\s*:\s*(\S+)", torrent_description) + if codec_matches: + torrent_data["codec"] = codec_matches[0] + torrent_data["audio"] = codec_matches[1] if len(codec_matches) > 1 else None + + torrent_data["poster"] = self.default_poster + torrent_data["is_add_title_to_poster"] = True + + episodes = [] + for index, file_detail in enumerate(torrent_data.get("file_details", [])): + file_name = file_detail.get("file_name") + file_size = file_detail.get("file_size") + + episodes.append( + Episode( + episode_number=index + 1, + filename=file_name, + size=convert_size_to_bytes(file_size), + file_index=index, + title=" ".join(file_name.split(".")[1:-1]), + released=torrent_data.get("created_at"), + ) + ) + + torrent_data["episodes"] = episodes + + class QueueBasedPipeline: def __init__(self): self.queue = asyncio.Queue() @@ -509,7 +635,7 @@ class QueueBasedPipeline: raise NotImplementedError -class FormulaStorePipeline(QueueBasedPipeline): +class EventSeriesStorePipeline(QueueBasedPipeline): def __init__(self): super().__init__() self.redis = redis_async.Redis.from_url(settings.redis_url) diff --git a/mediafusion_scrapy/settings.py b/mediafusion_scrapy/settings.py index adba81c..6c8b8d1 100644 --- a/mediafusion_scrapy/settings.py +++ b/mediafusion_scrapy/settings.py @@ -94,4 +94,4 @@ HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" FEED_EXPORT_ENCODING = "utf-8" -LOG_LEVEL = "DEBUG" +LOG_LEVEL = "INFO" diff --git a/mediafusion_scrapy/spiders/crictime.py b/mediafusion_scrapy/spiders/crictime.py index 1a82897..786bf9d 100644 --- a/mediafusion_scrapy/spiders/crictime.py +++ b/mediafusion_scrapy/spiders/crictime.py @@ -5,7 +5,7 @@ import redis import scrapy from db.config import settings -from utils.parser import get_json_data +from utils.runtime_const import SPORTS_ARTIFACTS class CricTimeSpider(scrapy.Spider): @@ -25,7 +25,6 @@ class CricTimeSpider(scrapy.Spider): self.redis = redis.Redis( connection_pool=redis.ConnectionPool.from_url(settings.redis_url) ) - self.sports_artifacts = get_json_data("resources/json/sports_artifacts.json") def __del__(self): self.redis.close() @@ -57,11 +56,11 @@ class CricTimeSpider(scrapy.Spider): item = { "genres": [category], - "poster": random.choice(self.sports_artifacts[category]["poster"]), + "poster": random.choice(SPORTS_ARTIFACTS[category]["poster"]), "background": random.choice( - self.sports_artifacts[category]["background"] + SPORTS_ARTIFACTS[category]["background"] ), - "logo": random.choice(self.sports_artifacts[category]["logo"]), + "logo": random.choice(SPORTS_ARTIFACTS[category]["logo"]), "is_add_title_to_poster": True, "title": event_name, "url": response.urljoin(event_url), diff --git a/mediafusion_scrapy/spiders/dlhd.py b/mediafusion_scrapy/spiders/dlhd.py index d9a74b8..aee0a4a 100644 --- a/mediafusion_scrapy/spiders/dlhd.py +++ b/mediafusion_scrapy/spiders/dlhd.py @@ -5,7 +5,7 @@ from datetime import datetime import pytz import scrapy -from utils.parser import get_json_data +from utils.runtime_const import SPORTS_ARTIFACTS class DaddyLiveHDSpider(scrapy.Spider): @@ -60,7 +60,6 @@ class DaddyLiveHDSpider(scrapy.Spider): def __init__(self, *args, **kwargs): super(DaddyLiveHDSpider, self).__init__(*args, **kwargs) - self.sports_artifacts = get_json_data("resources/json/sports_artifacts.json") self.gmt = pytz.timezone("Etc/GMT") def parse(self, response, **kwargs): @@ -85,13 +84,11 @@ class DaddyLiveHDSpider(scrapy.Spider): item = { "stream_source": "DaddyLiveHD (1.dlhd.sx)", "genres": [category], - "poster": random.choice( - self.sports_artifacts[category]["poster"] - ), + "poster": random.choice(SPORTS_ARTIFACTS[category]["poster"]), "background": random.choice( - self.sports_artifacts[category]["background"] + SPORTS_ARTIFACTS[category]["background"] ), - "logo": random.choice(self.sports_artifacts[category]["logo"]), + "logo": random.choice(SPORTS_ARTIFACTS[category]["logo"]), "is_add_title_to_poster": True, "title": event["event"], "channels": event["channels"], diff --git a/mediafusion_scrapy/spiders/formula_tgx.py b/mediafusion_scrapy/spiders/formula_tgx.py index e4e3c82..1b74d29 100644 --- a/mediafusion_scrapy/spiders/formula_tgx.py +++ b/mediafusion_scrapy/spiders/formula_tgx.py @@ -1,205 +1,26 @@ -import random import re -from datetime import datetime -import redis.asyncio as redis -import scrapy - -from db.config import settings -from db.models import TorrentStreams -from utils.parser import convert_size_to_bytes -from utils.torrent import parse_magnet +from mediafusion_scrapy.spiders.tgx import TgxSpider -class FormulaTgxSpider(scrapy.Spider): +class FormulaTgxSpider(TgxSpider): name = "formula_tgx" - allowed_domains = ["torrentgalaxy.to", "tgx.rs", "torrentgalaxy.mx", "tgx.sb"] - formula_uploader_profiles = [ + uploader_profiles = [ "egortech", "F1Carreras", "smcgill1969", ] + catalog = ["formula_racing"] + background_image = "https://i.postimg.cc/S4wcrGRZ/f1background.png?dl=1" + logo_image = "https://i.postimg.cc/Sqf4V8tj/f1logo.png?dl=1" - formula1_keyword_patterns = re.compile(r"formula[ .+]*[1234e]+", re.IGNORECASE) + keyword_patterns = re.compile(r"formula[ .+]*[1234e]+", re.IGNORECASE) + scraped_info_hash_key = "formula_tgx_scraped_info_hash" custom_settings = { "ITEM_PIPELINES": { "mediafusion_scrapy.pipelines.TorrentDuplicatesPipeline": 100, "mediafusion_scrapy.pipelines.FormulaParserPipeline": 200, - "mediafusion_scrapy.pipelines.FormulaStorePipeline": 300, + "mediafusion_scrapy.pipelines.EventSeriesStorePipeline": 300, } } - - def __init__(self, scrape_all: str = "True", *args, **kwargs): - super(FormulaTgxSpider, self).__init__(*args, **kwargs) - self.scrape_all = scrape_all.lower() == "true" - self.redis = redis.Redis( - connection_pool=redis.ConnectionPool.from_url(settings.redis_url) - ) - self.scraped_info_hash_key = "formula_tgx_scraped_info_hash" - - async def __aexit__(self, exc_type, exc_val, exc_tb): - await self.redis.aclose() - - def start_requests(self): - for uploader_profile in self.formula_uploader_profiles: - yield scrapy.Request( - f"https://{random.choice(self.allowed_domains)}/profile/{uploader_profile}/torrents/0", - self.parse, - meta={"uploader_profile": uploader_profile}, - ) - - async def parse(self, response, **kwargs): - uploader_profile_name = response.meta["uploader_profile"] - self.logger.info(f"Scraping torrents from {uploader_profile_name}") - - # Extract the last page number only once at the beginning - if self.scrape_all and response.url.endswith("0"): - last_page_number = response.css( - "ul.pagination li.page-item:not(.disabled) a::attr(href)" - ).re(r"/profile/.*/torrents/(\d+)")[-2] - last_page_number = ( - int(last_page_number) if last_page_number.isdigit() else 0 - ) - - # Generate requests for all pages - for page_number in range(1, last_page_number + 1): - next_page_url = ( - f"{response.url.split('/torrents/')[0]}/torrents/{page_number}" - ) - yield response.follow(next_page_url, self.parse, meta=response.meta) - - # Extract torrents from the page - for torrent in response.css("div.tgxtablerow.txlight"): - urls = torrent.css("div.tgxtablecell a::attr(href)").getall() - - torrent_name = torrent.css( - "div.tgxtablecell.clickable-row.click.textshadow.rounded.txlight a b::text" - ).get() - - if not self.formula1_keyword_patterns.search(torrent_name): - continue - - tgx_unique_id = urls[0].split("/")[-2] - torrent_page_link = response.urljoin(urls[0]) - torrent_link = torrent.css( - 'a[href*="watercache.nanobytes.org"]::attr(href)' - ).get() - magnet_link = torrent.css('a[href^="magnet:?"]::attr(href)').get() - info_hash, announce_list = parse_magnet(magnet_link) - if not info_hash: - self.logger.warning( - f"Failed to parse magnet link: {response.url}, {torrent_name}" - ) - continue - - seeders = torrent.css( - "div.tgxtablecell span[title='Seeders/Leechers'] font[color='green'] b::text" - ).get() - - seeders = int(seeders) if seeders and seeders.isdigit() else None - - torrent_data = { - "info_hash": info_hash, - "torrent_name": torrent_name, - "torrent_link": torrent_link, - "magnet_link": magnet_link, - "background": "https://i.postimg.cc/S4wcrGRZ/f1background.png?dl=1", - "logo": "https://i.postimg.cc/Sqf4V8tj/f1logo.png?dl=1", - "seeders": seeders, - "torrent_page_link": torrent_page_link, - "unique_id": tgx_unique_id, - "source": f"TorrentGalaxy ({uploader_profile_name})", - "uploader": uploader_profile_name, - "announce_list": announce_list, - "catalog": ["formula_racing"], - "scraped_info_hash_key": self.scraped_info_hash_key, - } - - if await self.redis.sismember(self.scraped_info_hash_key, info_hash): - self.logger.info(f"Torrent already scraped: {torrent_name}") - await TorrentStreams.find_one({"_id": info_hash}).update( - {"$set": {"seeders": seeders}}, - ) - else: - yield response.follow( - torrent_page_link, - self.parse_torrent_details, - meta={"torrent_data": torrent_data}, - ) - - def parse_torrent_details(self, response): - torrent_data = response.meta["torrent_data"] - - # Extracting file details and sizes - file_details = [] - for row in response.xpath('//table[contains(@class, "table-striped")]/tr'): - file_name = row.xpath('td[@class="table_col1"]/text()').get() - file_size = row.xpath('td[@class="table_col2"]/text()').get() - if file_name and file_size: - file_details.append({"file_name": file_name, "file_size": file_size}) - if not file_details: - self.logger.warning( - f"File details not found for {torrent_data['torrent_name']}. Retrying" - ) - yield response.follow( - response.url, - self.parse_torrent_details, - meta={"torrent_data": torrent_data}, - ) - return - torrent_data["file_details"] = file_details - - cover_image_url = response.xpath( - "//img[contains(@class, 'img-responsive') and contains(@data-src, '.png')]/@data-src" - ).get() - torrent_data["poster"] = cover_image_url - - # Getting the description for parsing video, audio, and other details - torrent_description = "".join( - response.xpath( - "//font/following-sibling::*[1]/following-sibling::text() | " - "//font/following-sibling::*[1]/following-sibling::*//text() | " - "//center/font/following::br/following-sibling::text() | " - "//strong/following-sibling::text()[normalize-space()] | " - "//strong/following-sibling::br/following-sibling::text()[normalize-space()] | " - "//div[contains(@class, 'container-fluid')]//text()[normalize-space()]" - ).extract() - ) - torrent_data["description"] = torrent_description.replace("\xa0", " ") - - total_size = response.xpath( - "//div[b='Total Size:']/following-sibling::div/text()" - ).get() - if total_size: - torrent_data["total_size"] = convert_size_to_bytes(total_size) - else: - # if the total size is not found, then tgx has shown captcha validation. - # so we need to slow down and retry the request - self.logger.warning( - f"Total size not found for {torrent_data['torrent_name']}. Retrying" - ) - yield response.follow( - response.url, - self.parse_torrent_details, - meta={"torrent_data": torrent_data}, - ) - - # Extracting date created - date_created = response.xpath( - "//div[b[contains(., 'Added:')]]/following-sibling::div/text()" - ).get() - if date_created: - # Processing to extract the date and time - torrent_data["created_at"] = datetime.strptime( - date_created.strip(), "%d-%m-%Y %H:%M" - ) - - # Extracting language - language = response.xpath( - "//div[b='Language:']/following-sibling::div/text()" - ).get() - if language: - torrent_data["languages"] = [language.strip()] - - yield torrent_data diff --git a/mediafusion_scrapy/spiders/motogp_tgx.py b/mediafusion_scrapy/spiders/motogp_tgx.py new file mode 100644 index 0000000..f13a9f3 --- /dev/null +++ b/mediafusion_scrapy/spiders/motogp_tgx.py @@ -0,0 +1,26 @@ +import re +import random + +from mediafusion_scrapy.spiders.tgx import TgxSpider +from utils.runtime_const import SPORTS_ARTIFACTS + + +class MotoGPTgxSpider(TgxSpider): + name = "motogp_tgx" + uploader_profiles = [ + "smcgill1969", + ] + catalog = ["motogp_racing"] + + keyword_patterns = re.compile(r"MotoGP[ .+]*", re.IGNORECASE) + scraped_info_hash_key = "motogp_tgx_scraped_info_hash" + background_image = random.choice(SPORTS_ARTIFACTS["MotoGP"]["background"]) + logo_image = random.choice(SPORTS_ARTIFACTS["MotoGP"]["logo"]) + + custom_settings = { + "ITEM_PIPELINES": { + "mediafusion_scrapy.pipelines.TorrentDuplicatesPipeline": 100, + "mediafusion_scrapy.pipelines.MotoGPParserPipeline": 200, + "mediafusion_scrapy.pipelines.EventSeriesStorePipeline": 300, + } + } diff --git a/mediafusion_scrapy/spiders/mrgamingstreams.py b/mediafusion_scrapy/spiders/mrgamingstreams.py index 5abf272..be89e20 100644 --- a/mediafusion_scrapy/spiders/mrgamingstreams.py +++ b/mediafusion_scrapy/spiders/mrgamingstreams.py @@ -7,7 +7,7 @@ import redis import scrapy from db.config import settings -from utils.parser import get_json_data +from utils.runtime_const import SPORTS_ARTIFACTS class MrGamingStreamsSpider(scrapy.Spider): @@ -38,7 +38,6 @@ class MrGamingStreamsSpider(scrapy.Spider): self.redis = redis.Redis( connection_pool=redis.ConnectionPool.from_url(settings.redis_url) ) - self.sports_artifacts = get_json_data("resources/json/sports_artifacts.json") def __del__(self): self.redis.close() @@ -82,11 +81,9 @@ class MrGamingStreamsSpider(scrapy.Spider): item = { "genres": [category], - "poster": random.choice(self.sports_artifacts[category]["poster"]), - "background": random.choice( - self.sports_artifacts[category]["background"] - ), - "logo": random.choice(self.sports_artifacts[category]["logo"]), + "poster": random.choice(SPORTS_ARTIFACTS[category]["poster"]), + "background": random.choice(SPORTS_ARTIFACTS[category]["background"]), + "logo": random.choice(SPORTS_ARTIFACTS[category]["logo"]), "is_add_title_to_poster": True, "event_start_timestamp": event_start_timestamp, "title": event_name, diff --git a/mediafusion_scrapy/spiders/streambtw.py b/mediafusion_scrapy/spiders/streambtw.py index e4e5660..ec8e401 100644 --- a/mediafusion_scrapy/spiders/streambtw.py +++ b/mediafusion_scrapy/spiders/streambtw.py @@ -5,7 +5,7 @@ import redis import scrapy from db.config import settings -from utils.parser import get_json_data +from utils.runtime_const import SPORTS_ARTIFACTS class StreamBTWSpider(scrapy.Spider): @@ -34,7 +34,6 @@ class StreamBTWSpider(scrapy.Spider): self.redis = redis.Redis( connection_pool=redis.ConnectionPool.from_url(settings.redis_url) ) - self.sports_artifacts = get_json_data("resources/json/sports_artifacts.json") def __del__(self): self.redis.close() @@ -58,9 +57,9 @@ class StreamBTWSpider(scrapy.Spider): item = { "genres": [category], "description": description, - "poster": random.choice(self.sports_artifacts[category]["poster"]), + "poster": random.choice(SPORTS_ARTIFACTS[category]["poster"]), "background": random.choice( - self.sports_artifacts[category]["background"] + SPORTS_ARTIFACTS[category]["background"] ), "logo": logo, "is_add_title_to_poster": True, diff --git a/mediafusion_scrapy/spiders/streamed.py b/mediafusion_scrapy/spiders/streamed.py index 49b29cb..ced0c6a 100644 --- a/mediafusion_scrapy/spiders/streamed.py +++ b/mediafusion_scrapy/spiders/streamed.py @@ -3,7 +3,7 @@ import re import scrapy -from utils.parser import get_json_data +from utils.runtime_const import SPORTS_ARTIFACTS class StreamedSpider(scrapy.Spider): @@ -46,7 +46,6 @@ class StreamedSpider(scrapy.Spider): def __init__(self, *args, **kwargs): super(StreamedSpider, self).__init__(*args, **kwargs) - self.sports_artifacts = get_json_data("resources/json/sports_artifacts.json") def start_requests(self): for category, url in self.categories.items(): @@ -63,11 +62,9 @@ class StreamedSpider(scrapy.Spider): item = { "stream_source": "Streamed (streamed.su)", "genres": [category], - "poster": random.choice(self.sports_artifacts[category]["poster"]), - "background": random.choice( - self.sports_artifacts[category]["background"] - ), - "logo": random.choice(self.sports_artifacts[category]["logo"]), + "poster": random.choice(SPORTS_ARTIFACTS[category]["poster"]), + "background": random.choice(SPORTS_ARTIFACTS[category]["background"]), + "logo": random.choice(SPORTS_ARTIFACTS[category]["logo"]), "is_add_title_to_poster": True, "title": event_name, "url": response.urljoin(event_url), diff --git a/mediafusion_scrapy/spiders/tgx.py b/mediafusion_scrapy/spiders/tgx.py new file mode 100644 index 0000000..96e0e3f --- /dev/null +++ b/mediafusion_scrapy/spiders/tgx.py @@ -0,0 +1,210 @@ +import random +import re +from datetime import datetime + +import redis.asyncio as redis +import scrapy + +from db.config import settings +from db.models import TorrentStreams +from utils.parser import convert_size_to_bytes +from utils.torrent import parse_magnet + + +class TgxSpider(scrapy.Spider): + allowed_domains = ["torrentgalaxy.to", "tgx.rs", "torrentgalaxy.mx", "tgx.sb"] + uploader_profiles: list[str] + catalog: list[str] + background_image: str + logo_image: str + + keyword_patterns = re.compile(r"formula[ .+]*[1234e]+", re.IGNORECASE) + scraped_info_hash_key = "formula_tgx_scraped_info_hash" + + custom_settings = { + "ITEM_PIPELINES": { + "mediafusion_scrapy.pipelines.TorrentDuplicatesPipeline": 100, + "mediafusion_scrapy.pipelines.FormulaParserPipeline": 200, + "mediafusion_scrapy.pipelines.EventSeriesStorePipeline": 300, + } + } + + def __init__(self, scrape_all: str = "True", *args, **kwargs): + super(TgxSpider, self).__init__(*args, **kwargs) + self.scrape_all = scrape_all.lower() == "true" + self.redis = redis.Redis( + connection_pool=redis.ConnectionPool.from_url(settings.redis_url) + ) + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.redis.aclose() + + def start_requests(self): + for uploader_profile in self.uploader_profiles: + yield scrapy.Request( + f"https://{random.choice(self.allowed_domains)}/profile/{uploader_profile}/torrents/0", + self.parse, + meta={"uploader_profile": uploader_profile}, + ) + + async def parse(self, response, **kwargs): + uploader_profile_name = response.meta["uploader_profile"] + self.logger.info(f"Scraping torrents from {uploader_profile_name}") + + # Extract the last page number only once at the beginning + if self.scrape_all and response.url.endswith("/0"): + last_page_number = response.css( + "ul.pagination li.page-item:not(.disabled) a::attr(href)" + ).re(r"/profile/.*/torrents/(\d+)")[-2] + last_page_number = ( + int(last_page_number) if last_page_number.isdigit() else 0 + ) + + # Generate requests for all pages + for page_number in range(1, last_page_number + 1): + next_page_url = ( + f"{response.url.split('/torrents/')[0]}/torrents/{page_number}" + ) + yield response.follow(next_page_url, self.parse, meta=response.meta) + + # Extract torrents from the page + for torrent in response.css("div.tgxtablerow.txlight"): + urls = torrent.css("div.tgxtablecell a::attr(href)").getall() + + torrent_name = torrent.css( + "div.tgxtablecell.clickable-row.click.textshadow.rounded.txlight a b::text" + ).get() + + if not self.keyword_patterns.search(torrent_name): + continue + + tgx_unique_id = urls[0].split("/")[-2] + torrent_page_link = response.urljoin(urls[0]) + torrent_link = torrent.css( + 'a[href*="watercache.nanobytes.org"]::attr(href)' + ).get() + magnet_link = torrent.css('a[href^="magnet:?"]::attr(href)').get() + info_hash, announce_list = parse_magnet(magnet_link) + if not info_hash: + self.logger.warning( + f"Failed to parse magnet link: {response.url}, {torrent_name}" + ) + continue + + seeders = torrent.css( + "div.tgxtablecell span[title='Seeders/Leechers'] font[color='green'] b::text" + ).get() + + seeders = int(seeders) if seeders and seeders.isdigit() else None + + torrent_data = { + "info_hash": info_hash, + "torrent_name": torrent_name, + "torrent_link": torrent_link, + "magnet_link": magnet_link, + "background": self.background_image, + "logo": self.logo_image, + "seeders": seeders, + "torrent_page_link": torrent_page_link, + "unique_id": tgx_unique_id, + "source": f"TorrentGalaxy ({uploader_profile_name})", + "uploader": uploader_profile_name, + "announce_list": announce_list, + "catalog": self.catalog, + "scraped_info_hash_key": self.scraped_info_hash_key, + } + + if await self.redis.sismember(self.scraped_info_hash_key, info_hash): + self.logger.info(f"Torrent already scraped: {torrent_name}") + await TorrentStreams.find_one({"_id": info_hash}).update( + {"$set": {"seeders": seeders}}, + ) + else: + yield response.follow( + torrent_page_link, + self.parse_torrent_details, + meta={"torrent_data": torrent_data}, + ) + + def parse_torrent_details(self, response): + torrent_data = response.meta["torrent_data"] + + # Extracting file details and sizes + file_details = [] + for row in response.xpath('//table[contains(@class, "table-striped")]/tr'): + file_name = row.xpath('td[@class="table_col1"]/text()').get() + file_size = row.xpath('td[@class="table_col2"]/text()').get() + if file_name and file_size: + file_details.append({"file_name": file_name, "file_size": file_size}) + if not file_details: + self.logger.warning( + f"File details not found for {torrent_data['torrent_name']}. Retrying" + ) + yield response.follow( + response.url.replace( + response.url.split("/")[2], random.choice(self.allowed_domains) + ), + self.parse_torrent_details, + meta={"torrent_data": torrent_data}, + ) + return + + torrent_data["file_details"] = file_details + + cover_image_url = response.xpath( + "//img[contains(@class, 'img-responsive') and contains(@data-src, '.png')]/@data-src" + ).get() + torrent_data["poster"] = cover_image_url + + # Getting the description for parsing video, audio, and other details + torrent_description = "".join( + response.xpath( + "//font/following-sibling::*[1]/following-sibling::text() | " + "//font/following-sibling::*[1]/following-sibling::*//text() | " + "//center/font/following::br/following-sibling::text() | " + "//strong/following-sibling::text()[normalize-space()] | " + "//strong/following-sibling::br/following-sibling::text()[normalize-space()] | " + "//div[contains(@class, 'container-fluid')]//text()[normalize-space()]" + ).extract() + ) + torrent_data["description"] = torrent_description.replace("\xa0", " ") + + total_size = response.xpath( + "//div[b='Total Size:']/following-sibling::div/text()" + ).get() + if total_size: + torrent_data["total_size"] = convert_size_to_bytes(total_size) + else: + # if the total size is not found, then tgx has shown captcha validation. + # so we need to slow down and retry the request + self.logger.warning( + f"Total size not found for {torrent_data['torrent_name']}. Retrying" + ) + # change the hostname randomly to avoid captcha + yield response.follow( + response.url.replace( + response.url.split("/")[2], random.choice(self.allowed_domains) + ), + self.parse_torrent_details, + meta={"torrent_data": torrent_data}, + ) + return + + # Extracting date created + date_created = response.xpath( + "//div[b[contains(., 'Added:')]]/following-sibling::div/text()" + ).get() + if date_created: + # Processing to extract the date and time + torrent_data["created_at"] = datetime.strptime( + date_created.strip(), "%d-%m-%Y %H:%M" + ) + + # Extracting language + language = response.xpath( + "//div[b='Language:']/following-sibling::div/text()" + ).get() + if language: + torrent_data["languages"] = [language.strip()] + + yield torrent_data diff --git a/resources/json/sports_artifacts.json b/resources/json/sports_artifacts.json index 5bba245..ffa1afa 100644 --- a/resources/json/sports_artifacts.json +++ b/resources/json/sports_artifacts.json @@ -15,19 +15,19 @@ "Basketball": { "background": [ "https://images.designtrends.com/wp-content/uploads/2015/12/21135619/Basketball-Background.jpg", - "http://wallpapercave.com/wp/acIknTe.jpg" + "https://external-content.duckduckgo.com/iu/?u=http%3A//wallpapercave.com/wp/acIknTe.jpg&f=1&nofb=1" ], "poster": [ "https://www.pixelstalk.net/wp-content/uploads/2016/05/Free-Desktop-basketball-wallpapers-court.jpg", "https://www.pixelstalk.net/wp-content/uploads/2016/10/Basketball-Court-HD-Wallpaper.jpg" ], "logo": [ - "http://cliparts.co/cliparts/Aib/Kr4/AibKr4gGT.png" + "https://external-content.duckduckgo.com/iu/?u=http%3A//cliparts.co/cliparts/Aib/Kr4/AibKr4gGT.png&f=1&nofb=1" ] }, "Baseball": { "background": [ - "http://www.pixelstalk.net/wp-content/uploads/2016/03/Nice-baseball-fields-wallpaper-background.jpg", + "https://external-content.duckduckgo.com/iu/?u=http%3A//www.pixelstalk.net/wp-content/uploads/2016/03/Nice-baseball-fields-wallpaper-background.jpg&f=1&nofb=1", "https://www.pixelstalk.net/wp-content/uploads/2016/03/Download-free-baseball-wallpaper-HD.jpg" ], "poster": [ @@ -53,7 +53,7 @@ }, "Football": { "background": [ - "http://wallpapercave.com/wp/Mx8QOkb.jpg", + "https://external-content.duckduckgo.com/iu/?u=http%3A//wallpapercave.com/wp/Mx8QOkb.jpg&f=1&nofb=1", "https://wallpapercave.com/wp/wp2694966.jpg", "https://wallpapercave.com/wp/wp8255684.jpg", "https://wallpapercave.com/wp/wp2167295.jpg" @@ -108,11 +108,11 @@ "Tennis": { "background": [ "https://wallpapercave.com/wp/wp3088699.jpg", - "http://yesofcorsa.com/wp-content/uploads/2015/08/3307_tennis.jpg", - "http://3.bp.blogspot.com/-galOVHt3kp0/UZvDZpqBBiI/AAAAAAAALD0/c0zQxrH0LxY/s1600/Tennis+Wallpapers+4.jpg" + "https://external-content.duckduckgo.com/iu/?u=http%3A//yesofcorsa.com/wp-content/uploads/2015/08/3307_tennis.jpg&f=1&nofb=1", + "https://external-content.duckduckgo.com/iu/?u=http%3A//3.bp.blogspot.com/-galOVHt3kp0/UZvDZpqBBiI/AAAAAAAALD0/c0zQxrH0LxY/s1600/Tennis%2BWallpapers%2B4.jpg&f=1&nofb=1" ], "poster": [ - "http://3.bp.blogspot.com/-galOVHt3kp0/UZvDZpqBBiI/AAAAAAAALD0/c0zQxrH0LxY/s1600/Tennis+Wallpapers+4.jpg", + "https://external-content.duckduckgo.com/iu/?u=http%3A//3.bp.blogspot.com/-galOVHt3kp0/UZvDZpqBBiI/AAAAAAAALD0/c0zQxrH0LxY/s1600/Tennis%2BWallpapers%2B4.jpg&f=1&nofb=1", "https://images.vexels.com/media/users/3/76768/raw/b6882e62c248e8d5ff5f1fd9259d9f73-tennis-racket-background.jpg" ], "logo": [ @@ -136,11 +136,11 @@ "background": [ "https://wallpaperaccess.com/full/1543778.jpg", "https://free4kwallpapers.com/uploads/originals/2018/02/06/stunning-golf-course-wallpaper.jpg", - "http://www.pixelstalk.net/wp-content/uploads/2016/04/Free-golf-backgrounds-photos-pics.jpg" + "https://external-content.duckduckgo.com/iu/?u=http%3A//www.pixelstalk.net/wp-content/uploads/2016/04/Free-golf-backgrounds-photos-pics.jpg&f=1&nofb=1" ], "poster": [ "https://images.all-free-download.com/images/graphiclarge/golf_picture_9_168173.jpg", - "http://cdn.wallpapersafari.com/97/40/W4gJLI.jpg", + "https://external-content.duckduckgo.com/iu/?u=http%3A//cdn.wallpapersafari.com/97/40/W4gJLI.jpg&f=1&nofb=1", "https://www.pixelstalk.net/wp-content/uploads/2016/04/Golf-wallpapers-HD-Photos.jpg" ], "logo": [ @@ -153,7 +153,7 @@ "https://www.baltana.com/files/wallpapers-21/Darts-Arrow-HD-Background-Wallpaper-52811.jpg" ], "poster": [ - "http://4.bp.blogspot.com/-HbSvu5nU_KQ/URFbg6VXfFI/AAAAAAAASg0/92zXW__vnqA/s1600/Darts+Wallpapers+01.jpg", + "https://external-content.duckduckgo.com/iu/?u=http%3A//4.bp.blogspot.com/-HbSvu5nU_KQ/URFbg6VXfFI/AAAAAAAASg0/92zXW__vnqA/s1600/Darts%2BWallpapers%2B01.jpg&f=1&nofb=1", "https://wallpapershome.com/images/wallpapers/darts-1920x1080-4k-5k-wallpaper-hd-wheel-target-fire-water-561.jpg" ], "logo": [ @@ -166,7 +166,7 @@ "https://wallpapercave.com/wp/wp1998982.jpg" ], "poster": [ - "http://rickrifici.com/wp-content/uploads/2011/03/AFL.jpg", + "https://external-content.duckduckgo.com/iu/?u=http%3A//rickrifici.com/wp-content/uploads/2011/03/AFL.jpg&f=1&nofb=1", "https://i.pinimg.com/736x/c0/54/73/c0547342c769706d2ed606d2b6f7246b.jpg" ], "logo": [ @@ -187,6 +187,20 @@ "https://static.vecteezy.com/system/resources/thumbnails/000/130/098/small/turbocharger-racing-logo-template.jpg" ] }, + "MotoGP": { + "background": [ + "https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2Fwallpaperbat.com%2Fimg%2F569936-marc-marquez-wallpaper-in-2020-yamaha-motogp-racing-motogp-race.jpg&f=1&nofb=1&ipt=a0e19f2afd8aa9cc2aaeecf47502ed36fa0d2c6565eaf01d437505dcc0e8923f&ipo=images", + "https://external-content.duckduckgo.com/iu/?u=http%3A%2F%2Fgetwallpapers.com%2Fwallpaper%2Ffull%2F9%2F1%2F1%2F479678.jpg&f=1&nofb=1&ipt=074cb784018cb9d5ebf40c6a97511dcdf9b18e87b2a27e3fe43296ec12437f9c&ipo=images" + ], + "poster": [ + "https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2Fi.pinimg.com%2Foriginals%2F62%2F0e%2F24%2F620e24ceb7964d0168606932e4a976cf.jpg&f=1&nofb=1&ipt=1965087b235796e0dbf101d6329c1f7f14b5dfc5acfb48c075739108c6eddba9&ipo=images", + "https://images.wallpapersden.com/image/wxl-motogp-19-game_66061.jpg" + ], + "logo": [ + "https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2F1.bp.blogspot.com%2F-aLMSit3Q6XY%2FUQJO5Eiac4I%2FAAAAAAAAFcE%2Fueo2j7cdk-c%2Fs320%2FLogo%2BMotogp.jpg&f=1&nofb=1&ipt=8e81f529ea52045afffc5c64a7bbd8cde17f35a4c5f8eca685a0488f9605bfcf&ipo=images", + "https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2Fbesthqwallpapers.com%2FUploads%2F25-12-2017%2F34975%2Fthumb-motogp-4k-logo-grunge-black-background.jpg&f=1&nofb=1&ipt=e934a488b5c925ba56ad11f3b2cb728e85b4e34ac217441c0bf6c4cb4a07db9d&ipo=images" + ] + }, "Other Sports": { "background": [ "https://wallpapercave.com/wp/Cd7iyFl.jpg" @@ -195,7 +209,7 @@ "https://png.pngtree.com/thumb_back/fh260/background/20210312/pngtree-green-gradient-sports-background-image_582399.jpg" ], "logo": [ - "http://images.gofreedownload.net/sport-105683.jpg" + "https://external-content.duckduckgo.com/iu/?u=http%3A//images.gofreedownload.net/sport-105683.jpg&f=1&nofb=1" ] }, "MLB": { @@ -205,7 +219,7 @@ ], "poster": [ "https://i.pinimg.com/736x/c0/54/73/c0547342c769706d2ed606d2b6f7246b.jpg", - "http://rickrifici.com/wp-content/uploads/2011/03/AFL.jpg" + "https://external-content.duckduckgo.com/iu/?u=http%3A//rickrifici.com/wp-content/uploads/2011/03/AFL.jpg&f=1&nofb=1" ], "logo": [ "https://botw-pd.s3.amazonaws.com/styles/logo-thumbnail/s3/0003/4002/brand.gif?itok=F2rATKp8" diff --git a/resources/manifest.json b/resources/manifest.json index f874fc9..9b012c8 100644 --- a/resources/manifest.json +++ b/resources/manifest.json @@ -415,6 +415,17 @@ } ] }, + { + "id": "motogp_racing", + "type": "series", + "name": "MotoGP Racing", + "extra": [ + { + "name": "skip", + "isRequired": false + } + ] + }, { "id": "american_football", "type": "movie", diff --git a/scrapers/helpers.py b/scrapers/helpers.py index 2c537b7..2229d66 100644 --- a/scrapers/helpers.py +++ b/scrapers/helpers.py @@ -11,7 +11,7 @@ from urllib3.util.retry import Retry from db.config import settings from db.models import TorrentStreams, Episode, Season from utils.const import UA_HEADER -from utils.parser import get_json_data +from utils import get_json_data from utils.torrent import extract_torrent_metadata, info_hashes_to_torrent_metadata # set httpx logging level diff --git a/utils/__init__.py b/utils/__init__.py index e69de29..1848f0e 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -0,0 +1,6 @@ +import json + + +def get_json_data(file_name: str) -> dict: + with open(file_name) as file: + return json.load(file) diff --git a/utils/const.py b/utils/const.py index a947225..5701fa4 100644 --- a/utils/const.py +++ b/utils/const.py @@ -28,6 +28,7 @@ CATALOG_ID_DATA = [ "mediafusion_search_movies", "mediafusion_search_series", "mediafusion_search_tv", + "motogp_racing", "other_sports", "prowlarr_streams", "rugby", @@ -74,6 +75,7 @@ CATALOG_NAME_DATA = [ "MediaFusion Search Movies", "MediaFusion Search Series", "MediaFusion Search TV", + "MotoGP Racing", "Other Sports", "Prowlarr Streams", "Rugby/AFL", diff --git a/utils/parser.py b/utils/parser.py index acee40b..d4b7317 100644 --- a/utils/parser.py +++ b/utils/parser.py @@ -1,10 +1,7 @@ import asyncio -import json import math import re -import requests -from imdb import Cinemagoer from redis.asyncio import Redis from db.config import settings @@ -355,8 +352,3 @@ def is_contain_18_plus_keywords(title: str) -> bool: Check if the title contains 18+ keywords to filter out adult content. """ return ADULT_CONTENT_KEYWORDS.search(title) is not None - - -def get_json_data(file_name: str) -> dict: - with open(file_name) as file: - return json.load(file) diff --git a/utils/runtime_const.py b/utils/runtime_const.py index e2ef034..ceee811 100644 --- a/utils/runtime_const.py +++ b/utils/runtime_const.py @@ -1,7 +1,7 @@ import re from db.config import settings - +from utils import get_json_data ADULT_CONTENT_KEYWORDS = re.compile( settings.adult_content_regex_keywords, @@ -15,3 +15,5 @@ PARENT_GUIDE_CERTIFICATES_FILTER_REGEX = re.compile( settings.parent_guide_certificates_filter_regex, re.IGNORECASE, ) + +SPORTS_ARTIFACTS = get_json_data("resources/json/sports_artifacts.json")