Viren070/MediaFusion
1
0
Fork 0
mirror of https://github.com/Viren070/MediaFusion.git synced 2025-12-01 23:21:11 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix TGX scrapy spiders & scrape file data with the magnet pipeline if the page provides wrong file information

Browse Source
This commit is contained in:
mhdzumair
2024-12-14 07:17:28 +05:30
parent b3e2ca8f64
commit 07a0c61d12
5 changed files with 86 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediafusion_scrapy/pipelines/formula_parser_pipeline.py
+40 -16
View File
@@ -1,5 +1,4 @@
import logging
import logging
import re
from datetime import datetime
@@ -39,7 +38,7 @@ class FormulaParserPipeline:
r"Formula\s*(?P<Series>[123])" # Series name with optional space
r"\.\s*(?P<Year>\d{4})" # Year with flexible spacing
r"\.\s*R(?P<Round>\d+)" # Round number with flexible spacing
r"\.\s*(?P<Event>[^\s]+)" # Event name without space before broadcaster
r"\.\s*(?P<Event>\S+)" # Event name without space before broadcaster
r"\s*(?P<Broadcaster>SkyF1HD|Sky Sports Main Event UHD|Sky Sports F1 UHD|Sky Sports Arena|V Sport Ultra HD|SkyF1)" # Broadcaster without preceding dot
r"\.\s*(?P<Resolution>\d+P)" # Resolution with flexible spacing
),
@@ -184,7 +183,7 @@ class FormulaParserPipeline:
def process_item(self, item, spider):
uploader = item["uploader"]
title = re.sub(r"\.\.+", ".", item["torrent_name"])
title = re.sub(r"\.\.+", ".", item["torrent_title"])
self.title_parser_functions[uploader](title, item)
if not item.get("title"):
raise DropItem(f"Title not parsed: {title}")
@@ -313,7 +312,9 @@ class FormulaParserPipeline:
def parse_egortech_description(self, torrent_data: dict):
torrent_description = torrent_data.get("description")
file_details = torrent_data.get("file_details")
file_data = torrent_data.get("file_data")
if not file_data:
return
quality_match = re.search(r"Quality:\s*(\S+)", torrent_description)
codec_match = re.search(r"Video:\s*([A-Za-z0-9]+)", torrent_description)
@@ -340,15 +341,20 @@ class FormulaParserPipeline:
if item.strip()
]
for index, (event, file_detail) in enumerate(zip(events, file_details)):
for index, (event, file_detail) in enumerate(zip(events, file_data)):
data = self.episode_name_parser_egortech(
event, torrent_data["created_at"]
)
size = (
convert_size_to_bytes(file_detail.get("size"))
if isinstance(file_detail.get("size"), str)
else file_detail.get("size")
)
episodes.append(
Episode(
episode_number=index + 1,
filename=file_detail.get("file_name"),
size=convert_size_to_bytes(file_detail.get("file_size")),
filename=file_detail.get("filename"),
size=size,
file_index=index,
title=data["title"],
released=data["date"],
@@ -356,17 +362,23 @@ class FormulaParserPipeline:
)
else:
# logic to parse episode details directly from file details when description does not contain "Contains:"
for index, file_detail in enumerate(file_details):
file_name = file_detail.get("file_name")
file_size = file_detail.get("file_size")
for index, file_detail in enumerate(file_data):
file_name = file_detail.get("filename")
file_size = file_detail.get("size")
data = self.episode_name_parser_egortech(
file_name, torrent_data["created_at"]
)
size = (
convert_size_to_bytes(file_size)
if isinstance(file_size, str)
else file_size
)
episodes.append(
Episode(
episode_number=index + 1,
filename=file_name,
size=convert_size_to_bytes(file_size),
size=size,
file_index=index,
title=data["title"],
released=data["date"],
@@ -392,6 +404,10 @@ class FormulaParserPipeline:
if torrent_description is None:
return
file_data = torrent_data.get("file_data")
if not file_data:
return
audio_section = re.search(r"Audios:\n(.+)", torrent_description)
if audio_section:
@@ -404,7 +420,7 @@ class FormulaParserPipeline:
torrent_data["episodes"] = [
Episode(
episode_number=1,
filename=torrent_data["file_details"][0].get("file_name"),
filename=file_data[0].get("filename"),
size=torrent_data.get("total_size"),
title=torrent_data.get("episode_name"),
released=torrent_data.get("created_at"),
@@ -414,6 +430,9 @@ class FormulaParserPipeline:
def parse_smcgill1969_description(self, torrent_data: dict):
torrent_description = torrent_data.get("description")
file_data = torrent_data.get("file_data")
if not file_data:
return
codec_matches = re.findall(r"Codec ID\s*:\s*(\S+)", torrent_description)
if codec_matches:
@@ -424,15 +443,20 @@ class FormulaParserPipeline:
torrent_data["is_add_title_to_poster"] = True
episodes = []
for index, file_detail in enumerate(torrent_data.get("file_details", [])):
file_name = file_detail.get("file_name")
file_size = file_detail.get("file_size")
for index, file_detail in enumerate(file_data):
file_name = file_detail.get("filename")
file_size = file_detail.get("size")
size = (
convert_size_to_bytes(file_size)
if isinstance(file_size, str)
else file_size
)
episodes.append(
Episode(
episode_number=index + 1,
filename=file_name,
size=convert_size_to_bytes(file_size),
size=size,
file_index=index,
title=" ".join(file_name.split(".")[1:-1]),
released=torrent_data.get("created_at"),
mediafusion_scrapy/pipelines/moto_gp_parser_pipeline.py
+10 -5
View File
@@ -52,7 +52,7 @@ class MotoGPParserPipeline:
def process_item(self, item, spider):
uploader = item["uploader"]
title = re.sub(r"\.\.+", ".", item["torrent_name"])
title = re.sub(r"\.\.+", ".", item["torrent_title"])
self.title_parser_functions[uploader](title, item)
if not item.get("title"):
raise DropItem(f"Title not parsed: {title}")
@@ -117,15 +117,20 @@ class MotoGPParserPipeline:
torrent_data["is_add_title_to_poster"] = True
episodes = []
for index, file_detail in enumerate(torrent_data.get("file_details", [])):
file_name = file_detail.get("file_name")
file_size = file_detail.get("file_size")
for index, file_detail in enumerate(torrent_data.get("file_data", [])):
file_name = file_detail.get("filename")
file_size = file_detail.get("size")
size = (
convert_size_to_bytes(file_size)
if isinstance(file_size, str)
else file_size
)
episodes.append(
Episode(
episode_number=index + 1,
filename=file_name,
size=convert_size_to_bytes(file_size),
size=size,
file_index=index,
title=" ".join(file_name.split(".")[1:-1]),
released=torrent_data.get("created_at"),
mediafusion_scrapy/pipelines/sports_parser_pipeline.py
+10 -5
View File
@@ -40,7 +40,7 @@ class BaseParserPipeline:
self.static_poster = static_poster or {}
def process_item(self, item, spider):
title = re.sub(r"\.\.+", ".", item["torrent_name"])
title = re.sub(r"\.\.+", ".", item["torrent_title"])
self.parse_title(title, item)
if not item.get("title"):
raise DropItem(f"Title not parsed: {title}")
@@ -108,13 +108,17 @@ class BaseParserPipeline:
torrent_data["resolution"] = resolution_match.group(1) + "p"
largest_file = max(
torrent_data["file_details"],
key=lambda x: convert_size_to_bytes(x["file_size"]),
torrent_data["file_data"],
key=lambda x: (
convert_size_to_bytes(x["size"])
if isinstance(x["size"], str)
else x["size"]
),
)
largest_file_index = torrent_data["file_details"].index(largest_file)
largest_file_index = torrent_data["file_data"].index(largest_file)
torrent_data["largest_file"] = {
"index": largest_file_index,
"filename": largest_file["file_name"],
"filename": largest_file["filename"],
}
def update_imdb_data(self, torrent_data: dict):
@@ -227,6 +231,7 @@ class UFCParserPipeline(BaseParserPipeline):
dict(
poster=tmdb_data["poster"],
background=tmdb_data["background"],
is_add_title_to_poster=False,
imdb_rating=tmdb_data["tmdb_rating"],
description=tmdb_data["description"],
)
mediafusion_scrapy/pipelines/torrent_parser_pipeline.py
+4 -2
View File
@@ -58,8 +58,10 @@ class TorrentDownloadAndParsePipeline:
class MagnetDownloadAndParsePipeline:
async def process_item(self, item, spider):
adapter = ItemAdapter(item)
magnet_link = adapter.get("magnet_link")
if item.get("file_data"):
return item
magnet_link = item.get("magnet_link")
if not magnet_link:
raise DropItem(f"No magnet link found in item: {item}")
mediafusion_scrapy/spiders/tgx.py
+22 -43
View File
@@ -46,13 +46,6 @@ class TgxSpider(scrapy.Spider):
"wait_until": "domcontentloaded",
"timeout": 60000,
},
# "playwright_page_methods": [
# PageMethod(
# "wait_for_selector",
# "#smallguestnav",
# timeout=60000,
# ),
# ],
"is_search_query": False,
"parse_url": parse_url,
},
@@ -68,13 +61,6 @@ class TgxSpider(scrapy.Spider):
"wait_until": "domcontentloaded",
"timeout": 60000,
},
# "playwright_page_methods": [
# PageMethod(
# "wait_for_selector",
# "#smallguestnav",
# timeout=60000,
# ),
# ],
"is_search_query": True,
"parse_url": parse_url,
},
@@ -168,13 +154,14 @@ class TgxSpider(scrapy.Spider):
torrent_data = {
"info_hash": info_hash,
"torrent_title": torrent_name,
"torrent_name": torrent_name,
"torrent_link": torrent_link,
"magnet_link": magnet_link,
"background": self.background_image,
"logo": self.logo_image,
"seeders": seeders,
"torrent_page_link": torrent_page_link,
"website": torrent_page_link,
"unique_id": tgx_unique_id,
"source": "TorrentGalaxy",
"uploader": uploader_profile_name,
@@ -200,35 +187,34 @@ class TgxSpider(scrapy.Spider):
"referer": response.url,
"timeout": 60000,
},
# "playwright_page_methods": [
# PageMethod(
# "wait_for_selector",
# "#smallguestnav",
# timeout=60000,
# ),
# ],
"torrent_data": torrent_data,
},
)
def parse_torrent_details(self, response):
torrent_data = response.meta["torrent_data"]
torrent_data = response.meta["torrent_data"].copy()
# Extracting file details and sizes
file_details = []
for row in response.xpath('//table[contains(@class, "table-striped")]//tr'):
file_name = row.xpath('td[@class="table_col1"]/text()').get()
file_size = row.xpath('td[@class="table_col2"]/text()').get()
if file_name and file_size:
file_details.append({"file_name": file_name, "file_size": file_size})
if not file_details:
if response.xpath("//blockquote[contains(., 'GALAXY CHECKPOINT')]"):
self.logger.warning(
f"File details not found for {torrent_data['torrent_name']}. Retrying"
f"Encountered GALAXY CHECKPOINT. Retrying: {response.url}"
)
yield self.retry_request(response)
return
torrent_data["file_details"] = file_details
# Extracting file details and sizes if available
file_data = []
file_list = response.xpath(
'//button[contains(@class, "flist")]//em/text()'
).get()
file_count = int(file_list.strip("()")) if file_list else 0
for row in response.xpath('//table[contains(@class, "table-striped")]//tr'):
file_name = row.xpath('td[@class="table_col1"]/text()').get()
file_size = row.xpath('td[@class="table_col2"]/text()').get()
if file_name and file_size:
file_data.append({"filename": file_name, "size": file_size})
if file_count == len(file_data):
torrent_data["file_data"] = file_data
cover_image_url = response.xpath(
"//div[contains(@class, 'container-fluid')]/center//img[contains(@class, 'img-responsive')]/@data-src"
@@ -300,13 +286,6 @@ class TgxSpider(scrapy.Spider):
"referer": request.url,
"timeout": 60000,
},
# "playwright_page_methods": [
# PageMethod(
# "wait_for_selector",
# "#smallguestnav",
# timeout=60000,
# ),
# ],
"torrent_data": request.meta["torrent_data"],
"retry_count": retry_count + 1,
},
@@ -332,7 +311,7 @@ class FormulaTgxSpider(TgxSpider):
custom_settings = {
"ITEM_PIPELINES": {
"mediafusion_scrapy.pipelines.TorrentDuplicatesPipeline": 100,
"mediafusion_scrapy.pipelines.MagnetDownloadAndParsePipeline": 100,
"mediafusion_scrapy.pipelines.FormulaParserPipeline": 200,
"mediafusion_scrapy.pipelines.EventSeriesStorePipeline": 300,
},
@@ -361,7 +340,7 @@ class MotoGPTgxSpider(TgxSpider):
custom_settings = {
"ITEM_PIPELINES": {
"mediafusion_scrapy.pipelines.TorrentDuplicatesPipeline": 100,
"mediafusion_scrapy.pipelines.MagnetDownloadAndParsePipeline": 100,
"mediafusion_scrapy.pipelines.MotoGPParserPipeline": 200,
"mediafusion_scrapy.pipelines.EventSeriesStorePipeline": 300,
},
@@ -381,7 +360,7 @@ class BaseEventSpider(TgxSpider):
def get_custom_settings(pipeline):
return {
"ITEM_PIPELINES": {
"mediafusion_scrapy.pipelines.TorrentDuplicatesPipeline": 100,
"mediafusion_scrapy.pipelines.MagnetDownloadAndParsePipeline": 100,
f"mediafusion_scrapy.pipelines.{pipeline}": 200,
"mediafusion_scrapy.pipelines.MovieStorePipeline": 300,
},