Viren070/MediaFusion
1
0
Fork 0
mirror of https://github.com/Viren070/MediaFusion.git synced 2025-12-01 23:21:11 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add support for MotoGP & refactor

Browse Source
This commit is contained in:
mhdzumair
2024-05-07 13:58:43 +05:30
parent c22bb09c23
commit 9768a19a0e
18 changed files with 444 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/main.py
+2 -2
View File
@@ -26,13 +26,13 @@ from metrics.routes import metrics_router
from scrapers.routes import router as scrapers_router
from streaming_providers import mapper
from streaming_providers.routes import router as streaming_provider_router
from utils import crypto, torrent, poster, const, wrappers
from utils import crypto, torrent, poster, const, wrappers, get_json_data
from utils.lock import (
acquire_scheduler_lock,
maintain_heartbeat,
release_scheduler_lock,
)
from utils.parser import generate_manifest, get_json_data
from utils.parser import generate_manifest
logging.basicConfig(
format="%(levelname)s::%(asctime)s - %(message)s",
mediafusion_scrapy/pipelines.py
+127 -1
View File
@@ -1,5 +1,6 @@
import asyncio
import logging
import random
import re
from datetime import datetime
from uuid import uuid4
@@ -24,6 +25,7 @@ from db.models import (
from db.schemas import TVMetaData
from utils import torrent, const
from utils.parser import convert_size_to_bytes
from utils.runtime_const import SPORTS_ARTIFACTS
class TorrentDuplicatesPipeline:
@@ -470,6 +472,130 @@ class FormulaParserPipeline:
torrent_data["episodes"] = episodes
class MotoGPParserPipeline:
def __init__(self):
self.name_parser_patterns = {
"smcgill1969": [
re.compile(
r"MotoGP" # Series name with flexible space or dot
r"\.(?P<Year>\d{4})" # Year
r"x(?P<Round>\d{2})" # Round
r"\.(?P<EventAndEpisodeName>.+?)" # Event and Episode name
r"\.(?P<Broadcaster>BTSportHD|TNTSportsHD)" # Broadcaster
r"\.(?P<Resolution>4K|SD|1080p)" # Resolution and Quality
),
re.compile(
r"MotoGP" # Series name with flexible space or dot
r"\.(?P<Year>\d{4})" # Year
r"(?:-\d{2}-\d{2})?" # ignore Date part starting with a hyphen
r"\.(?P<EventAndEpisodeName>.+?)" # Event and Episode name
r"(?:\.(?P<Quality>WEB-DL|HDTV))?" # Optional Quality
r"(?:\.(?P<Broadcaster>BTSportHD|TNTSportsHD))?" # Optional Broadcaster
r"\.(?P<Resolution>4K|SD|1080p)" # Resolution
),
],
}
self.default_poster = random.choice(SPORTS_ARTIFACTS["MotoGP"]["poster"])
self.smcgill1969_resolutions = {
"4K": "4K",
"SD": "576p",
"1080p": "1080p",
}
self.known_countries_first_words = ["San", "Great"]
self.title_parser_functions = {
"smcgill1969": self.parse_smcgill1969_title,
}
self.description_parser_functions = {
"smcgill1969": self.parse_smcgill1969_description,
}
def process_item(self, item, spider):
uploader = item["uploader"]
title = re.sub(r"\.\.+", ".", item["torrent_name"])
self.title_parser_functions[uploader](title, item)
if not item.get("title"):
raise DropItem(f"Title not parsed: {title}")
self.description_parser_functions[uploader](item)
if not item.get("episodes"):
raise DropItem(f"Episodes not parsed: {item!r}")
return item
def event_and_episode_name_parser(self, event_and_episode_name):
parts = event_and_episode_name.split(".")
if parts[0] in self.known_countries_first_words and len(parts) > 1:
event = f"{parts[0]} {parts[1]}"
episode_name = " ".join(parts[2:])
else:
event = parts[0]
episode_name = " ".join(parts[1:])
return event, episode_name
def parse_smcgill1969_title(self, title, torrent_data: dict):
for pattern in self.name_parser_patterns.get("smcgill1969"):
match = pattern.match(title)
if match:
data = match.groupdict()
series = "MotoGP"
event, episode_name = self.event_and_episode_name_parser(
data["EventAndEpisodeName"]
)
torrent_data.update(
{
"title": " ".join(
filter(
None,
[
series,
data.get("Year"),
event,
"(smcgill1969)", # add the uploader to the title for uniqueness
],
)
),
"year": int(data["Year"]),
"resolution": self.smcgill1969_resolutions.get(
data["Resolution"]
),
"event": event,
"episode_name": episode_name,
}
)
return
logging.warning(f"Failed to parse title: {title}")
def parse_smcgill1969_description(self, torrent_data: dict):
torrent_description = torrent_data.get("description")
codec_matches = re.findall(r"Codec ID\s*:\s*(\S+)", torrent_description)
if codec_matches:
torrent_data["codec"] = codec_matches[0]
torrent_data["audio"] = codec_matches[1] if len(codec_matches) > 1 else None
torrent_data["poster"] = self.default_poster
torrent_data["is_add_title_to_poster"] = True
episodes = []
for index, file_detail in enumerate(torrent_data.get("file_details", [])):
file_name = file_detail.get("file_name")
file_size = file_detail.get("file_size")
episodes.append(
Episode(
episode_number=index + 1,
filename=file_name,
size=convert_size_to_bytes(file_size),
file_index=index,
title=" ".join(file_name.split(".")[1:-1]),
released=torrent_data.get("created_at"),
)
)
torrent_data["episodes"] = episodes
class QueueBasedPipeline:
def __init__(self):
self.queue = asyncio.Queue()
@@ -509,7 +635,7 @@ class QueueBasedPipeline:
raise NotImplementedError
class FormulaStorePipeline(QueueBasedPipeline):
class EventSeriesStorePipeline(QueueBasedPipeline):
def __init__(self):
super().__init__()
self.redis = redis_async.Redis.from_url(settings.redis_url)
mediafusion_scrapy/settings.py
+1 -1
View File
@@ -94,4 +94,4 @@ HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
FEED_EXPORT_ENCODING = "utf-8"
LOG_LEVEL = "DEBUG"
LOG_LEVEL = "INFO"
mediafusion_scrapy/spiders/crictime.py
+4 -5
View File
@@ -5,7 +5,7 @@ import redis
import scrapy
from db.config import settings
from utils.parser import get_json_data
from utils.runtime_const import SPORTS_ARTIFACTS
class CricTimeSpider(scrapy.Spider):
@@ -25,7 +25,6 @@ class CricTimeSpider(scrapy.Spider):
self.redis = redis.Redis(
connection_pool=redis.ConnectionPool.from_url(settings.redis_url)
)
self.sports_artifacts = get_json_data("resources/json/sports_artifacts.json")
def __del__(self):
self.redis.close()
@@ -57,11 +56,11 @@ class CricTimeSpider(scrapy.Spider):
item = {
"genres": [category],
"poster": random.choice(self.sports_artifacts[category]["poster"]),
"poster": random.choice(SPORTS_ARTIFACTS[category]["poster"]),
"background": random.choice(
self.sports_artifacts[category]["background"]
SPORTS_ARTIFACTS[category]["background"]
),
"logo": random.choice(self.sports_artifacts[category]["logo"]),
"logo": random.choice(SPORTS_ARTIFACTS[category]["logo"]),
"is_add_title_to_poster": True,
"title": event_name,
"url": response.urljoin(event_url),
mediafusion_scrapy/spiders/dlhd.py
+4 -7
View File
@@ -5,7 +5,7 @@ from datetime import datetime
import pytz
import scrapy
from utils.parser import get_json_data
from utils.runtime_const import SPORTS_ARTIFACTS
class DaddyLiveHDSpider(scrapy.Spider):
@@ -60,7 +60,6 @@ class DaddyLiveHDSpider(scrapy.Spider):
def __init__(self, *args, **kwargs):
super(DaddyLiveHDSpider, self).__init__(*args, **kwargs)
self.sports_artifacts = get_json_data("resources/json/sports_artifacts.json")
self.gmt = pytz.timezone("Etc/GMT")
def parse(self, response, **kwargs):
@@ -85,13 +84,11 @@ class DaddyLiveHDSpider(scrapy.Spider):
item = {
"stream_source": "DaddyLiveHD (1.dlhd.sx)",
"genres": [category],
"poster": random.choice(
self.sports_artifacts[category]["poster"]
),
"poster": random.choice(SPORTS_ARTIFACTS[category]["poster"]),
"background": random.choice(
self.sports_artifacts[category]["background"]
SPORTS_ARTIFACTS[category]["background"]
),
"logo": random.choice(self.sports_artifacts[category]["logo"]),
"logo": random.choice(SPORTS_ARTIFACTS[category]["logo"]),
"is_add_title_to_poster": True,
"title": event["event"],
"channels": event["channels"],
mediafusion_scrapy/spiders/formula_tgx.py
+9 -188
View File
@@ -1,205 +1,26 @@
import random
import re
from datetime import datetime
import redis.asyncio as redis
import scrapy
from db.config import settings
from db.models import TorrentStreams
from utils.parser import convert_size_to_bytes
from utils.torrent import parse_magnet
from mediafusion_scrapy.spiders.tgx import TgxSpider
class FormulaTgxSpider(scrapy.Spider):
class FormulaTgxSpider(TgxSpider):
name = "formula_tgx"
allowed_domains = ["torrentgalaxy.to", "tgx.rs", "torrentgalaxy.mx", "tgx.sb"]
formula_uploader_profiles = [
uploader_profiles = [
"egortech",
"F1Carreras",
"smcgill1969",
]
catalog = ["formula_racing"]
background_image = "https://i.postimg.cc/S4wcrGRZ/f1background.png?dl=1"
logo_image = "https://i.postimg.cc/Sqf4V8tj/f1logo.png?dl=1"
formula1_keyword_patterns = re.compile(r"formula[ .+]*[1234e]+", re.IGNORECASE)
keyword_patterns = re.compile(r"formula[ .+]*[1234e]+", re.IGNORECASE)
scraped_info_hash_key = "formula_tgx_scraped_info_hash"
custom_settings = {
"ITEM_PIPELINES": {
"mediafusion_scrapy.pipelines.TorrentDuplicatesPipeline": 100,
"mediafusion_scrapy.pipelines.FormulaParserPipeline": 200,
"mediafusion_scrapy.pipelines.FormulaStorePipeline": 300,
"mediafusion_scrapy.pipelines.EventSeriesStorePipeline": 300,
}
}
def __init__(self, scrape_all: str = "True", *args, **kwargs):
super(FormulaTgxSpider, self).__init__(*args, **kwargs)
self.scrape_all = scrape_all.lower() == "true"
self.redis = redis.Redis(
connection_pool=redis.ConnectionPool.from_url(settings.redis_url)
)
self.scraped_info_hash_key = "formula_tgx_scraped_info_hash"
async def __aexit__(self, exc_type, exc_val, exc_tb):
await self.redis.aclose()
def start_requests(self):
for uploader_profile in self.formula_uploader_profiles:
yield scrapy.Request(
f"https://{random.choice(self.allowed_domains)}/profile/{uploader_profile}/torrents/0",
self.parse,
meta={"uploader_profile": uploader_profile},
)
async def parse(self, response, **kwargs):
uploader_profile_name = response.meta["uploader_profile"]
self.logger.info(f"Scraping torrents from {uploader_profile_name}")
# Extract the last page number only once at the beginning
if self.scrape_all and response.url.endswith("0"):
last_page_number = response.css(
"ul.pagination li.page-item:not(.disabled) a::attr(href)"
).re(r"/profile/.*/torrents/(\d+)")[-2]
last_page_number = (
int(last_page_number) if last_page_number.isdigit() else 0
)
# Generate requests for all pages
for page_number in range(1, last_page_number + 1):
next_page_url = (
f"{response.url.split('/torrents/')[0]}/torrents/{page_number}"
)
yield response.follow(next_page_url, self.parse, meta=response.meta)
# Extract torrents from the page
for torrent in response.css("div.tgxtablerow.txlight"):
urls = torrent.css("div.tgxtablecell a::attr(href)").getall()
torrent_name = torrent.css(
"div.tgxtablecell.clickable-row.click.textshadow.rounded.txlight a b::text"
).get()
if not self.formula1_keyword_patterns.search(torrent_name):
continue
tgx_unique_id = urls[0].split("/")[-2]
torrent_page_link = response.urljoin(urls[0])
torrent_link = torrent.css(
'a[href*="watercache.nanobytes.org"]::attr(href)'
).get()
magnet_link = torrent.css('a[href^="magnet:?"]::attr(href)').get()
info_hash, announce_list = parse_magnet(magnet_link)
if not info_hash:
self.logger.warning(
f"Failed to parse magnet link: {response.url}, {torrent_name}"
)
continue
seeders = torrent.css(
"div.tgxtablecell span[title='Seeders/Leechers'] font[color='green'] b::text"
).get()
seeders = int(seeders) if seeders and seeders.isdigit() else None
torrent_data = {
"info_hash": info_hash,
"torrent_name": torrent_name,
"torrent_link": torrent_link,
"magnet_link": magnet_link,
"background": "https://i.postimg.cc/S4wcrGRZ/f1background.png?dl=1",
"logo": "https://i.postimg.cc/Sqf4V8tj/f1logo.png?dl=1",
"seeders": seeders,
"torrent_page_link": torrent_page_link,
"unique_id": tgx_unique_id,
"source": f"TorrentGalaxy ({uploader_profile_name})",
"uploader": uploader_profile_name,
"announce_list": announce_list,
"catalog": ["formula_racing"],
"scraped_info_hash_key": self.scraped_info_hash_key,
}
if await self.redis.sismember(self.scraped_info_hash_key, info_hash):
self.logger.info(f"Torrent already scraped: {torrent_name}")
await TorrentStreams.find_one({"_id": info_hash}).update(
{"$set": {"seeders": seeders}},
)
else:
yield response.follow(
torrent_page_link,
self.parse_torrent_details,
meta={"torrent_data": torrent_data},
)
def parse_torrent_details(self, response):
torrent_data = response.meta["torrent_data"]
# Extracting file details and sizes
file_details = []
for row in response.xpath('//table[contains(@class, "table-striped")]/tr'):
file_name = row.xpath('td[@class="table_col1"]/text()').get()
file_size = row.xpath('td[@class="table_col2"]/text()').get()
if file_name and file_size:
file_details.append({"file_name": file_name, "file_size": file_size})
if not file_details:
self.logger.warning(
f"File details not found for {torrent_data['torrent_name']}. Retrying"
)
yield response.follow(
response.url,
self.parse_torrent_details,
meta={"torrent_data": torrent_data},
)
return
torrent_data["file_details"] = file_details
cover_image_url = response.xpath(
"//img[contains(@class, 'img-responsive') and contains(@data-src, '.png')]/@data-src"
).get()
torrent_data["poster"] = cover_image_url
# Getting the description for parsing video, audio, and other details
torrent_description = "".join(
response.xpath(
"//font/following-sibling::*[1]/following-sibling::text() | "
"//font/following-sibling::*[1]/following-sibling::*//text() | "
"//center/font/following::br/following-sibling::text() | "
"//strong/following-sibling::text()[normalize-space()] | "
"//strong/following-sibling::br/following-sibling::text()[normalize-space()] | "
"//div[contains(@class, 'container-fluid')]//text()[normalize-space()]"
).extract()
)
torrent_data["description"] = torrent_description.replace("\xa0", " ")
total_size = response.xpath(
"//div[b='Total Size:']/following-sibling::div/text()"
).get()
if total_size:
torrent_data["total_size"] = convert_size_to_bytes(total_size)
else:
# if the total size is not found, then tgx has shown captcha validation.
# so we need to slow down and retry the request
self.logger.warning(
f"Total size not found for {torrent_data['torrent_name']}. Retrying"
)
yield response.follow(
response.url,
self.parse_torrent_details,
meta={"torrent_data": torrent_data},
)
# Extracting date created
date_created = response.xpath(
"//div[b[contains(., 'Added:')]]/following-sibling::div/text()"
).get()
if date_created:
# Processing to extract the date and time
torrent_data["created_at"] = datetime.strptime(
date_created.strip(), "%d-%m-%Y %H:%M"
)
# Extracting language
language = response.xpath(
"//div[b='Language:']/following-sibling::div/text()"
).get()
if language:
torrent_data["languages"] = [language.strip()]
yield torrent_data
mediafusion_scrapy/spiders/motogp_tgx.py
+26
View File
@@ -0,0 +1,26 @@
import re
import random
from mediafusion_scrapy.spiders.tgx import TgxSpider
from utils.runtime_const import SPORTS_ARTIFACTS
class MotoGPTgxSpider(TgxSpider):
name = "motogp_tgx"
uploader_profiles = [
"smcgill1969",
]
catalog = ["motogp_racing"]
keyword_patterns = re.compile(r"MotoGP[ .+]*", re.IGNORECASE)
scraped_info_hash_key = "motogp_tgx_scraped_info_hash"
background_image = random.choice(SPORTS_ARTIFACTS["MotoGP"]["background"])
logo_image = random.choice(SPORTS_ARTIFACTS["MotoGP"]["logo"])
custom_settings = {
"ITEM_PIPELINES": {
"mediafusion_scrapy.pipelines.TorrentDuplicatesPipeline": 100,
"mediafusion_scrapy.pipelines.MotoGPParserPipeline": 200,
"mediafusion_scrapy.pipelines.EventSeriesStorePipeline": 300,
}
}
mediafusion_scrapy/spiders/mrgamingstreams.py
+4 -7
View File
@@ -7,7 +7,7 @@ import redis
import scrapy
from db.config import settings
from utils.parser import get_json_data
from utils.runtime_const import SPORTS_ARTIFACTS
class MrGamingStreamsSpider(scrapy.Spider):
@@ -38,7 +38,6 @@ class MrGamingStreamsSpider(scrapy.Spider):
self.redis = redis.Redis(
connection_pool=redis.ConnectionPool.from_url(settings.redis_url)
)
self.sports_artifacts = get_json_data("resources/json/sports_artifacts.json")
def __del__(self):
self.redis.close()
@@ -82,11 +81,9 @@ class MrGamingStreamsSpider(scrapy.Spider):
item = {
"genres": [category],
"poster": random.choice(self.sports_artifacts[category]["poster"]),
"background": random.choice(
self.sports_artifacts[category]["background"]
),
"logo": random.choice(self.sports_artifacts[category]["logo"]),
"poster": random.choice(SPORTS_ARTIFACTS[category]["poster"]),
"background": random.choice(SPORTS_ARTIFACTS[category]["background"]),
"logo": random.choice(SPORTS_ARTIFACTS[category]["logo"]),
"is_add_title_to_poster": True,
"event_start_timestamp": event_start_timestamp,
"title": event_name,
mediafusion_scrapy/spiders/streambtw.py
+3 -4
View File
@@ -5,7 +5,7 @@ import redis
import scrapy
from db.config import settings
from utils.parser import get_json_data
from utils.runtime_const import SPORTS_ARTIFACTS
class StreamBTWSpider(scrapy.Spider):
@@ -34,7 +34,6 @@ class StreamBTWSpider(scrapy.Spider):
self.redis = redis.Redis(
connection_pool=redis.ConnectionPool.from_url(settings.redis_url)
)
self.sports_artifacts = get_json_data("resources/json/sports_artifacts.json")
def __del__(self):
self.redis.close()
@@ -58,9 +57,9 @@ class StreamBTWSpider(scrapy.Spider):
item = {
"genres": [category],
"description": description,
"poster": random.choice(self.sports_artifacts[category]["poster"]),
"poster": random.choice(SPORTS_ARTIFACTS[category]["poster"]),
"background": random.choice(
self.sports_artifacts[category]["background"]
SPORTS_ARTIFACTS[category]["background"]
),
"logo": logo,
"is_add_title_to_poster": True,
mediafusion_scrapy/spiders/streamed.py
+4 -7
View File
@@ -3,7 +3,7 @@ import re
import scrapy
from utils.parser import get_json_data
from utils.runtime_const import SPORTS_ARTIFACTS
class StreamedSpider(scrapy.Spider):
@@ -46,7 +46,6 @@ class StreamedSpider(scrapy.Spider):
def __init__(self, *args, **kwargs):
super(StreamedSpider, self).__init__(*args, **kwargs)
self.sports_artifacts = get_json_data("resources/json/sports_artifacts.json")
def start_requests(self):
for category, url in self.categories.items():
@@ -63,11 +62,9 @@ class StreamedSpider(scrapy.Spider):
item = {
"stream_source": "Streamed (streamed.su)",
"genres": [category],
"poster": random.choice(self.sports_artifacts[category]["poster"]),
"background": random.choice(
self.sports_artifacts[category]["background"]
),
"logo": random.choice(self.sports_artifacts[category]["logo"]),
"poster": random.choice(SPORTS_ARTIFACTS[category]["poster"]),
"background": random.choice(SPORTS_ARTIFACTS[category]["background"]),
"logo": random.choice(SPORTS_ARTIFACTS[category]["logo"]),
"is_add_title_to_poster": True,
"title": event_name,
"url": response.urljoin(event_url),
mediafusion_scrapy/spiders/tgx.py
+210
View File
@@ -0,0 +1,210 @@
import random
import re
from datetime import datetime
import redis.asyncio as redis
import scrapy
from db.config import settings
from db.models import TorrentStreams
from utils.parser import convert_size_to_bytes
from utils.torrent import parse_magnet
class TgxSpider(scrapy.Spider):
allowed_domains = ["torrentgalaxy.to", "tgx.rs", "torrentgalaxy.mx", "tgx.sb"]
uploader_profiles: list[str]
catalog: list[str]
background_image: str
logo_image: str
keyword_patterns = re.compile(r"formula[ .+]*[1234e]+", re.IGNORECASE)
scraped_info_hash_key = "formula_tgx_scraped_info_hash"
custom_settings = {
"ITEM_PIPELINES": {
"mediafusion_scrapy.pipelines.TorrentDuplicatesPipeline": 100,
"mediafusion_scrapy.pipelines.FormulaParserPipeline": 200,
"mediafusion_scrapy.pipelines.EventSeriesStorePipeline": 300,
}
}
def __init__(self, scrape_all: str = "True", *args, **kwargs):
super(TgxSpider, self).__init__(*args, **kwargs)
self.scrape_all = scrape_all.lower() == "true"
self.redis = redis.Redis(
connection_pool=redis.ConnectionPool.from_url(settings.redis_url)
)
async def __aexit__(self, exc_type, exc_val, exc_tb):
await self.redis.aclose()
def start_requests(self):
for uploader_profile in self.uploader_profiles:
yield scrapy.Request(
f"https://{random.choice(self.allowed_domains)}/profile/{uploader_profile}/torrents/0",
self.parse,
meta={"uploader_profile": uploader_profile},
)
async def parse(self, response, **kwargs):
uploader_profile_name = response.meta["uploader_profile"]
self.logger.info(f"Scraping torrents from {uploader_profile_name}")
# Extract the last page number only once at the beginning
if self.scrape_all and response.url.endswith("/0"):
last_page_number = response.css(
"ul.pagination li.page-item:not(.disabled) a::attr(href)"
).re(r"/profile/.*/torrents/(\d+)")[-2]
last_page_number = (
int(last_page_number) if last_page_number.isdigit() else 0
)
# Generate requests for all pages
for page_number in range(1, last_page_number + 1):
next_page_url = (
f"{response.url.split('/torrents/')[0]}/torrents/{page_number}"
)
yield response.follow(next_page_url, self.parse, meta=response.meta)
# Extract torrents from the page
for torrent in response.css("div.tgxtablerow.txlight"):
urls = torrent.css("div.tgxtablecell a::attr(href)").getall()
torrent_name = torrent.css(
"div.tgxtablecell.clickable-row.click.textshadow.rounded.txlight a b::text"
).get()
if not self.keyword_patterns.search(torrent_name):
continue
tgx_unique_id = urls[0].split("/")[-2]
torrent_page_link = response.urljoin(urls[0])
torrent_link = torrent.css(
'a[href*="watercache.nanobytes.org"]::attr(href)'
).get()
magnet_link = torrent.css('a[href^="magnet:?"]::attr(href)').get()
info_hash, announce_list = parse_magnet(magnet_link)
if not info_hash:
self.logger.warning(
f"Failed to parse magnet link: {response.url}, {torrent_name}"
)
continue
seeders = torrent.css(
"div.tgxtablecell span[title='Seeders/Leechers'] font[color='green'] b::text"
).get()
seeders = int(seeders) if seeders and seeders.isdigit() else None
torrent_data = {
"info_hash": info_hash,
"torrent_name": torrent_name,
"torrent_link": torrent_link,
"magnet_link": magnet_link,
"background": self.background_image,
"logo": self.logo_image,
"seeders": seeders,
"torrent_page_link": torrent_page_link,
"unique_id": tgx_unique_id,
"source": f"TorrentGalaxy ({uploader_profile_name})",
"uploader": uploader_profile_name,
"announce_list": announce_list,
"catalog": self.catalog,
"scraped_info_hash_key": self.scraped_info_hash_key,
}
if await self.redis.sismember(self.scraped_info_hash_key, info_hash):
self.logger.info(f"Torrent already scraped: {torrent_name}")
await TorrentStreams.find_one({"_id": info_hash}).update(
{"$set": {"seeders": seeders}},
)
else:
yield response.follow(
torrent_page_link,
self.parse_torrent_details,
meta={"torrent_data": torrent_data},
)
def parse_torrent_details(self, response):
torrent_data = response.meta["torrent_data"]
# Extracting file details and sizes
file_details = []
for row in response.xpath('//table[contains(@class, "table-striped")]/tr'):
file_name = row.xpath('td[@class="table_col1"]/text()').get()
file_size = row.xpath('td[@class="table_col2"]/text()').get()
if file_name and file_size:
file_details.append({"file_name": file_name, "file_size": file_size})
if not file_details:
self.logger.warning(
f"File details not found for {torrent_data['torrent_name']}. Retrying"
)
yield response.follow(
response.url.replace(
response.url.split("/")[2], random.choice(self.allowed_domains)
),
self.parse_torrent_details,
meta={"torrent_data": torrent_data},
)
return
torrent_data["file_details"] = file_details
cover_image_url = response.xpath(
"//img[contains(@class, 'img-responsive') and contains(@data-src, '.png')]/@data-src"
).get()
torrent_data["poster"] = cover_image_url
# Getting the description for parsing video, audio, and other details
torrent_description = "".join(
response.xpath(
"//font/following-sibling::*[1]/following-sibling::text() | "
"//font/following-sibling::*[1]/following-sibling::*//text() | "
"//center/font/following::br/following-sibling::text() | "
"//strong/following-sibling::text()[normalize-space()] | "
"//strong/following-sibling::br/following-sibling::text()[normalize-space()] | "
"//div[contains(@class, 'container-fluid')]//text()[normalize-space()]"
).extract()
)
torrent_data["description"] = torrent_description.replace("\xa0", " ")
total_size = response.xpath(
"//div[b='Total Size:']/following-sibling::div/text()"
).get()
if total_size:
torrent_data["total_size"] = convert_size_to_bytes(total_size)
else:
# if the total size is not found, then tgx has shown captcha validation.
# so we need to slow down and retry the request
self.logger.warning(
f"Total size not found for {torrent_data['torrent_name']}. Retrying"
)
# change the hostname randomly to avoid captcha
yield response.follow(
response.url.replace(
response.url.split("/")[2], random.choice(self.allowed_domains)
),
self.parse_torrent_details,
meta={"torrent_data": torrent_data},
)
return
# Extracting date created
date_created = response.xpath(
"//div[b[contains(., 'Added:')]]/following-sibling::div/text()"
).get()
if date_created:
# Processing to extract the date and time
torrent_data["created_at"] = datetime.strptime(
date_created.strip(), "%d-%m-%Y %H:%M"
)
# Extracting language
language = response.xpath(
"//div[b='Language:']/following-sibling::div/text()"
).get()
if language:
torrent_data["languages"] = [language.strip()]
yield torrent_data
resources/json/sports_artifacts.json
+27 -13
View File
@@ -15,19 +15,19 @@
"Basketball": {
"background": [
"https://images.designtrends.com/wp-content/uploads/2015/12/21135619/Basketball-Background.jpg",
"http://wallpapercave.com/wp/acIknTe.jpg"
"https://external-content.duckduckgo.com/iu/?u=http%3A//wallpapercave.com/wp/acIknTe.jpg&f=1&nofb=1"
],
"poster": [
"https://www.pixelstalk.net/wp-content/uploads/2016/05/Free-Desktop-basketball-wallpapers-court.jpg",
"https://www.pixelstalk.net/wp-content/uploads/2016/10/Basketball-Court-HD-Wallpaper.jpg"
],
"logo": [
"http://cliparts.co/cliparts/Aib/Kr4/AibKr4gGT.png"
"https://external-content.duckduckgo.com/iu/?u=http%3A//cliparts.co/cliparts/Aib/Kr4/AibKr4gGT.png&f=1&nofb=1"
]
},
"Baseball": {
"background": [
"http://www.pixelstalk.net/wp-content/uploads/2016/03/Nice-baseball-fields-wallpaper-background.jpg",
"https://external-content.duckduckgo.com/iu/?u=http%3A//www.pixelstalk.net/wp-content/uploads/2016/03/Nice-baseball-fields-wallpaper-background.jpg&f=1&nofb=1",
"https://www.pixelstalk.net/wp-content/uploads/2016/03/Download-free-baseball-wallpaper-HD.jpg"
],
"poster": [
@@ -53,7 +53,7 @@
},
"Football": {
"background": [
"http://wallpapercave.com/wp/Mx8QOkb.jpg",
"https://external-content.duckduckgo.com/iu/?u=http%3A//wallpapercave.com/wp/Mx8QOkb.jpg&f=1&nofb=1",
"https://wallpapercave.com/wp/wp2694966.jpg",
"https://wallpapercave.com/wp/wp8255684.jpg",
"https://wallpapercave.com/wp/wp2167295.jpg"
@@ -108,11 +108,11 @@
"Tennis": {
"background": [
"https://wallpapercave.com/wp/wp3088699.jpg",
"http://yesofcorsa.com/wp-content/uploads/2015/08/3307_tennis.jpg",
"http://3.bp.blogspot.com/-galOVHt3kp0/UZvDZpqBBiI/AAAAAAAALD0/c0zQxrH0LxY/s1600/Tennis+Wallpapers+4.jpg"
"https://external-content.duckduckgo.com/iu/?u=http%3A//yesofcorsa.com/wp-content/uploads/2015/08/3307_tennis.jpg&f=1&nofb=1",
"https://external-content.duckduckgo.com/iu/?u=http%3A//3.bp.blogspot.com/-galOVHt3kp0/UZvDZpqBBiI/AAAAAAAALD0/c0zQxrH0LxY/s1600/Tennis%2BWallpapers%2B4.jpg&f=1&nofb=1"
],
"poster": [
"http://3.bp.blogspot.com/-galOVHt3kp0/UZvDZpqBBiI/AAAAAAAALD0/c0zQxrH0LxY/s1600/Tennis+Wallpapers+4.jpg",
"https://external-content.duckduckgo.com/iu/?u=http%3A//3.bp.blogspot.com/-galOVHt3kp0/UZvDZpqBBiI/AAAAAAAALD0/c0zQxrH0LxY/s1600/Tennis%2BWallpapers%2B4.jpg&f=1&nofb=1",
"https://images.vexels.com/media/users/3/76768/raw/b6882e62c248e8d5ff5f1fd9259d9f73-tennis-racket-background.jpg"
],
"logo": [
@@ -136,11 +136,11 @@
"background": [
"https://wallpaperaccess.com/full/1543778.jpg",
"https://free4kwallpapers.com/uploads/originals/2018/02/06/stunning-golf-course-wallpaper.jpg",
"http://www.pixelstalk.net/wp-content/uploads/2016/04/Free-golf-backgrounds-photos-pics.jpg"
"https://external-content.duckduckgo.com/iu/?u=http%3A//www.pixelstalk.net/wp-content/uploads/2016/04/Free-golf-backgrounds-photos-pics.jpg&f=1&nofb=1"
],
"poster": [
"https://images.all-free-download.com/images/graphiclarge/golf_picture_9_168173.jpg",
"http://cdn.wallpapersafari.com/97/40/W4gJLI.jpg",
"https://external-content.duckduckgo.com/iu/?u=http%3A//cdn.wallpapersafari.com/97/40/W4gJLI.jpg&f=1&nofb=1",
"https://www.pixelstalk.net/wp-content/uploads/2016/04/Golf-wallpapers-HD-Photos.jpg"
],
"logo": [
@@ -153,7 +153,7 @@
"https://www.baltana.com/files/wallpapers-21/Darts-Arrow-HD-Background-Wallpaper-52811.jpg"
],
"poster": [
"http://4.bp.blogspot.com/-HbSvu5nU_KQ/URFbg6VXfFI/AAAAAAAASg0/92zXW__vnqA/s1600/Darts+Wallpapers+01.jpg",
"https://external-content.duckduckgo.com/iu/?u=http%3A//4.bp.blogspot.com/-HbSvu5nU_KQ/URFbg6VXfFI/AAAAAAAASg0/92zXW__vnqA/s1600/Darts%2BWallpapers%2B01.jpg&f=1&nofb=1",
"https://wallpapershome.com/images/wallpapers/darts-1920x1080-4k-5k-wallpaper-hd-wheel-target-fire-water-561.jpg"
],
"logo": [
@@ -166,7 +166,7 @@
"https://wallpapercave.com/wp/wp1998982.jpg"
],
"poster": [
"http://rickrifici.com/wp-content/uploads/2011/03/AFL.jpg",
"https://external-content.duckduckgo.com/iu/?u=http%3A//rickrifici.com/wp-content/uploads/2011/03/AFL.jpg&f=1&nofb=1",
"https://i.pinimg.com/736x/c0/54/73/c0547342c769706d2ed606d2b6f7246b.jpg"
],
"logo": [
@@ -187,6 +187,20 @@
"https://static.vecteezy.com/system/resources/thumbnails/000/130/098/small/turbocharger-racing-logo-template.jpg"
]
},
"MotoGP": {
"background": [
"https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2Fwallpaperbat.com%2Fimg%2F569936-marc-marquez-wallpaper-in-2020-yamaha-motogp-racing-motogp-race.jpg&f=1&nofb=1&ipt=a0e19f2afd8aa9cc2aaeecf47502ed36fa0d2c6565eaf01d437505dcc0e8923f&ipo=images",
"https://external-content.duckduckgo.com/iu/?u=http%3A%2F%2Fgetwallpapers.com%2Fwallpaper%2Ffull%2F9%2F1%2F1%2F479678.jpg&f=1&nofb=1&ipt=074cb784018cb9d5ebf40c6a97511dcdf9b18e87b2a27e3fe43296ec12437f9c&ipo=images"
],
"poster": [
"https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2Fi.pinimg.com%2Foriginals%2F62%2F0e%2F24%2F620e24ceb7964d0168606932e4a976cf.jpg&f=1&nofb=1&ipt=1965087b235796e0dbf101d6329c1f7f14b5dfc5acfb48c075739108c6eddba9&ipo=images",
"https://images.wallpapersden.com/image/wxl-motogp-19-game_66061.jpg"
],
"logo": [
"https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2F1.bp.blogspot.com%2F-aLMSit3Q6XY%2FUQJO5Eiac4I%2FAAAAAAAAFcE%2Fueo2j7cdk-c%2Fs320%2FLogo%2BMotogp.jpg&f=1&nofb=1&ipt=8e81f529ea52045afffc5c64a7bbd8cde17f35a4c5f8eca685a0488f9605bfcf&ipo=images",
"https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2Fbesthqwallpapers.com%2FUploads%2F25-12-2017%2F34975%2Fthumb-motogp-4k-logo-grunge-black-background.jpg&f=1&nofb=1&ipt=e934a488b5c925ba56ad11f3b2cb728e85b4e34ac217441c0bf6c4cb4a07db9d&ipo=images"
]
},
"Other Sports": {
"background": [
"https://wallpapercave.com/wp/Cd7iyFl.jpg"
@@ -195,7 +209,7 @@
"https://png.pngtree.com/thumb_back/fh260/background/20210312/pngtree-green-gradient-sports-background-image_582399.jpg"
],
"logo": [
"http://images.gofreedownload.net/sport-105683.jpg"
"https://external-content.duckduckgo.com/iu/?u=http%3A//images.gofreedownload.net/sport-105683.jpg&f=1&nofb=1"
]
},
"MLB": {
@@ -205,7 +219,7 @@
],
"poster": [
"https://i.pinimg.com/736x/c0/54/73/c0547342c769706d2ed606d2b6f7246b.jpg",
"http://rickrifici.com/wp-content/uploads/2011/03/AFL.jpg"
"https://external-content.duckduckgo.com/iu/?u=http%3A//rickrifici.com/wp-content/uploads/2011/03/AFL.jpg&f=1&nofb=1"
],
"logo": [
"https://botw-pd.s3.amazonaws.com/styles/logo-thumbnail/s3/0003/4002/brand.gif?itok=F2rATKp8"
resources/manifest.json
+11
View File
@@ -415,6 +415,17 @@
}
]
},
{
"id": "motogp_racing",
"type": "series",
"name": "MotoGP Racing",
"extra": [
{
"name": "skip",
"isRequired": false
}
]
},
{
"id": "american_football",
"type": "movie",
scrapers/helpers.py
+1 -1
View File
@@ -11,7 +11,7 @@ from urllib3.util.retry import Retry
from db.config import settings
from db.models import TorrentStreams, Episode, Season
from utils.const import UA_HEADER
from utils.parser import get_json_data
from utils import get_json_data
from utils.torrent import extract_torrent_metadata, info_hashes_to_torrent_metadata
# set httpx logging level
utils/__init__.py
+6
View File
@@ -0,0 +1,6 @@
import json
def get_json_data(file_name: str) -> dict:
with open(file_name) as file:
return json.load(file)
utils/const.py
+2
View File
@@ -28,6 +28,7 @@ CATALOG_ID_DATA = [
"mediafusion_search_movies",
"mediafusion_search_series",
"mediafusion_search_tv",
"motogp_racing",
"other_sports",
"prowlarr_streams",
"rugby",
@@ -74,6 +75,7 @@ CATALOG_NAME_DATA = [
"MediaFusion Search Movies",
"MediaFusion Search Series",
"MediaFusion Search TV",
"MotoGP Racing",
"Other Sports",
"Prowlarr Streams",
"Rugby/AFL",
utils/parser.py
-8
View File
@@ -1,10 +1,7 @@
import asyncio
import json
import math
import re
import requests
from imdb import Cinemagoer
from redis.asyncio import Redis
from db.config import settings
@@ -355,8 +352,3 @@ def is_contain_18_plus_keywords(title: str) -> bool:
Check if the title contains 18+ keywords to filter out adult content.
"""
return ADULT_CONTENT_KEYWORDS.search(title) is not None
def get_json_data(file_name: str) -> dict:
with open(file_name) as file:
return json.load(file)
utils/runtime_const.py
+3 -1
View File
@@ -1,7 +1,7 @@
import re
from db.config import settings
from utils import get_json_data
ADULT_CONTENT_KEYWORDS = re.compile(
settings.adult_content_regex_keywords,
@@ -15,3 +15,5 @@ PARENT_GUIDE_CERTIFICATES_FILTER_REGEX = re.compile(
settings.parent_guide_certificates_filter_regex,
re.IGNORECASE,
)
SPORTS_ARTIFACTS = get_json_data("resources/json/sports_artifacts.json")