From 6baab1dc2a6909d1b22425c37276810fd80c59f2 Mon Sep 17 00:00:00 2001 From: mhdzumair Date: Mon, 1 Apr 2024 16:45:07 +0530 Subject: [PATCH] Add MrGamingStreams scrapy for sports events --- api/scheduler.py | 7 + db/config.py | 1 + db/schemas.py | 1 + docs/configuration.md | 1 + mediafusion_scrapy/spiders/mrgamingstreams.py | 165 ++++++++++++++++++ resources/html/home.html | 3 + resources/html/scraper.html | 1 + resources/json/sports_artifacts.json | 14 +- 8 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 mediafusion_scrapy/spiders/mrgamingstreams.py diff --git a/api/scheduler.py b/api/scheduler.py index 05a691b..690c948 100644 --- a/api/scheduler.py +++ b/api/scheduler.py @@ -86,3 +86,10 @@ def setup_scheduler(scheduler: AsyncIOScheduler): scheduler.add_job( crud.delete_search_history, CronTrigger(day="*/1"), name="delete_search_history" ) + + scheduler.add_job( + run_spider.send, + CronTrigger.from_crontab(settings.mrgamingstreams_scheduler_crontab), + name="mrgamingstreams", + kwargs={"spider_name": "mrgamingstreams"}, + ) diff --git a/db/config.py b/db/config.py index 2530f67..9f950cf 100644 --- a/db/config.py +++ b/db/config.py @@ -37,6 +37,7 @@ class Settings(BaseSettings): streamed_scheduler_crontab: str = "*/15 * * * *" meta_cache_ttl: int = 1800 # 30 minutes validate_m3u8_urls_liveness: bool = True + mrgamingstreams_scheduler_crontab: str = "*/15 * * * *" class Config: env_file = ".env" diff --git a/db/schemas.py b/db/schemas.py index bd22b50..f661225 100644 --- a/db/schemas.py +++ b/db/schemas.py @@ -214,6 +214,7 @@ class ScraperTask(BaseModel): "tamilultra", "sport_video", "streamed", + "mrgamingstreams", ] api_password: str = None diff --git a/docs/configuration.md b/docs/configuration.md index 1470be1..c236978 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -53,6 +53,7 @@ This guide describes the environment variables available in MediaFusion for conf - **mhdtvworld_scheduler_crontab** (default: `"0 0 * * 5"`): Scheduler for MHDTVWorld. - **mhdtvsports_scheduler_crontab** (default: `"0 10 * * *"`): Scheduler for MHDTVSports. - **streamed_scheduler_crontab** (default: `"*/15 * * * *"`): Scheduler for Streamed.su Sports Events. +- **mrgamingstreams_scheduler_crontab** (default: `"*/15 * * * *"`): Scheduler for MrGamingStreams Sports Events. ### How to Configure diff --git a/mediafusion_scrapy/spiders/mrgamingstreams.py b/mediafusion_scrapy/spiders/mrgamingstreams.py new file mode 100644 index 0000000..60042e0 --- /dev/null +++ b/mediafusion_scrapy/spiders/mrgamingstreams.py @@ -0,0 +1,165 @@ +import random +import re +from datetime import datetime + +import pytz +import redis +import scrapy + +from db.config import settings +from utils import const +from utils.parser import get_json_data + + +class MrGamingStreamsSpider(scrapy.Spider): + name = "mrgamingstreams" + allowed_domains = ["mrgamingstreams.com"] + categories = { + "Baseball": "https://mrgamingstreams.com/mlb", + "American Football": "https://mrgamingstreams.com/nfl", + "Basketball": "https://mrgamingstreams.com/nba", + "Hockey": "https://mrgamingstreams.com/nhl", + "Football": "https://mrgamingstreams.com/soccer", + "Fighting": "https://mrgamingstreams.com/fighting", + "Motor Sport": "https://mrgamingstreams.com/motorsports", + } + + et_tz = pytz.timezone("America/New_York") # Eastern Time zone + + custom_settings = { + "ITEM_PIPELINES": { + "mediafusion_scrapy.pipelines.LiveEventStorePipeline": 300, + }, + "LOG_LEVEL": "DEBUG", + "DUPEFILTER_DEBUG": True, + } + + def __init__(self, *args, **kwargs): + super(MrGamingStreamsSpider, self).__init__(*args, **kwargs) + self.redis = redis.Redis( + connection_pool=redis.ConnectionPool.from_url(settings.redis_url) + ) + self.sports_artifacts = get_json_data("resources/json/sports_artifacts.json") + + def __del__(self): + self.redis.close() + + def start_requests(self): + for category, url in self.categories.items(): + yield scrapy.Request(url, self.parse, meta={"category": category}) + + def parse(self, response, **kwargs): + category = response.meta["category"] + rows = response.xpath('//table[@id="thealmightytable"]/tr') + + for row in rows: + event_url = row.xpath("./td[3]/a/@href").get() + if not event_url: + continue + + time_str = row.xpath("./td[1]/text()").get().strip() + event_name = row.xpath("./td[2]/text()").get().strip() + + # Combine date with time and parse it into a datetime object + now = datetime.now(self.et_tz) + event_time = datetime.strptime(time_str, "%I:%M %p").replace( + year=now.year, month=now.month, day=now.day + ) + event_time = self.et_tz.localize(event_time) + event_start_timestamp = event_time.timestamp() + + item = { + "genres": [category], + "poster": random.choice(self.sports_artifacts[category]["poster"]), + "background": random.choice( + self.sports_artifacts[category]["background"] + ), + "logo": random.choice(self.sports_artifacts[category]["logo"]), + "is_add_title_to_poster": True, + "event_start_timestamp": event_start_timestamp, + "title": event_name, + "url": response.urljoin(event_url), + } + + yield response.follow( + event_url, self.parse_event, meta={"item": item}, dont_filter=True + ) + + def parse_event(self, response): + item = response.meta["item"].copy() + item.update( + { + "event_url": response.url, + "streams": [], + } + ) + + stream_buttons = response.xpath( + "//div[contains(@class, 'streambuttoncase')]/button" + ) + + if not stream_buttons: + self.logger.info("No streams available for this event yet.") + return + + for button in stream_buttons: + stream_data = button.xpath("./@onclick").get() + if not stream_data: + continue + + # Extract the M3U8 URL from the onclick attribute using a regular expression + m3u8_url_match = re.search( + r"showPlayer\('\w+', '(https?://[^']+)'\)", stream_data + ) + if not m3u8_url_match: + continue + + m3u8_url = m3u8_url_match.group(1) + stream_name = button.xpath("text()").get().strip() + + yield scrapy.Request( + url=m3u8_url, + callback=self.validate_m3u8_url, + meta={ + "item": item, + "stream_name": stream_name, + "stream_url": m3u8_url, + }, + dont_filter=True, + ) + + def validate_m3u8_url(self, response): + meta = response.meta + item = meta["item"] + content_type = response.headers.get("Content-Type", b"").decode().lower() + + if response.status == 200 and content_type in const.M3U8_VALID_CONTENT_TYPES: + # Stream is valid; add it to the item's streams list + item["streams"].append( + { + "name": meta["stream_name"], + "url": meta["stream_url"], + "source": "MrGamingStreams", + "behaviorHints": { + "notWebReady": True, + "is_redirect": True + if response.meta.get("redirect_times", 0) > 0 + else False, + "proxyHeaders": { + "request": { + "User-Agent": response.request.headers.get( + "User-Agent" + ).decode(), + "Referer": response.request.headers.get( + "Referer" + ).decode(), + } + }, + }, + } + ) + return item + else: + self.logger.error( + f"Invalid M3U8 URL: {meta['stream_url']} with Content-Type: {content_type} response: {response.status}" + ) diff --git a/resources/html/home.html b/resources/html/home.html index 7ce3719..1e76448 100644 --- a/resources/html/home.html +++ b/resources/html/home.html @@ -65,6 +65,9 @@ ⚙️ Configure Add-on + + 🕸️ Scraper Controls + diff --git a/resources/html/scraper.html b/resources/html/scraper.html index fe97d0d..4d3fc01 100644 --- a/resources/html/scraper.html +++ b/resources/html/scraper.html @@ -59,6 +59,7 @@ + diff --git a/resources/json/sports_artifacts.json b/resources/json/sports_artifacts.json index 279e5ff..94557ae 100644 --- a/resources/json/sports_artifacts.json +++ b/resources/json/sports_artifacts.json @@ -31,7 +31,6 @@ "https://www.pixelstalk.net/wp-content/uploads/2016/03/Download-free-baseball-wallpaper-HD.jpg" ], "poster": [ - "https://static.vecteezy.com/system/resources/previews/000/511/101/non_2x/baseball-texture-sport-vector-background.jpg", "https://mcdn.wallpapersafari.com/374/21/33/UR7xC4.jpg" ], "logo": [ @@ -185,5 +184,18 @@ "logo": [ "http://images.gofreedownload.net/sport-105683.jpg" ] + }, + "MLB": { + "background": [ + "https://wallpapercave.com/wp/wp1998982.jpg", + "https://wallpapercave.com/wp/wp1998978.jpg" + ], + "poster": [ + "https://i.pinimg.com/736x/c0/54/73/c0547342c769706d2ed606d2b6f7246b.jpg", + "http://rickrifici.com/wp-content/uploads/2011/03/AFL.jpg" + ], + "logo": [ + "https://botw-pd.s3.amazonaws.com/styles/logo-thumbnail/s3/0003/4002/brand.gif?itok=F2rATKp8" + ] } } \ No newline at end of file