Add MrGamingStreams scrapy for sports events

api/scheduler.py

@@ -86,3 +86,10 @@ def setup_scheduler(scheduler: AsyncIOScheduler):
     scheduler.add_job(
         crud.delete_search_history, CronTrigger(day="*/1"), name="delete_search_history"
     )
+
+    scheduler.add_job(
+        run_spider.send,
+        CronTrigger.from_crontab(settings.mrgamingstreams_scheduler_crontab),
+        name="mrgamingstreams",
+        kwargs={"spider_name": "mrgamingstreams"},
+    )

db/config.py

@@ -37,6 +37,7 @@ class Settings(BaseSettings):
     streamed_scheduler_crontab: str = "*/15 * * * *"
     meta_cache_ttl: int = 1800  # 30 minutes
     validate_m3u8_urls_liveness: bool = True
+    mrgamingstreams_scheduler_crontab: str = "*/15 * * * *"
 
     class Config:
         env_file = ".env"

db/schemas.py

@@ -214,6 +214,7 @@ class ScraperTask(BaseModel):
         "tamilultra",
         "sport_video",
         "streamed",
+        "mrgamingstreams",
     ]
     api_password: str = None
 

docs/configuration.md

@@ -53,6 +53,7 @@ This guide describes the environment variables available in MediaFusion for conf
 - **mhdtvworld_scheduler_crontab** (default: `"0 0 * * 5"`): Scheduler for MHDTVWorld.
 - **mhdtvsports_scheduler_crontab** (default: `"0 10 * * *"`): Scheduler for MHDTVSports.
 - **streamed_scheduler_crontab** (default: `"*/15 * * * *"`): Scheduler for Streamed.su Sports Events.
+- **mrgamingstreams_scheduler_crontab** (default: `"*/15 * * * *"`): Scheduler for MrGamingStreams Sports Events.
 
 ### How to Configure
 

mediafusion_scrapy/spiders/mrgamingstreams.py

@@ -0,0 +1,165 @@
+import random
+import re
+from datetime import datetime
+
+import pytz
+import redis
+import scrapy
+
+from db.config import settings
+from utils import const
+from utils.parser import get_json_data
+
+
+class MrGamingStreamsSpider(scrapy.Spider):
+    name = "mrgamingstreams"
+    allowed_domains = ["mrgamingstreams.com"]
+    categories = {
+        "Baseball": "https://mrgamingstreams.com/mlb",
+        "American Football": "https://mrgamingstreams.com/nfl",
+        "Basketball": "https://mrgamingstreams.com/nba",
+        "Hockey": "https://mrgamingstreams.com/nhl",
+        "Football": "https://mrgamingstreams.com/soccer",
+        "Fighting": "https://mrgamingstreams.com/fighting",
+        "Motor Sport": "https://mrgamingstreams.com/motorsports",
+    }
+
+    et_tz = pytz.timezone("America/New_York")  # Eastern Time zone
+
+    custom_settings = {
+        "ITEM_PIPELINES": {
+            "mediafusion_scrapy.pipelines.LiveEventStorePipeline": 300,
+        },
+        "LOG_LEVEL": "DEBUG",
+        "DUPEFILTER_DEBUG": True,
+    }
+
+    def __init__(self, *args, **kwargs):
+        super(MrGamingStreamsSpider, self).__init__(*args, **kwargs)
+        self.redis = redis.Redis(
+            connection_pool=redis.ConnectionPool.from_url(settings.redis_url)
+        )
+        self.sports_artifacts = get_json_data("resources/json/sports_artifacts.json")
+
+    def __del__(self):
+        self.redis.close()
+
+    def start_requests(self):
+        for category, url in self.categories.items():
+            yield scrapy.Request(url, self.parse, meta={"category": category})
+
+    def parse(self, response, **kwargs):
+        category = response.meta["category"]
+        rows = response.xpath('//table[@id="thealmightytable"]/tr')
+
+        for row in rows:
+            event_url = row.xpath("./td[3]/a/@href").get()
+            if not event_url:
+                continue
+
+            time_str = row.xpath("./td[1]/text()").get().strip()
+            event_name = row.xpath("./td[2]/text()").get().strip()
+
+            # Combine date with time and parse it into a datetime object
+            now = datetime.now(self.et_tz)
+            event_time = datetime.strptime(time_str, "%I:%M %p").replace(
+                year=now.year, month=now.month, day=now.day
+            )
+            event_time = self.et_tz.localize(event_time)
+            event_start_timestamp = event_time.timestamp()
+
+            item = {
+                "genres": [category],
+                "poster": random.choice(self.sports_artifacts[category]["poster"]),
+                "background": random.choice(
+                    self.sports_artifacts[category]["background"]
+                ),
+                "logo": random.choice(self.sports_artifacts[category]["logo"]),
+                "is_add_title_to_poster": True,
+                "event_start_timestamp": event_start_timestamp,
+                "title": event_name,
+                "url": response.urljoin(event_url),
+            }
+
+            yield response.follow(
+                event_url, self.parse_event, meta={"item": item}, dont_filter=True
+            )
+
+    def parse_event(self, response):
+        item = response.meta["item"].copy()
+        item.update(
+            {
+                "event_url": response.url,
+                "streams": [],
+            }
+        )
+
+        stream_buttons = response.xpath(
+            "//div[contains(@class, 'streambuttoncase')]/button"
+        )
+
+        if not stream_buttons:
+            self.logger.info("No streams available for this event yet.")
+            return
+
+        for button in stream_buttons:
+            stream_data = button.xpath("./@onclick").get()
+            if not stream_data:
+                continue
+
+            # Extract the M3U8 URL from the onclick attribute using a regular expression
+            m3u8_url_match = re.search(
+                r"showPlayer\('\w+', '(https?://[^']+)'\)", stream_data
+            )
+            if not m3u8_url_match:
+                continue
+
+            m3u8_url = m3u8_url_match.group(1)
+            stream_name = button.xpath("text()").get().strip()
+
+            yield scrapy.Request(
+                url=m3u8_url,
+                callback=self.validate_m3u8_url,
+                meta={
+                    "item": item,
+                    "stream_name": stream_name,
+                    "stream_url": m3u8_url,
+                },
+                dont_filter=True,
+            )
+
+    def validate_m3u8_url(self, response):
+        meta = response.meta
+        item = meta["item"]
+        content_type = response.headers.get("Content-Type", b"").decode().lower()
+
+        if response.status == 200 and content_type in const.M3U8_VALID_CONTENT_TYPES:
+            # Stream is valid; add it to the item's streams list
+            item["streams"].append(
+                {
+                    "name": meta["stream_name"],
+                    "url": meta["stream_url"],
+                    "source": "MrGamingStreams",
+                    "behaviorHints": {
+                        "notWebReady": True,
+                        "is_redirect": True
+                        if response.meta.get("redirect_times", 0) > 0
+                        else False,
+                        "proxyHeaders": {
+                            "request": {
+                                "User-Agent": response.request.headers.get(
+                                    "User-Agent"
+                                ).decode(),
+                                "Referer": response.request.headers.get(
+                                    "Referer"
+                                ).decode(),
+                            }
+                        },
+                    },
+                }
+            )
+            return item
+        else:
+            self.logger.error(
+                f"Invalid M3U8 URL: {meta['stream_url']} with Content-Type: {content_type} response: {response.status}"
+            )

resources/html/home.html

@@ -65,6 +65,9 @@
                 <a id="installLink" class="install-link btn btn-block mx-auto" href="/configure">
                     ⚙️ Configure Add-on
                 </a>
+                <a id="scraperLink" class="btn btn-block mx-auto" href="/scraper">
+                    🕸️ Scraper Controls
+                </a>
             </div>
         </div>
     </div>

resources/html/scraper.html

@@ -59,6 +59,7 @@
                                 <option value="tamilultra">Tamil Ultra</option>
                                 <option value="sport_video">Sport Video</option>
                                 <option value="streamed">Streamed Sport Events</option>
+                                <option value="mrgamingstreams">MrGamingStreams Sport Events</option>
                             </select>
                         </div>
                     </div>

resources/json/sports_artifacts.json

@@ -31,7 +31,6 @@
 	  "https://www.pixelstalk.net/wp-content/uploads/2016/03/Download-free-baseball-wallpaper-HD.jpg"
 	],
 	"poster": [
-	  "https://static.vecteezy.com/system/resources/previews/000/511/101/non_2x/baseball-texture-sport-vector-background.jpg",
 	  "https://mcdn.wallpapersafari.com/374/21/33/UR7xC4.jpg"
 	],
 	"logo": [
@@ -185,5 +184,18 @@
 	"logo": [
 	  "http://images.gofreedownload.net/sport-105683.jpg"
 	]
+  },
+  "MLB": {
+	"background": [
+	  "https://wallpapercave.com/wp/wp1998982.jpg",
+	  "https://wallpapercave.com/wp/wp1998978.jpg"
+	],
+	"poster": [
+	  "https://i.pinimg.com/736x/c0/54/73/c0547342c769706d2ed606d2b6f7246b.jpg",
+	  "http://rickrifici.com/wp-content/uploads/2011/03/AFL.jpg"
+	],
+	"logo": [
+	  "https://botw-pd.s3.amazonaws.com/styles/logo-thumbnail/s3/0003/4002/brand.gif?itok=F2rATKp8"
+	]
   }
 }