Remove obsolete spiders Streamed and StreamBTW

README.md

@@ -31,7 +31,7 @@
   - 🏎️ **Formula Racing**: Exclusive scraping from TorrentGalaxy for all your racing needs.
   - 🥊 **Fighting Sports**: Catch up on all the latest fighting sports content from UFC and WWE.
   - 🏈🏀⚾⚽🏒🏉🎾 **American Football, Basketball, Baseball, Football, Hockey, Rugby/AFL, and Other Sports**: Now all scraping through sport-video.org.ua for catchup videos.
-  - 🏈🏀⚾⚽🏒🏉🎾🏏 **Sports Live Events**: Watch live sports events from streamed.su, streambtw.com
+  - 🏈🏀⚾⚽🏒🏉🎾🏏 **Sports Live Events**: Watch live sports events from DaddyLiveHD
   - 🎥 **TamilMV**: Specialized scraping for regional contents.
   - 🌟 **TamilBlasters**: Dedicated access to an extensive library of regional content.
   - 📺 **TamilUltra & NowMeTV**: Get the best of Live TV channels right at your fingertips.

api/scheduler.py

@@ -109,30 +109,6 @@ def setup_scheduler(scheduler: AsyncIOScheduler):
             },
         )
 
-    # Schedule streamed scraper
-    if not settings.disable_streamed_scheduler:
-        scheduler.add_job(
-            run_spider.send,
-            CronTrigger.from_crontab(settings.streamed_scheduler_crontab),
-            name="streamed",
-            kwargs={
-                "spider_name": "streamed",
-                "crontab_expression": settings.streamed_scheduler_crontab,
-            },
-        )
-
-    # Schedule streambtw scraper
-    if not settings.disable_streambtw_scheduler:
-        scheduler.add_job(
-            run_spider.send,
-            CronTrigger.from_crontab(settings.streambtw_scheduler_crontab),
-            name="streambtw",
-            kwargs={
-                "spider_name": "streambtw",
-                "crontab_expression": settings.streambtw_scheduler_crontab,
-            },
-        )
-
     # Schedule dlhd scraper
     if not settings.disable_dlhd_scheduler:
         scheduler.add_job(

db/config.py

@@ -124,10 +124,6 @@ class Settings(BaseSettings):
     disable_validate_tv_streams_in_db: bool = False
     sport_video_scheduler_crontab: str = "*/20 * * * *"
     disable_sport_video_scheduler: bool = False
-    streamed_scheduler_crontab: str = "*/30 * * * *"
-    disable_streamed_scheduler: bool = False
-    streambtw_scheduler_crontab: str = "*/15 * * * *"
-    disable_streambtw_scheduler: bool = True
     dlhd_scheduler_crontab: str = "25 * * * *"
     disable_dlhd_scheduler: bool = False
     update_imdb_data_crontab: str = "0 2 * * *"

db/schemas.py

@@ -358,10 +358,8 @@ class ScraperTask(BaseModel):
         "nowsports",
         "tamilultra",
         "sport_video",
-        "streamed",
         "tamilmv",
         "tamil_blasters",
-        "streambtw",
         "dlhd",
         "motogp_tgx",
         "arab_torrents",

docs/configuration.md

@@ -122,10 +122,6 @@ Each scheduler has a crontab expression to define when it runs and a correspondi
 - **disable_validate_tv_streams_in_db** (default: `False`)
 - **sport_video_scheduler_crontab** (default: `"*/20 * * * *"`)
 - **disable_sport_video_scheduler** (default: `False`)
-- **streamed_scheduler_crontab** (default: `"*/30 * * * *"`)
-- **disable_streamed_scheduler** (default: `False`)
-- **streambtw_scheduler_crontab** (default: `"*/15 * * * *"`)
-- **disable_streambtw_scheduler** (default: `False`)
 - **dlhd_scheduler_crontab** (default: `"25 * * * *"`)
 - **disable_dlhd_scheduler** (default: `False`)
 - **update_imdb_data_crontab** (default: `"0 2 * * *"`)
@@ -160,12 +156,8 @@ Note: Crontab expressions follow the standard cron format: "minute hour day-of-m
 - **disable_nowmetv_scheduler** (default: `False`): Disable NowMeTV scheduler.
 - **nowsports_scheduler_crontab** (default: `"0 10 * * *"`): Scheduler for NowSports.
 - **disable_nowsports_scheduler** (default: `False`): Disable NowSports scheduler.
-- **streamed_scheduler_crontab** (default: `"*/15 * * * *"`): Scheduler for Streamed.su Sports Events.
-- **disable_streamed_scheduler** (default: `False`): Disable Streamed.su scheduler.
 - **sport_video_scheduler_crontab** (default: `"*/20 * * * *"`): Scheduler for Sport Video.
 - **disable_sport_video_scheduler** (default: `False`): Disable Sport Video scheduler.
-- **streambtw_scheduler_crontab** (default: `"*/15 * * * *"`): Scheduler for Streambtw.
-- **disable_streambtw_scheduler** (default: `False`): Disable Streambtw scheduler.
 - **dlhd_scheduler_crontab** (default: `"25 * * * *"`): Scheduler for DLHD.
 - **disable_dlhd_scheduler**: Disable DLHD scheduler.
 - **update_imdb_data_crontab** (default: `"0 2 * * *"`): Scheduler for updating IMDb data.

mediafusion_scrapy/spiders/streambtw.py

@@ -1,97 +0,0 @@
-import random
-import re
-
-import scrapy
-
-from utils.runtime_const import SPORTS_ARTIFACTS
-from db.redis_database import REDIS_SYNC_CLIENT
-
-
-class StreamBTWSpider(scrapy.Spider):
-    name = "streambtw"
-    start_urls = ["https://streambtw.com/"]
-    referer = "https://streambtw.com/"
-
-    custom_settings = {
-        "ITEM_PIPELINES": {
-            "mediafusion_scrapy.pipelines.LiveStreamResolverPipeline": 100,
-            "mediafusion_scrapy.pipelines.LiveEventStorePipeline": 300,
-        },
-        "LOG_LEVEL": "DEBUG",
-        "DUPEFILTER_DEBUG": True,
-    }
-    category_map = {
-        "Soccer": "Football",
-        "NFL": "American Football",
-        "NBA": "Basketball",
-        "NHL": "Hockey",
-        "MLB": "Baseball",
-        "24x7": "Other Sports",
-    }
-
-    def __init__(self, *args, **kwargs):
-        super(StreamBTWSpider, self).__init__(*args, **kwargs)
-        self.redis = REDIS_SYNC_CLIENT
-
-    def __del__(self):
-        self.redis.close()
-
-    def parse(self, response, **kwargs):
-        for timeline in response.xpath(
-            '//div[contains(@class,"single-timeline-area")]'
-        ):
-            category = timeline.xpath("./div/p/text()").get()
-            category = self.category_map.get(category, "Other Sports")
-
-            events = timeline.xpath(
-                './/a[div[contains(@class, "single-timeline-content")]]'
-            )
-            for event in events:
-                event_name = event.xpath(".//h6/text()").get()
-                description = event.xpath(".//p/text()").get()
-                event_url = event.xpath("@href").get()
-                logo = event.xpath(".//img/@src").get()
-
-                item = {
-                    "genres": [category],
-                    "description": description,
-                    "poster": random.choice(SPORTS_ARTIFACTS[category]["poster"]),
-                    "background": random.choice(
-                        SPORTS_ARTIFACTS[category]["background"]
-                    ),
-                    "logo": logo,
-                    "is_add_title_to_poster": True,
-                    "title": event_name,
-                    "url": event_url,
-                    "streams": [],
-                }
-
-                yield response.follow(
-                    event_url, self.parse_event, meta={"item": item}, dont_filter=True
-                )
-
-    def parse_event(self, response):
-        script_text = response.xpath(
-            "//script[contains(text(), 'm3u8List') or contains(text(), 'Clappr.Player')]/text()"
-        ).get()
-
-        if script_text:
-            m3u8_urls = re.findall(r'"(https?://[^"]+)"', script_text)
-            for m3u8_url in m3u8_urls:
-                item = response.meta["item"].copy()
-                item.update(
-                    {
-                        "stream_name": f"{item['title']} - Live Stream",
-                        "stream_url": m3u8_url,
-                        "stream_source": "StreamBTW",
-                        "stream_headers": {
-                            "Referer": self.referer,
-                            "Origin": self.referer.rstrip("/"),
-                        },
-                    }
-                )
-                yield item
-        else:
-            self.logger.error(
-                f"No suitable script found in event to extract M3U8 URLs. URL: {response.url}"
-            )

mediafusion_scrapy/spiders/streamed.py

@@ -1,180 +0,0 @@
-import json
-import random
-import re
-from datetime import datetime
-from urllib.parse import urljoin, urlparse
-
-import pytz
-import scrapy
-
-from utils.config import config_manager
-from utils.runtime_const import SPORTS_ARTIFACTS
-
-
-class StreamedSpider(scrapy.Spider):
-    name = "streamed"
-
-    api_base_url = config_manager.get_scraper_config(name, "api_base_url")
-    live_matches_url = f"{api_base_url}{config_manager.get_scraper_config(name, 'live_matches_endpoint')}"
-    stream_url_template = f"{api_base_url}{config_manager.get_scraper_config(name, 'stream_url_template')}"
-    image_url_template = (
-        f"{api_base_url}{config_manager.get_scraper_config(name, 'image_url_template')}"
-    )
-    m3u8_base_url = config_manager.get_scraper_config(name, "m3u8_base_url")
-    mediafusion_referer = config_manager.get_scraper_config(name, "mediafusion_referer")
-    category_mapping = config_manager.get_scraper_config(name, "category_mapping")
-    session_url = config_manager.get_scraper_config(name, "session_url")
-
-    domains = None
-    domain_host = None
-    gmt = pytz.timezone("Etc/GMT")
-
-    custom_settings = {
-        "ITEM_PIPELINES": {
-            "mediafusion_scrapy.pipelines.LiveStreamResolverPipeline": 100,
-            "mediafusion_scrapy.pipelines.LiveEventStorePipeline": 300,
-        },
-        "DUPEFILTER_DEBUG": True,
-        "DEFAULT_REQUEST_HEADERS": {"Referer": mediafusion_referer},
-    }
-
-    def start_requests(self):
-        yield scrapy.Request(self.live_matches_url, self.parse_live_matches)
-
-    def parse_live_matches(self, response):
-        matches = response.json()
-        for match in matches:
-            if not match.get("sources"):
-                self.logger.info(f"No sources available for match: {match['title']}")
-                continue
-
-            category = self.category_mapping.get(match["category"], "Other")
-
-            item = {
-                "stream_source": "Streamed (streamed.su)",
-                "genres": [category],
-                "title": match["title"],
-                "event_start_timestamp": match["date"] / 1000,  # Convert to seconds
-                "streams": [],
-            }
-
-            # Handle poster image
-            if "poster" in match and match["poster"]:
-                item["poster"] = urljoin(self.api_base_url, match["poster"])
-            elif (
-                match.get("teams")
-                and match["teams"].get("home", {}).get("badge")
-                and match["teams"].get("away", {}).get("badge")
-            ):
-                item["poster"] = self.image_url_template.format(
-                    batch_id_1=match["teams"]["home"]["badge"],
-                    batch_id_2=match["teams"]["away"]["badge"],
-                )
-            else:
-                item["poster"] = random.choice(SPORTS_ARTIFACTS[category]["poster"])
-
-            item["background"] = random.choice(SPORTS_ARTIFACTS[category]["background"])
-            item["logo"] = random.choice(SPORTS_ARTIFACTS[category]["logo"])
-            item["is_add_title_to_poster"] = True
-
-            for source in match["sources"]:
-                yield scrapy.Request(
-                    self.stream_url_template.format(
-                        source=source["source"], id=source["id"]
-                    ),
-                    self.parse_stream,
-                    meta={"item": item},
-                )
-
-    def parse_stream(self, response):
-        item = response.meta["item"].copy()
-        stream_data_list = response.json()
-
-        for stream_data in stream_data_list:
-            if self.domains and self.domain_host:
-                yield from self.parse_stream_data(stream_data, item)
-            else:
-                yield scrapy.Request(
-                    stream_data["embedUrl"],
-                    self.parse_embed,
-                    meta={"item": item, "stream_data": stream_data},
-                    headers={"Referer": self.mediafusion_referer},
-                )
-
-    def parse_embed(self, response):
-        item = response.meta["item"]
-        stream_data = response.meta["stream_data"]
-
-        if self.domains is None or self.domain_host is None:
-            self.extract_domain_info(response)
-
-        if self.domains and self.domain_host:
-            yield from self.parse_stream_data(stream_data, item)
-        else:
-            self.logger.error(
-                f"Failed to extract domain information for stream: {stream_data['id']}"
-            )
-
-    def extract_domain_info(self, response):
-        script_content = response.xpath(
-            '//script[contains(text(), "var k=")]/text()'
-        ).get()
-        if script_content:
-            vars_match = re.search(
-                r'var k="(\w+)",i="([^"]+)",s="(\d+)",l=(\[.+?]),h="([^"]+)";',
-                script_content,
-            )
-            if vars_match:
-                _, _, _, domains, domain_host = vars_match.groups()
-                self.domains = json.loads(domains)
-                self.domain_host = domain_host
-            else:
-                self.logger.warning(
-                    "Failed to extract domain variables from script content."
-                )
-        else:
-            self.logger.warning(
-                "Failed to find script content with domain variables in response."
-            )
-
-    def parse_stream_data(self, stream_data, item):
-        m3u8_url = self.m3u8_base_url.format(
-            domain=f"{random.choice(self.domains)}.{self.domain_host}",
-            source=stream_data["source"],
-            id=stream_data["id"],
-            stream_no=stream_data["streamNo"],
-        )
-
-        yield scrapy.Request(
-            self.session_url,
-            self.parse_session,
-            method="POST",
-            headers={"Content-Type": "application/json"},
-            body=json.dumps({"path": urlparse(m3u8_url).path}),
-            meta={"item": item, "stream_data": stream_data, "m3u8_url": m3u8_url},
-        )
-
-    def parse_session(self, response):
-        session_data = response.json()
-        session_id = session_data.get("id")
-        if not session_id:
-            self.logger.error("Failed to get session ID for stream.")
-            return
-
-        stream_data = response.meta["stream_data"].copy()
-        m3u8_url = response.meta["m3u8_url"]
-        stream_item = response.meta["item"].copy()
-
-        m3u8_url = f"{m3u8_url}?id={session_id}"
-
-        stream_item.update(
-            {
-                "stream_name": f"{'HD' if stream_data['hd'] else 'SD'} - 🌐 {stream_data['language']}\n"
-                f"🔗 {stream_data['source'].title()} Stream {stream_data['streamNo']}",
-                "stream_url": m3u8_url,
-                "referer": self.mediafusion_referer,
-                "description": f"{stream_item['title']} - {datetime.fromtimestamp(stream_item['event_start_timestamp'], tz=self.gmt).strftime('%I:%M%p GMT')}",
-            }
-        )
-
-        return stream_item

resources/html/home.html

@@ -24,7 +24,7 @@
             <ul>
                 <li>🏎️ Formula Racing: Exclusive scraping from TorrentGalaxy for all your racing needs.</li>
                 <li>🏈🏀⚾⚽🏒🏉🎾 American Football, Basketball, Baseball, Football, Hockey, Rugby/AFL, and Other Sports: Now all scraping through sport-video.org.ua for catchup videos.</li>
-                <li> 🏈🏀⚾⚽🏒🏉🎾🏏 Sports Live Events: Watch live sports events from streamed.su, streambtw.com</li>
+                <li> 🏈🏀⚾⚽🏒🏉🎾🏏 Sports Live Events: Watch live sports events from DaddyLiveHD</li>
                 <li>🎥 TamilMV: Specialized scraping for regional contents.</li>
                 <li>🌟 TamilBlasters: Dedicated access to an extensive library of regional content.</li>
                 <li>📺 TamilUltra & NowMeTV: Get the best of Live TV channels right at your fingertips.</li>

resources/json/scraper_config.json

@@ -256,32 +256,6 @@
 	  }
 	}
   },
-  "streamed": {
-    "api_base_url": "https://streamed.su/api",
-    "live_matches_endpoint": "/matches/live",
-    "stream_url_template": "/stream/{source}/{id}",
-    "image_url_template": "/images/poster/{batch_id_1}/{batch_id_2}.webp",
-    "m3u8_base_url": "https://{domain}/{source}/js/{id}/{stream_no}/playlist.m3u8",
-    "mediafusion_referer": "https://mediafusion.addon/",
-	"session_url": "https://secure.bigcoolersonline.top/init-session",
-    "category_mapping": {
-      "afl": "Afl",
-      "american-football": "American Football",
-      "baseball": "Baseball",
-      "basketball": "Basketball",
-      "billiards": "Billiards",
-      "cricket": "Cricket",
-      "darts": "Dart",
-      "fight": "Fighting",
-      "football": "Football",
-      "golf": "Golf",
-      "hockey": "Hockey",
-      "motor-sports": "Motor Sport",
-      "other": "Other Sports",
-      "rugby": "Rugby",
-      "tennis": "Tennis"
-    }
-  },
   "dlhd": {
 	"schedule_url": "https://dlhd.so/schedule/schedule-generated.json",
 	"m3u8_base_url": "https://xyzdddd.mizhls.ru/lb/premium{channel_id}/index.m3u8",

utils/const.py

@@ -159,10 +159,8 @@ SCRAPY_SPIDERS = {
     "nowsports": "NowSports",
     "tamilultra": "Tamil Ultra",
     "sport_video": "Sport Video",
-    "streamed": "Streamed Sport Events",
     "tamil_blasters": "TamilBlasters",
     "tamilmv": "TamilMV",
-    "streambtw": "StreamBTW",
     "dlhd": "DaddyLiveHD",
     "motogp_tgx": "MotoGP TGX",
     "arab_torrents": "Arab Torrents",