diff --git a/README.md b/README.md index 5bc0faa..a4f3597 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ - 🏎️ **Formula Racing**: Exclusive scraping from TorrentGalaxy for all your racing needs. - πŸ₯Š **Fighting Sports**: Catch up on all the latest fighting sports content from UFC and WWE. - πŸˆπŸ€βšΎβš½πŸ’πŸ‰πŸŽΎ **American Football, Basketball, Baseball, Football, Hockey, Rugby/AFL, and Other Sports**: Now all scraping through sport-video.org.ua for catchup videos. - - πŸˆπŸ€βšΎβš½πŸ’πŸ‰πŸŽΎπŸ **Sports Live Events**: Watch live sports events from streamed.su, streambtw.com + - πŸˆπŸ€βšΎβš½πŸ’πŸ‰πŸŽΎπŸ **Sports Live Events**: Watch live sports events from DaddyLiveHD - πŸŽ₯ **TamilMV**: Specialized scraping for regional contents. - 🌟 **TamilBlasters**: Dedicated access to an extensive library of regional content. - πŸ“Ί **TamilUltra & NowMeTV**: Get the best of Live TV channels right at your fingertips. diff --git a/api/scheduler.py b/api/scheduler.py index 5daa379..d1173c7 100644 --- a/api/scheduler.py +++ b/api/scheduler.py @@ -109,30 +109,6 @@ def setup_scheduler(scheduler: AsyncIOScheduler): }, ) - # Schedule streamed scraper - if not settings.disable_streamed_scheduler: - scheduler.add_job( - run_spider.send, - CronTrigger.from_crontab(settings.streamed_scheduler_crontab), - name="streamed", - kwargs={ - "spider_name": "streamed", - "crontab_expression": settings.streamed_scheduler_crontab, - }, - ) - - # Schedule streambtw scraper - if not settings.disable_streambtw_scheduler: - scheduler.add_job( - run_spider.send, - CronTrigger.from_crontab(settings.streambtw_scheduler_crontab), - name="streambtw", - kwargs={ - "spider_name": "streambtw", - "crontab_expression": settings.streambtw_scheduler_crontab, - }, - ) - # Schedule dlhd scraper if not settings.disable_dlhd_scheduler: scheduler.add_job( diff --git a/db/config.py b/db/config.py index 1957c5c..8ac1e05 100644 --- a/db/config.py +++ b/db/config.py @@ -124,10 +124,6 @@ class Settings(BaseSettings): disable_validate_tv_streams_in_db: bool = False sport_video_scheduler_crontab: str = "*/20 * * * *" disable_sport_video_scheduler: bool = False - streamed_scheduler_crontab: str = "*/30 * * * *" - disable_streamed_scheduler: bool = False - streambtw_scheduler_crontab: str = "*/15 * * * *" - disable_streambtw_scheduler: bool = True dlhd_scheduler_crontab: str = "25 * * * *" disable_dlhd_scheduler: bool = False update_imdb_data_crontab: str = "0 2 * * *" diff --git a/db/schemas.py b/db/schemas.py index f4450f6..3d1952f 100644 --- a/db/schemas.py +++ b/db/schemas.py @@ -358,10 +358,8 @@ class ScraperTask(BaseModel): "nowsports", "tamilultra", "sport_video", - "streamed", "tamilmv", "tamil_blasters", - "streambtw", "dlhd", "motogp_tgx", "arab_torrents", diff --git a/docs/configuration.md b/docs/configuration.md index 23c67b5..b7eaa5a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -122,10 +122,6 @@ Each scheduler has a crontab expression to define when it runs and a correspondi - **disable_validate_tv_streams_in_db** (default: `False`) - **sport_video_scheduler_crontab** (default: `"*/20 * * * *"`) - **disable_sport_video_scheduler** (default: `False`) -- **streamed_scheduler_crontab** (default: `"*/30 * * * *"`) -- **disable_streamed_scheduler** (default: `False`) -- **streambtw_scheduler_crontab** (default: `"*/15 * * * *"`) -- **disable_streambtw_scheduler** (default: `False`) - **dlhd_scheduler_crontab** (default: `"25 * * * *"`) - **disable_dlhd_scheduler** (default: `False`) - **update_imdb_data_crontab** (default: `"0 2 * * *"`) @@ -160,12 +156,8 @@ Note: Crontab expressions follow the standard cron format: "minute hour day-of-m - **disable_nowmetv_scheduler** (default: `False`): Disable NowMeTV scheduler. - **nowsports_scheduler_crontab** (default: `"0 10 * * *"`): Scheduler for NowSports. - **disable_nowsports_scheduler** (default: `False`): Disable NowSports scheduler. -- **streamed_scheduler_crontab** (default: `"*/15 * * * *"`): Scheduler for Streamed.su Sports Events. -- **disable_streamed_scheduler** (default: `False`): Disable Streamed.su scheduler. - **sport_video_scheduler_crontab** (default: `"*/20 * * * *"`): Scheduler for Sport Video. - **disable_sport_video_scheduler** (default: `False`): Disable Sport Video scheduler. -- **streambtw_scheduler_crontab** (default: `"*/15 * * * *"`): Scheduler for Streambtw. -- **disable_streambtw_scheduler** (default: `False`): Disable Streambtw scheduler. - **dlhd_scheduler_crontab** (default: `"25 * * * *"`): Scheduler for DLHD. - **disable_dlhd_scheduler**: Disable DLHD scheduler. - **update_imdb_data_crontab** (default: `"0 2 * * *"`): Scheduler for updating IMDb data. diff --git a/mediafusion_scrapy/spiders/streambtw.py b/mediafusion_scrapy/spiders/streambtw.py deleted file mode 100644 index ef7d4f8..0000000 --- a/mediafusion_scrapy/spiders/streambtw.py +++ /dev/null @@ -1,97 +0,0 @@ -import random -import re - -import scrapy - -from utils.runtime_const import SPORTS_ARTIFACTS -from db.redis_database import REDIS_SYNC_CLIENT - - -class StreamBTWSpider(scrapy.Spider): - name = "streambtw" - start_urls = ["https://streambtw.com/"] - referer = "https://streambtw.com/" - - custom_settings = { - "ITEM_PIPELINES": { - "mediafusion_scrapy.pipelines.LiveStreamResolverPipeline": 100, - "mediafusion_scrapy.pipelines.LiveEventStorePipeline": 300, - }, - "LOG_LEVEL": "DEBUG", - "DUPEFILTER_DEBUG": True, - } - category_map = { - "Soccer": "Football", - "NFL": "American Football", - "NBA": "Basketball", - "NHL": "Hockey", - "MLB": "Baseball", - "24x7": "Other Sports", - } - - def __init__(self, *args, **kwargs): - super(StreamBTWSpider, self).__init__(*args, **kwargs) - self.redis = REDIS_SYNC_CLIENT - - def __del__(self): - self.redis.close() - - def parse(self, response, **kwargs): - for timeline in response.xpath( - '//div[contains(@class,"single-timeline-area")]' - ): - category = timeline.xpath("./div/p/text()").get() - category = self.category_map.get(category, "Other Sports") - - events = timeline.xpath( - './/a[div[contains(@class, "single-timeline-content")]]' - ) - for event in events: - event_name = event.xpath(".//h6/text()").get() - description = event.xpath(".//p/text()").get() - event_url = event.xpath("@href").get() - logo = event.xpath(".//img/@src").get() - - item = { - "genres": [category], - "description": description, - "poster": random.choice(SPORTS_ARTIFACTS[category]["poster"]), - "background": random.choice( - SPORTS_ARTIFACTS[category]["background"] - ), - "logo": logo, - "is_add_title_to_poster": True, - "title": event_name, - "url": event_url, - "streams": [], - } - - yield response.follow( - event_url, self.parse_event, meta={"item": item}, dont_filter=True - ) - - def parse_event(self, response): - script_text = response.xpath( - "//script[contains(text(), 'm3u8List') or contains(text(), 'Clappr.Player')]/text()" - ).get() - - if script_text: - m3u8_urls = re.findall(r'"(https?://[^"]+)"', script_text) - for m3u8_url in m3u8_urls: - item = response.meta["item"].copy() - item.update( - { - "stream_name": f"{item['title']} - Live Stream", - "stream_url": m3u8_url, - "stream_source": "StreamBTW", - "stream_headers": { - "Referer": self.referer, - "Origin": self.referer.rstrip("/"), - }, - } - ) - yield item - else: - self.logger.error( - f"No suitable script found in event to extract M3U8 URLs. URL: {response.url}" - ) diff --git a/mediafusion_scrapy/spiders/streamed.py b/mediafusion_scrapy/spiders/streamed.py deleted file mode 100644 index cc4faff..0000000 --- a/mediafusion_scrapy/spiders/streamed.py +++ /dev/null @@ -1,180 +0,0 @@ -import json -import random -import re -from datetime import datetime -from urllib.parse import urljoin, urlparse - -import pytz -import scrapy - -from utils.config import config_manager -from utils.runtime_const import SPORTS_ARTIFACTS - - -class StreamedSpider(scrapy.Spider): - name = "streamed" - - api_base_url = config_manager.get_scraper_config(name, "api_base_url") - live_matches_url = f"{api_base_url}{config_manager.get_scraper_config(name, 'live_matches_endpoint')}" - stream_url_template = f"{api_base_url}{config_manager.get_scraper_config(name, 'stream_url_template')}" - image_url_template = ( - f"{api_base_url}{config_manager.get_scraper_config(name, 'image_url_template')}" - ) - m3u8_base_url = config_manager.get_scraper_config(name, "m3u8_base_url") - mediafusion_referer = config_manager.get_scraper_config(name, "mediafusion_referer") - category_mapping = config_manager.get_scraper_config(name, "category_mapping") - session_url = config_manager.get_scraper_config(name, "session_url") - - domains = None - domain_host = None - gmt = pytz.timezone("Etc/GMT") - - custom_settings = { - "ITEM_PIPELINES": { - "mediafusion_scrapy.pipelines.LiveStreamResolverPipeline": 100, - "mediafusion_scrapy.pipelines.LiveEventStorePipeline": 300, - }, - "DUPEFILTER_DEBUG": True, - "DEFAULT_REQUEST_HEADERS": {"Referer": mediafusion_referer}, - } - - def start_requests(self): - yield scrapy.Request(self.live_matches_url, self.parse_live_matches) - - def parse_live_matches(self, response): - matches = response.json() - for match in matches: - if not match.get("sources"): - self.logger.info(f"No sources available for match: {match['title']}") - continue - - category = self.category_mapping.get(match["category"], "Other") - - item = { - "stream_source": "Streamed (streamed.su)", - "genres": [category], - "title": match["title"], - "event_start_timestamp": match["date"] / 1000, # Convert to seconds - "streams": [], - } - - # Handle poster image - if "poster" in match and match["poster"]: - item["poster"] = urljoin(self.api_base_url, match["poster"]) - elif ( - match.get("teams") - and match["teams"].get("home", {}).get("badge") - and match["teams"].get("away", {}).get("badge") - ): - item["poster"] = self.image_url_template.format( - batch_id_1=match["teams"]["home"]["badge"], - batch_id_2=match["teams"]["away"]["badge"], - ) - else: - item["poster"] = random.choice(SPORTS_ARTIFACTS[category]["poster"]) - - item["background"] = random.choice(SPORTS_ARTIFACTS[category]["background"]) - item["logo"] = random.choice(SPORTS_ARTIFACTS[category]["logo"]) - item["is_add_title_to_poster"] = True - - for source in match["sources"]: - yield scrapy.Request( - self.stream_url_template.format( - source=source["source"], id=source["id"] - ), - self.parse_stream, - meta={"item": item}, - ) - - def parse_stream(self, response): - item = response.meta["item"].copy() - stream_data_list = response.json() - - for stream_data in stream_data_list: - if self.domains and self.domain_host: - yield from self.parse_stream_data(stream_data, item) - else: - yield scrapy.Request( - stream_data["embedUrl"], - self.parse_embed, - meta={"item": item, "stream_data": stream_data}, - headers={"Referer": self.mediafusion_referer}, - ) - - def parse_embed(self, response): - item = response.meta["item"] - stream_data = response.meta["stream_data"] - - if self.domains is None or self.domain_host is None: - self.extract_domain_info(response) - - if self.domains and self.domain_host: - yield from self.parse_stream_data(stream_data, item) - else: - self.logger.error( - f"Failed to extract domain information for stream: {stream_data['id']}" - ) - - def extract_domain_info(self, response): - script_content = response.xpath( - '//script[contains(text(), "var k=")]/text()' - ).get() - if script_content: - vars_match = re.search( - r'var k="(\w+)",i="([^"]+)",s="(\d+)",l=(\[.+?]),h="([^"]+)";', - script_content, - ) - if vars_match: - _, _, _, domains, domain_host = vars_match.groups() - self.domains = json.loads(domains) - self.domain_host = domain_host - else: - self.logger.warning( - "Failed to extract domain variables from script content." - ) - else: - self.logger.warning( - "Failed to find script content with domain variables in response." - ) - - def parse_stream_data(self, stream_data, item): - m3u8_url = self.m3u8_base_url.format( - domain=f"{random.choice(self.domains)}.{self.domain_host}", - source=stream_data["source"], - id=stream_data["id"], - stream_no=stream_data["streamNo"], - ) - - yield scrapy.Request( - self.session_url, - self.parse_session, - method="POST", - headers={"Content-Type": "application/json"}, - body=json.dumps({"path": urlparse(m3u8_url).path}), - meta={"item": item, "stream_data": stream_data, "m3u8_url": m3u8_url}, - ) - - def parse_session(self, response): - session_data = response.json() - session_id = session_data.get("id") - if not session_id: - self.logger.error("Failed to get session ID for stream.") - return - - stream_data = response.meta["stream_data"].copy() - m3u8_url = response.meta["m3u8_url"] - stream_item = response.meta["item"].copy() - - m3u8_url = f"{m3u8_url}?id={session_id}" - - stream_item.update( - { - "stream_name": f"{'HD' if stream_data['hd'] else 'SD'} - 🌐 {stream_data['language']}\n" - f"πŸ”— {stream_data['source'].title()} Stream {stream_data['streamNo']}", - "stream_url": m3u8_url, - "referer": self.mediafusion_referer, - "description": f"{stream_item['title']} - {datetime.fromtimestamp(stream_item['event_start_timestamp'], tz=self.gmt).strftime('%I:%M%p GMT')}", - } - ) - - return stream_item diff --git a/resources/html/home.html b/resources/html/home.html index eb1840c..902ccd5 100644 --- a/resources/html/home.html +++ b/resources/html/home.html @@ -24,7 +24,7 @@