From ac52a930c39e60790361150a2fdf237dd433e29c Mon Sep 17 00:00:00 2001 From: mhdzumair Date: Mon, 13 Nov 2023 21:35:23 +0530 Subject: [PATCH] Add support for Live TV & fix several issues --- api/main.py | 84 ++++++++++++-- api_test.http | 26 ++++- beamup.json | 2 +- db/crud.py | 142 +++++++++++++++++++++++- db/database.py | 19 +++- db/models.py | 28 ++++- db/schemas.py | 48 +++++++- resources/manifest.json | 54 ++++++++- scrappers/tamilultra.py | 220 +++++++++++++++++++++++++++++++++++++ utils/const.py | 4 + utils/parser.py | 26 ++++- utils/poster.py | 4 +- utils/validation_helper.py | 78 +++++++++++++ 13 files changed, 699 insertions(+), 36 deletions(-) create mode 100644 scrappers/tamilultra.py create mode 100644 utils/validation_helper.py diff --git a/api/main.py b/api/main.py index 615ad62..c13c8b4 100644 --- a/api/main.py +++ b/api/main.py @@ -4,7 +4,7 @@ from typing import Literal from apscheduler.schedulers.asyncio import AsyncIOScheduler from apscheduler.triggers.cron import CronTrigger -from fastapi import FastAPI, Request, Response, Depends, HTTPException +from fastapi import FastAPI, Request, Response, Depends, HTTPException, status from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import RedirectResponse, FileResponse, StreamingResponse from fastapi.staticfiles import StaticFiles @@ -19,7 +19,7 @@ from streaming_providers.seedr.api import router as seedr_router from streaming_providers.seedr.utils import get_direct_link_from_seedr from streaming_providers.debridlink.api import router as debridlink_router from streaming_providers.debridlink.utils import get_direct_link_from_debridlink -from utils import crypto, torrent, poster +from utils import crypto, torrent, poster, validation_helper from utils.const import CATALOG_ID_DATA, CATALOG_NAME_DATA from scrappers import tamil_blasters, tamilmv @@ -174,18 +174,47 @@ async def get_manifest( response_model_by_alias=False, tags=["catalog"], ) +@app.get( + "/{secret_str}/catalog/{catalog_type}/{catalog_id}/genre={genre}.json", + response_model=schemas.Metas, + response_model_exclude_none=True, + response_model_by_alias=False, + tags=["catalog"], +) +@app.get( + "/catalog/{catalog_type}/{catalog_id}/genre={genre}.json", + response_model=schemas.Metas, + response_model_exclude_none=True, + response_model_by_alias=False, + tags=["catalog"], +) async def get_catalog( response: Response, - catalog_type: Literal["movie", "series"], + catalog_type: Literal["movie", "series", "tv"], catalog_id: str, skip: int = 0, + genre: str = None, ): response.headers.update(headers) + if genre and "&" in genre: + genre, skip = genre.split("&") + skip = skip.split("=")[1] if "=" in skip else "0" + skip = int(skip) if skip and skip.isdigit() else 0 + metas = schemas.Metas() - metas.metas.extend(await crud.get_meta_list(catalog_type, catalog_id, skip)) + if catalog_type == "tv": + metas.metas.extend(await crud.get_tv_meta_list(genre, skip)) + else: + metas.metas.extend(await crud.get_meta_list(catalog_type, catalog_id, skip)) return metas +@app.get( + "/{secret_str}/catalog/{catalog_type}/{catalog_id}/search={search_query}.json", + tags=["search"], + response_model=schemas.Metas, + response_model_exclude_none=True, +) @app.get( "/catalog/{catalog_type}/{catalog_id}/search={search_query}.json", tags=["search"], @@ -194,8 +223,12 @@ async def get_catalog( ) async def search_movie( response: Response, - catalog_type: Literal["movie", "series"], - catalog_id: Literal["mediafusion_search_movies", "mediafusion_search_series"], + catalog_type: Literal["movie", "series", "tv"], + catalog_id: Literal[ + "mediafusion_search_movies", + "mediafusion_search_series", + "mediafusion_search_tv", + ], search_query: str, ): response.headers.update(headers) @@ -219,13 +252,15 @@ async def search_movie( response_model_by_alias=False, ) async def get_meta( - catalog_type: Literal["movie", "series"], meta_id: str, response: Response + catalog_type: Literal["movie", "series", "tv"], meta_id: str, response: Response ): response.headers.update(headers) if catalog_type == "movie": data = await crud.get_movie_meta(meta_id) - else: + elif catalog_type == "series": data = await crud.get_series_meta(meta_id) + else: + data = await crud.get_tv_meta(meta_id) if not data: raise HTTPException(status_code=404, detail="Meta ID not found.") @@ -258,7 +293,7 @@ async def get_meta( tags=["stream"], ) async def get_streams( - catalog_type: Literal["movie", "series"], + catalog_type: Literal["movie", "series", "tv"], video_id: str, response: Response, secret_str: str = None, @@ -270,10 +305,12 @@ async def get_streams( if catalog_type == "movie": fetched_streams = await crud.get_movie_streams(user_data, secret_str, video_id) - else: + elif catalog_type == "series": fetched_streams = await crud.get_series_streams( user_data, secret_str, video_id, season, episode ) + else: + fetched_streams = await crud.get_tv_streams(video_id) return {"streams": fetched_streams} @@ -328,12 +365,16 @@ async def streaming_provider_endpoint( @app.get("/poster/{catalog_type}/{mediafusion_id}.jpg", tags=["poster"]) -async def get_poster(catalog_type: Literal["movie", "series"], mediafusion_id: str): +async def get_poster( + catalog_type: Literal["movie", "series", "tv"], mediafusion_id: str +): # Query the MediaFusion data if catalog_type == "movie": mediafusion_data = await crud.get_movie_data_by_id(mediafusion_id) - else: + elif catalog_type == "series": mediafusion_data = await crud.get_series_data_by_id(mediafusion_id) + else: + mediafusion_data = await crud.get_tv_data_by_id(mediafusion_id) if not mediafusion_data: raise HTTPException(status_code=404, detail="MediaFusion ID not found.") @@ -351,6 +392,25 @@ async def get_poster(catalog_type: Literal["movie", "series"], mediafusion_id: s raise HTTPException(status_code=404, detail="Failed to create poster.") +@app.post("/tv-metadata", status_code=status.HTTP_201_CREATED, tags=["tv"]) +async def add_tv_metadata(metadata: schemas.TVMetaData): + try: + metadata.streams = validation_helper.validate_tv_metadata(metadata) + except validation_helper.ValidationError as e: + raise HTTPException(status_code=400, detail=str(e)) + + tv_channel_id, is_new = await crud.save_tv_channel_metadata(metadata) + + if is_new: + return { + "status": f"Metadata with ID {tv_channel_id} has been created and is pending approval. Thanks for your contribution." + } + + return { + "status": f"Tv Channel with ID {tv_channel_id} Streams has been updated. Thanks for your contribution." + } + + app.include_router(seedr_router, prefix="/seedr", tags=["seedr"]) app.include_router(realdebrid_router, prefix="/realdebrid", tags=["realdebrid"]) app.include_router(debridlink_router, prefix="/debridlink", tags=["debridlink"]) diff --git a/api_test.http b/api_test.http index 15d324a..e581d99 100644 --- a/api_test.http +++ b/api_test.http @@ -17,6 +17,21 @@ Accept: application/json ### +GET http://127.0.0.1:8000/catalog/series/tamil_series.json +Accept: application/json + +### + +GET http://127.0.0.1:8000/catalog/tv/live_tv.json +Accept: application/json + +### + +GET http://127.0.0.1:8000/stream/tv/mf7c537827a3.json +Accept: application/json + +### + GET http://127.0.0.1:8000/catalog/movie/tamil_hdrip/skip=10.json Accept: application/json @@ -37,7 +52,7 @@ Accept: application/json ### -GET http://127.0.0.1:8000/{{secret_str}}/meta/series/mf39644457048721.json +GET http://127.0.0.1:8000/{{secret_str}}/meta/series/mf279172080279251.json Accept: application/json ### @@ -53,4 +68,11 @@ Accept: application/json ### GET http://127.0.0.1:8000/meta/movie/tt11468258.json -Accept: application/json \ No newline at end of file +Accept: application/json + +### + +GET http://127.0.0.1:8000/stream/tv/mf1358a3be25.json +Accept: application/json + +### \ No newline at end of file diff --git a/beamup.json b/beamup.json index 0b9353d..ae1c9d0 100644 --- a/beamup.json +++ b/beamup.json @@ -1,4 +1,4 @@ { "projectName": "mediafusion", - "lastCommit": "65a6781" + "lastCommit": "8d1b505" } \ No newline at end of file diff --git a/db/crud.py b/db/crud.py index 1790977..e70b840 100644 --- a/db/crud.py +++ b/db/crud.py @@ -1,3 +1,4 @@ +import hashlib import logging from typing import Optional from uuid import uuid4 @@ -5,7 +6,7 @@ from uuid import uuid4 from beanie import WriteRules from beanie.operators import In -from db import schemas +from db import schemas, models from db.config import settings from db.models import ( MediaFusionMovieMetaData, @@ -13,9 +14,15 @@ from db.models import ( Streams, Season, Episode, + MediaFusionTVMetaData, ) from db.schemas import Stream, MetaIdProjection -from utils.parser import parse_stream_data, get_catalogs, search_imdb +from utils.parser import ( + parse_stream_data, + get_catalogs, + search_imdb, + parse_tv_stream_data, +) async def get_meta_list( @@ -43,6 +50,29 @@ async def get_meta_list( return meta_list +async def get_tv_meta_list( + genre: Optional[str] = None, skip: int = 0, limit: int = 25 +) -> list[schemas.Meta]: + query = MediaFusionTVMetaData.find( + MediaFusionTVMetaData.is_approved == True, fetch_links=True + ) + if genre: + query = query.find(In(MediaFusionTVMetaData.genres, [genre])) + + tv_meta_list = ( + await query.skip(skip) + .limit(limit) + .sort(-MediaFusionTVMetaData.streams.created_at) + .project(schemas.Meta) + .to_list() + ) + + for meta in tv_meta_list: + meta.poster = f"{settings.host_url}/poster/tv/{meta.id}.jpg" + + return tv_meta_list + + async def get_movie_data_by_id( movie_id: str, fetch_links: bool = False ) -> Optional[MediaFusionMovieMetaData]: @@ -59,6 +89,13 @@ async def get_series_data_by_id( return series_data +async def get_tv_data_by_id( + tv_id: str, fetch_links: bool = False +) -> Optional[MediaFusionTVMetaData]: + tv_data = await MediaFusionTVMetaData.get(tv_id, fetch_links=fetch_links) + return tv_data + + async def get_movie_streams(user_data, secret_str: str, video_id: str) -> list[Stream]: movie_data = await get_movie_data_by_id(video_id, True) if not movie_data: @@ -83,6 +120,14 @@ async def get_series_streams( ) +async def get_tv_streams(video_id: str) -> list[Stream]: + tv_data = await get_tv_data_by_id(video_id, True) + if not tv_data: + return [] + + return parse_tv_stream_data(tv_data.streams) + + async def get_movie_meta(meta_id: str): movie_data = await get_movie_data_by_id(meta_id) @@ -122,9 +167,23 @@ async def get_series_meta(meta_id: str): stream: Streams if stream.season: # Ensure the stream has season data for episode in stream.season.episodes: + stream_id = ( + f"{meta_id}:{stream.season.season_number}:{episode.episode_number}" + ) + # check if the stream is already in the list + if next( + ( + video + for video in metadata["meta"]["videos"] + if video["id"] == stream_id + ), + None, + ): + continue + metadata["meta"]["videos"].append( { - "id": f"{meta_id}:{stream.season.season_number}:{episode.episode_number}", + "id": stream_id, "name": f"S{stream.season.season_number} EP{episode.episode_number}", "season": stream.season.season_number, "episode": episode.episode_number, @@ -135,6 +194,17 @@ async def get_series_meta(meta_id: str): return metadata +async def get_tv_meta(meta_id: str): + tv_data = await get_tv_data_by_id(meta_id) + + if not tv_data: + return {} + + return { + "meta": {"_id": meta_id, **tv_data.model_dump()}, + } + + async def save_movie_metadata(metadata: dict): # Try to get the existing movie existing_movie = await MediaFusionMovieMetaData.find_one( @@ -313,6 +383,8 @@ async def save_series_metadata(metadata: dict): async def process_search_query(search_query: str, catalog_type: str) -> dict: if catalog_type == "movie": meta_class = MediaFusionMovieMetaData + elif catalog_type == "tv": + meta_class = MediaFusionTVMetaData else: meta_class = MediaFusionSeriesMetaData @@ -332,6 +404,8 @@ async def process_search_query(search_query: str, catalog_type: str) -> dict: # Use the appropriate function to get the meta data if catalog_type == "movie": meta = await get_movie_meta(item.id) + elif catalog_type == "tv": + meta = await get_tv_meta(item.id) else: meta = await get_series_meta(item.id) @@ -346,3 +420,65 @@ async def process_search_query(search_query: str, catalog_type: str) -> dict: async def get_stream_by_info_hash(info_hash: str) -> Streams | None: stream = await Streams.get(info_hash) return stream + + +async def save_tv_channel_metadata(tv_metadata: schemas.TVMetaData) -> tuple[str, bool]: + # Try to get the existing TV channel + channel_id = "mf" + hashlib.sha256(tv_metadata.title.encode()).hexdigest()[:10] + logging.info(f"Processing TV channel id {channel_id}") + existing_channel = await models.MediaFusionTVMetaData.get(channel_id) + + # Map the TVStreams schema to the TVStreams model + streams_models = [] + for stream in tv_metadata.streams: + streams_models.append( + models.TVStreams( + url=stream.url, + name=stream.name, + behaviorHints=stream.behaviorHints.model_dump(exclude_none=True), + ytId=stream.ytId, + source=stream.source, + ) + ) + + if existing_channel: + # Check for existing streams and update if necessary + await existing_channel.fetch_all_links() + for new_stream in streams_models: + matching_stream = next( + ( + s + for s in existing_channel.streams + if (s.url and s.url == new_stream.url) + or (s.ytId and s.ytId == new_stream.ytId) + ), + None, + ) + if not matching_stream: + existing_channel.streams.append(new_stream) + await existing_channel.save(link_rule=WriteRules.WRITE) + logging.info(f"Updated TV channel {existing_channel.title}") + is_new = False + else: + tv_metadata.genres.extend([tv_metadata.country, tv_metadata.tv_language]) + genres = list(set(tv_metadata.genres)) + + # If the channel doesn't exist, create a new one + tv_channel_data = models.MediaFusionTVMetaData( + id=channel_id, + title=tv_metadata.title, + poster=tv_metadata.poster, + background=tv_metadata.background, + streams=streams_models, + type="tv", + country=tv_metadata.country, + tv_language=tv_metadata.tv_language, + logo=tv_metadata.logo, + genres=genres, + is_approved=False, + ) + await tv_channel_data.insert(link_rule=WriteRules.WRITE) + logging.info(f"Added TV channel {tv_channel_data.title}") + is_new = True + + return channel_id, is_new diff --git a/db/database.py b/db/database.py index 1555e51..c0fe089 100644 --- a/db/database.py +++ b/db/database.py @@ -2,7 +2,13 @@ import motor.motor_asyncio from beanie import init_beanie from db.config import settings -from db.models import MediaFusionSeriesMetaData, MediaFusionMovieMetaData, Streams +from db.models import ( + MediaFusionSeriesMetaData, + MediaFusionMovieMetaData, + Streams, + TVStreams, + MediaFusionTVMetaData, +) async def init(): @@ -10,4 +16,13 @@ async def init(): client = motor.motor_asyncio.AsyncIOMotorClient(settings.mongo_uri) # Init beanie with the Product document class - await init_beanie(database=client.mediafusion, document_models=[MediaFusionMovieMetaData, MediaFusionSeriesMetaData, Streams]) + await init_beanie( + database=client.mediafusion, + document_models=[ + MediaFusionMovieMetaData, + MediaFusionSeriesMetaData, + Streams, + TVStreams, + MediaFusionTVMetaData, + ], + ) diff --git a/db/models.py b/db/models.py index feade45..257ff23 100644 --- a/db/models.py +++ b/db/models.py @@ -1,9 +1,10 @@ +import hashlib from datetime import datetime -from typing import Optional +from typing import Optional, Any import pymongo from beanie import Document, Link -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, computed_field from pymongo import IndexModel, ASCENDING @@ -50,12 +51,21 @@ class Streams(Document): return None +class TVStreams(Document): + name: str + url: str | None = None + ytId: str | None = None + source: str + behaviorHints: dict[str, Any] | None = None + created_at: datetime = Field(default_factory=datetime.now) + + class MediaFusionMetaData(Document): id: str title: str - year: Optional[int] + year: Optional[int] = None poster: str - background: str + background: Optional[str] = None streams: list[Link[Streams]] type: str @@ -73,3 +83,13 @@ class MediaFusionMovieMetaData(MediaFusionMetaData): class MediaFusionSeriesMetaData(MediaFusionMetaData): type: str = "series" + + +class MediaFusionTVMetaData(MediaFusionMetaData): + type: str = "tv" + country: str + tv_language: str + logo: Optional[str] = None + genres: Optional[list[str]] = None + is_approved: bool = False + streams: list[Link[TVStreams]] diff --git a/db/schemas.py b/db/schemas.py index fc7209e..483ddab 100644 --- a/db/schemas.py +++ b/db/schemas.py @@ -1,6 +1,6 @@ -from typing import Optional, Any, Literal +from typing import Optional, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator from utils.const import CATALOG_ID_DATA @@ -16,8 +16,12 @@ class Meta(BaseModel): name: str = Field(alias="title") type: str = Field(default="movie") poster: str - background: str + background: str | None = None videos: list | None = None + country: str | None = None + language: str | None = Field(None, alias="tv_language") + logo: Optional[str] = None + genres: Optional[list[str]] = None class MetaItem(BaseModel): @@ -28,13 +32,20 @@ class Metas(BaseModel): metas: list[Meta] = [] +class StreamBehaviorHints(BaseModel): + notWebReady: Optional[bool] = None + bingeGroup: Optional[str] = None + proxyHeaders: Optional[dict[Literal["request", "response"], dict]] = None + + class Stream(BaseModel): name: str description: str infoHash: str | None = None fileIdx: int | None = None url: str | None = None - behaviorHints: dict[str, Any] | None = None + ytId: str | None = None + behaviorHints: StreamBehaviorHints | None = None class Streams(BaseModel): @@ -63,3 +74,32 @@ class AuthorizeData(BaseModel): class MetaIdProjection(BaseModel): id: str = Field(alias="_id") + + +class TVStreamsBehaviorHints(StreamBehaviorHints): + is_redirect: bool = False + + +class TVStreams(BaseModel): + name: str + url: str | None = None + ytId: str | None = None + source: str + behaviorHints: TVStreamsBehaviorHints | None = None + + @model_validator(mode="after") + def validate_url_or_yt_id(self) -> "TVStreams": + if not self.url and not self.ytId: + raise ValueError("Either url or ytId must be present") + return self + + +class TVMetaData(BaseModel): + title: str + poster: str + background: Optional[str] = None + country: str + tv_language: str + logo: Optional[str] = None + genres: list[str] = [] + streams: list[TVStreams] diff --git a/resources/manifest.json b/resources/manifest.json index e5cd072..be61481 100644 --- a/resources/manifest.json +++ b/resources/manifest.json @@ -1,9 +1,9 @@ { "id": "mhdzumair.addons.mediafusion", - "version": "3.4.5", + "version": "3.5.0", "name": "Media Fusion", "contactEmail": "mhdzumair@gmail", - "description": "Watch Tamil, Hindi, Malayalam, Kannada, English, and dubbed movies & series with the Media Fusion add-on. Source: https://github.com/mhdzumair/MediaFusion", + "description": "Watch Tamil, Hindi, Malayalam, Kannada, English, and dubbed movies, series & live TV with the Media Fusion add-on. Source: https://github.com/mhdzumair/MediaFusion", "logo": "https://882b9915d0fe-mediafusion.baby-beamup.club/static/images/mediafusion_logo.png", "behaviorHints": { "configurable": true, @@ -15,7 +15,8 @@ "name": "stream", "types": [ "movie", - "series" + "series", + "tv" ], "idPrefixes": [ "tt", @@ -26,7 +27,8 @@ "name": "meta", "types": [ "movie", - "series" + "series", + "tv" ], "idPrefixes": [ "tt", @@ -36,7 +38,8 @@ ], "types": [ "movie", - "series" + "series", + "tv" ], "catalogs": [ { @@ -368,6 +371,47 @@ "isRequired": false } ] + }, + { + "id": "live_tv", + "type": "tv", + "name": "Live TV", + "extra": [ + { + "name": "skip", + "isRequired": false + }, + { + "name": "genre", + "isRequired": false, + "options": [ + "Tamil", + "Malayalam", + "English", + "Sports", + "Tamil Entertainment", + "India", + "Sri Lanka", + "Tamil Infotainment", + "Tamil Movies", + "Tamil Music", + "Tamil News", + "Tamil Kids", + "Hindi" + ] + } + ] + }, + { + "id": "mediafusion_search_tv", + "type": "tv", + "name": "Live TV", + "extra": [ + { + "name": "search", + "isRequired": true + } + ] } ] } \ No newline at end of file diff --git a/scrappers/tamilultra.py b/scrappers/tamilultra.py new file mode 100644 index 0000000..762b15c --- /dev/null +++ b/scrappers/tamilultra.py @@ -0,0 +1,220 @@ +import argparse +import asyncio +import json +import logging +from urllib.parse import urlparse, urljoin + +import requests +from playwright.async_api import async_playwright + +logging.basicConfig( + format="%(levelname)s::%(asctime)s - %(message)s", level=logging.INFO +) +BASE_URL = "https://tamilultra.in" +MEDIAFUSION_URL = "http://127.0.0.1:8000" + + +async def scrape_tv_channels(page): + # Scrape channel metadata + channels_data = [] + channel_elements = await page.query_selector_all("article.item.movies") + + # First, store all channel information in a list + channel_info_list = [] + for channel_element in channel_elements: + title_element = await channel_element.query_selector("h3 > a") + title = ( + (await title_element.text_content()) + .replace("\u2013", "-") + .split("-")[0] + .strip() + .title() + if title_element + else "No Title" + ) + poster_element = await channel_element.query_selector(".poster > img") + poster_url = ( + await poster_element.get_attribute("src") + if poster_element + else "No Poster URL" + ) + stream_page_link_element = await channel_element.query_selector(".poster > a") + stream_page_url = ( + await stream_page_link_element.get_attribute("href") + if stream_page_link_element + else "No Stream Page URL" + ) + + channel_info_list.append((title, poster_url, stream_page_url)) + + # Then, navigate to each channel's stream page and capture the M3U8 URLs + for title, poster_url, stream_page_url in channel_info_list: + # Navigate to the stream page + await page.goto(stream_page_url) + + m3u8_url_data = [] + + # Scrape genre tags + genre_elements = await page.query_selector_all(".sgeneros a[rel='tag']") + genres = [await genre.text_content() for genre in genre_elements] + genres = [genre.title() for genre in genres] + + # Query for player option elements and click to load M3U8 URL + player_option_elements = await page.query_selector_all( + "#playeroptionsul > li.dooplay_player_option" + ) + for player_option_element in player_option_elements: + # Click the player option element + await player_option_element.click() + # Wait for the iframe to load and get its 'src' attribute + iframe_element = await page.wait_for_selector("iframe.metaframe.rptss") + iframe_src = await iframe_element.get_attribute("src") + m3u8_url_part = iframe_src.replace("/player.php?", "") + # Check if iframe_src is a valid URL + parsed_src = urlparse(m3u8_url_part) + behavior_hints = {} + if parsed_src.scheme and parsed_src.netloc: + # If it's a valid URL, use it directly + m3u8_url = m3u8_url_part + if "jio.tamilultra.in" in m3u8_url: + behavior_hints = { + "notWebReady": True, + "proxyHeaders": { + "request": { + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36", + "Referer": BASE_URL + "/", + } + }, + } + + else: + # Otherwise, join with the BASE_URL + m3u8_url = urljoin(BASE_URL, m3u8_url_part) + behavior_hints = { + "is_redirect": True, + "proxyHeaders": { + "request": { + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36", + "Referer": urljoin(BASE_URL, iframe_src), + } + }, + } + m3u8_url_data.append((m3u8_url, behavior_hints)) + + channels_data.append( + { + "title": title.replace("Hd", "").strip(), + "poster": poster_url, + "genres": genres, + "country": "India", # Set default country + "tv_language": genres[0], + "streams": [ + { + "name": f"{title} - {index}", + "url": m3u8_url, + "source": "TamilUltra", + "behaviorHints": behavior_hints, + } + for index, (m3u8_url, behavior_hints) in enumerate(m3u8_url_data, 1) + ], + } + ) + logging.info("Scraped %s", title) + + return channels_data + + +async def scrape_category(category_url, page): + # Navigate to the category page + await page.goto(category_url) + + # Scrape channels from the current page + channels_data = await scrape_tv_channels( + page + ) # Assuming scrape_tv_channels accepts a page argument + + # Try to find a pagination control and collect all page URLs + pagination_links = await page.query_selector_all("div.pagination a.inactive") + page_urls = [category_url] # Include the first page + for link in pagination_links: + page_url = await link.get_attribute("href") + if page_url: + page_urls.append(page_url) + + logging.info("found %d pages", len(page_urls)) + + # Iterate over each page URL and scrape channels + for page_url in page_urls: + if page_url != category_url: # We already scraped the first page + await page.goto(page_url) + # Scrape channels from this page + channels_data.extend(await scrape_tv_channels(page)) + + return channels_data + + +async def scrape_all_categories(): + async with async_playwright() as p: + browser = await p.firefox.launch(headless=True) + page = await browser.new_page() + + # Extract category URLs + await page.goto(BASE_URL) + category_elements = await page.query_selector_all(".main-header a") + category_urls = [ + urljoin(BASE_URL, await element.get_attribute("href")) + for element in category_elements + ] + + # Scrape channels from each category + all_channels_data = [] + for category_url in category_urls: + logging.info("Scraping %s", category_url) + all_channels_data.extend(await scrape_category(category_url, page)) + + await browser.close() + + # remove duplicates + unique_channels = {channel["title"]: channel for channel in all_channels_data} + unique_channels_data = list(unique_channels.values()) + + logging.info("found %d channels", len(unique_channels_data)) + + with open("tamilultra.json", "w") as file: + json.dump({"channels": unique_channels_data}, file, indent=4) + + logging.info( + "Done scraping TamilUltra. Manually verify the data & add it via /tv-metadata endpoint" + ) + + +def main(is_scraping: bool = True): + if is_scraping: + asyncio.run(scrape_all_categories()) + return + + with open("tamilultra.json") as file: + channels = json.load(file)["channels"] + + for channel in channels: + logging.info("Adding %s", channel["title"]) + response = requests.post(f"{MEDIAFUSION_URL}/tv-metadata", json=channel) + + try: + response.raise_for_status() + except requests.HTTPError as err: + logging.info("Response data: %s", response.text) + continue + + logging.info("Response data: %s", response.json()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Scrape TamilUltra Live TV") + parser.add_argument( + "--no-scrape", + action="store_true", + help="Don't scrape TamilUltra. Use this option to add the data to MediaFusion", + ) + args = parser.parse_args() + main(not args.no_scrape) diff --git a/utils/const.py b/utils/const.py index 4f7e3a5..2be23e8 100644 --- a/utils/const.py +++ b/utils/const.py @@ -29,6 +29,8 @@ CATALOG_ID_DATA = [ "english_hdrip", "english_tcrip", "english_series", + "live_tv", + "mediafusion_search_tv", ] CATALOG_NAME_DATA = [ @@ -62,4 +64,6 @@ CATALOG_NAME_DATA = [ "English HD Movies", "English TCRip/DVDScr/HDCAM Movies", "English Series", + "Live TV", + "MediaFusion Search TV", ] diff --git a/utils/parser.py b/utils/parser.py index e4573f3..f466279 100644 --- a/utils/parser.py +++ b/utils/parser.py @@ -5,7 +5,7 @@ import requests from imdb import Cinemagoer, IMDbDataAccessError from db.config import settings -from db.models import Streams +from db.models import Streams, TVStreams from db.schemas import Stream, UserData from streaming_providers.realdebrid.utils import ( order_streams_by_instant_availability_and_date, @@ -149,3 +149,27 @@ def search_imdb(title: str, year: int, retry: int = 5) -> dict: "background": poster, } return {} + + +def parse_tv_stream_data(stream: list[TVStreams]) -> list[Stream]: + stream_list = [] + for stream in stream: + if stream.behaviorHints.get("is_redirect", False): + response = requests.get( + stream.url, + headers=stream.behaviorHints["proxyHeaders"]["request"], + allow_redirects=False, + ) + if response.status_code == 302: + stream.url = response.headers["Location"] + stream_list.append( + Stream( + name="MediaFusion", + description=f"{stream.name}, {stream.source}", + url=stream.url, + ytId=stream.ytId, + behaviorHints=stream.behaviorHints, + ) + ) + + return stream_list diff --git a/utils/poster.py b/utils/poster.py index 021fa1a..49fd44f 100644 --- a/utils/poster.py +++ b/utils/poster.py @@ -14,7 +14,7 @@ async def create_poster(mediafusion_data: MediaFusionMetaData) -> BytesIO: response.raise_for_status() # Check if the response content type is an image - if not response.headers["Content-Type"].startswith("image/"): + if not response.headers["Content-Type"].lower().startswith("image/"): raise ValueError( f"Unexpected content type: {response.headers['Content-Type']} for URL: {mediafusion_data.poster}" ) @@ -24,7 +24,7 @@ async def create_poster(mediafusion_data: MediaFusionMetaData) -> BytesIO: raise ValueError(f"Empty content for URL: {mediafusion_data.poster}") try: - image = Image.open(BytesIO(response.content)) + image = Image.open(BytesIO(response.content)).convert("RGBA") except UnidentifiedImageError: raise ValueError(f"Cannot identify image from URL: {mediafusion_data.poster}") diff --git a/utils/validation_helper.py b/utils/validation_helper.py new file mode 100644 index 0000000..bd1566e --- /dev/null +++ b/utils/validation_helper.py @@ -0,0 +1,78 @@ +import logging + +import requests +from urllib.parse import urlparse, quote +from requests.exceptions import RequestException + +from db import schemas + + +def is_valid_url(url: str) -> bool: + parsed_url = urlparse(url) + return all([parsed_url.scheme, parsed_url.netloc]) + + +def does_url_exist(url: str) -> bool: + try: + response = requests.head(url, allow_redirects=True, timeout=5) + return response.status_code == 200 + except RequestException: + return False + + +def validate_image_url(url: str) -> bool: + return is_valid_url(url) and does_url_exist(url) + + +def validate_m3u8_url(url: str, behaviour_hint: dict) -> bool: + if not is_valid_url(url): + return False + + try: + headers = behaviour_hint.get("proxyHeaders", {}).get("request", {}) + response = requests.get(url, allow_redirects=True, headers=headers, timeout=30) + content_type = response.headers.get("Content-Type", "").lower() + return ( + "application/vnd.apple.mpegurl" in content_type + or "application/x-mpegurl" in content_type + ) + except RequestException as err: + logging.error(err) + return False + + +class ValidationError(Exception): + pass + + +def validate_yt_id(yt_id: str) -> bool: + image_url = f"https://img.youtube.com/vi/{yt_id}/mqdefault.jpg" + response = requests.head(image_url, allow_redirects=True, timeout=5) + return response.status_code == 200 + + +def validate_tv_metadata(metadata: schemas.TVMetaData) -> list[schemas.TVStreams]: + if ( + not validate_image_url(metadata.poster) + or (metadata.logo and not validate_image_url(metadata.logo)) + or (metadata.background and not validate_image_url(metadata.background)) + ): + raise ValidationError("Invalid image URL provided.") + + # Validate stream URLs at least 1 stream should be valid + valid_streams = [ + stream + for stream in metadata.streams + if ( + stream.url + and validate_m3u8_url( + stream.url, stream.behaviorHints.model_dump(exclude_none=True) + ) + ) + or (stream.ytId and validate_yt_id(stream.ytId)) + ] + if not valid_streams: + raise ValidationError("Invalid stream URLs provided.") + + unique_streams = {stream.url or stream.ytId: stream for stream in valid_streams} + return list(unique_streams.values())