Add support for Live TV & fix several issues

api/main.py

@@ -4,7 +4,7 @@ from typing import Literal
 
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from apscheduler.triggers.cron import CronTrigger
-from fastapi import FastAPI, Request, Response, Depends, HTTPException
+from fastapi import FastAPI, Request, Response, Depends, HTTPException, status
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import RedirectResponse, FileResponse, StreamingResponse
 from fastapi.staticfiles import StaticFiles
@@ -19,7 +19,7 @@ from streaming_providers.seedr.api import router as seedr_router
 from streaming_providers.seedr.utils import get_direct_link_from_seedr
 from streaming_providers.debridlink.api import router as debridlink_router
 from streaming_providers.debridlink.utils import get_direct_link_from_debridlink
-from utils import crypto, torrent, poster
+from utils import crypto, torrent, poster, validation_helper
 from utils.const import CATALOG_ID_DATA, CATALOG_NAME_DATA
 from scrappers import tamil_blasters, tamilmv
 
@@ -174,18 +174,47 @@ async def get_manifest(
     response_model_by_alias=False,
     tags=["catalog"],
 )
+@app.get(
+    "/{secret_str}/catalog/{catalog_type}/{catalog_id}/genre={genre}.json",
+    response_model=schemas.Metas,
+    response_model_exclude_none=True,
+    response_model_by_alias=False,
+    tags=["catalog"],
+)
+@app.get(
+    "/catalog/{catalog_type}/{catalog_id}/genre={genre}.json",
+    response_model=schemas.Metas,
+    response_model_exclude_none=True,
+    response_model_by_alias=False,
+    tags=["catalog"],
+)
 async def get_catalog(
     response: Response,
-    catalog_type: Literal["movie", "series"],
+    catalog_type: Literal["movie", "series", "tv"],
     catalog_id: str,
     skip: int = 0,
+    genre: str = None,
 ):
     response.headers.update(headers)
+    if genre and "&" in genre:
+        genre, skip = genre.split("&")
+        skip = skip.split("=")[1] if "=" in skip else "0"
+        skip = int(skip) if skip and skip.isdigit() else 0
+
     metas = schemas.Metas()
-    metas.metas.extend(await crud.get_meta_list(catalog_type, catalog_id, skip))
+    if catalog_type == "tv":
+        metas.metas.extend(await crud.get_tv_meta_list(genre, skip))
+    else:
+        metas.metas.extend(await crud.get_meta_list(catalog_type, catalog_id, skip))
     return metas
 
 
+@app.get(
+    "/{secret_str}/catalog/{catalog_type}/{catalog_id}/search={search_query}.json",
+    tags=["search"],
+    response_model=schemas.Metas,
+    response_model_exclude_none=True,
+)
 @app.get(
     "/catalog/{catalog_type}/{catalog_id}/search={search_query}.json",
     tags=["search"],
@@ -194,8 +223,12 @@ async def get_catalog(
 )
 async def search_movie(
     response: Response,
-    catalog_type: Literal["movie", "series"],
+    catalog_type: Literal["movie", "series", "tv"],
-    catalog_id: Literal["mediafusion_search_movies", "mediafusion_search_series"],
+    catalog_id: Literal[
+        "mediafusion_search_movies",
+        "mediafusion_search_series",
+        "mediafusion_search_tv",
+    ],
     search_query: str,
 ):
     response.headers.update(headers)
@@ -219,13 +252,15 @@ async def search_movie(
     response_model_by_alias=False,
 )
 async def get_meta(
-    catalog_type: Literal["movie", "series"], meta_id: str, response: Response
+    catalog_type: Literal["movie", "series", "tv"], meta_id: str, response: Response
 ):
     response.headers.update(headers)
     if catalog_type == "movie":
         data = await crud.get_movie_meta(meta_id)
-    else:
+    elif catalog_type == "series":
         data = await crud.get_series_meta(meta_id)
+    else:
+        data = await crud.get_tv_meta(meta_id)
 
     if not data:
         raise HTTPException(status_code=404, detail="Meta ID not found.")
@@ -258,7 +293,7 @@ async def get_meta(
     tags=["stream"],
 )
 async def get_streams(
-    catalog_type: Literal["movie", "series"],
+    catalog_type: Literal["movie", "series", "tv"],
     video_id: str,
     response: Response,
     secret_str: str = None,
@@ -270,10 +305,12 @@ async def get_streams(
 
     if catalog_type == "movie":
         fetched_streams = await crud.get_movie_streams(user_data, secret_str, video_id)
-    else:
+    elif catalog_type == "series":
         fetched_streams = await crud.get_series_streams(
             user_data, secret_str, video_id, season, episode
         )
+    else:
+        fetched_streams = await crud.get_tv_streams(video_id)
 
     return {"streams": fetched_streams}
 
@@ -328,12 +365,16 @@ async def streaming_provider_endpoint(
 
 
 @app.get("/poster/{catalog_type}/{mediafusion_id}.jpg", tags=["poster"])
-async def get_poster(catalog_type: Literal["movie", "series"], mediafusion_id: str):
+async def get_poster(
+    catalog_type: Literal["movie", "series", "tv"], mediafusion_id: str
+):
     # Query the MediaFusion data
     if catalog_type == "movie":
         mediafusion_data = await crud.get_movie_data_by_id(mediafusion_id)
-    else:
+    elif catalog_type == "series":
         mediafusion_data = await crud.get_series_data_by_id(mediafusion_id)
+    else:
+        mediafusion_data = await crud.get_tv_data_by_id(mediafusion_id)
 
     if not mediafusion_data:
         raise HTTPException(status_code=404, detail="MediaFusion ID not found.")
@@ -351,6 +392,25 @@ async def get_poster(catalog_type: Literal["movie", "series"], mediafusion_id: s
         raise HTTPException(status_code=404, detail="Failed to create poster.")
 
 
+@app.post("/tv-metadata", status_code=status.HTTP_201_CREATED, tags=["tv"])
+async def add_tv_metadata(metadata: schemas.TVMetaData):
+    try:
+        metadata.streams = validation_helper.validate_tv_metadata(metadata)
+    except validation_helper.ValidationError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+    tv_channel_id, is_new = await crud.save_tv_channel_metadata(metadata)
+
+    if is_new:
+        return {
+            "status": f"Metadata with ID {tv_channel_id} has been created and is pending approval. Thanks for your contribution."
+        }
+
+    return {
+        "status": f"Tv Channel with ID {tv_channel_id} Streams has been updated. Thanks for your contribution."
+    }
+
+
 app.include_router(seedr_router, prefix="/seedr", tags=["seedr"])
 app.include_router(realdebrid_router, prefix="/realdebrid", tags=["realdebrid"])
 app.include_router(debridlink_router, prefix="/debridlink", tags=["debridlink"])

api_test.http

@@ -17,6 +17,21 @@ Accept: application/json
 
 ###
 
+GET http://127.0.0.1:8000/catalog/series/tamil_series.json
+Accept: application/json
+
+###
+
+GET http://127.0.0.1:8000/catalog/tv/live_tv.json
+Accept: application/json
+
+###
+
+GET http://127.0.0.1:8000/stream/tv/mf7c537827a3.json
+Accept: application/json
+
+###
+
 GET http://127.0.0.1:8000/catalog/movie/tamil_hdrip/skip=10.json
 Accept: application/json
 
@@ -37,7 +52,7 @@ Accept: application/json
 
 ###
 
-GET http://127.0.0.1:8000/{{secret_str}}/meta/series/mf39644457048721.json
+GET http://127.0.0.1:8000/{{secret_str}}/meta/series/mf279172080279251.json
 Accept: application/json
 
 ###
@@ -53,4 +68,11 @@ Accept: application/json
 ###
 
 GET http://127.0.0.1:8000/meta/movie/tt11468258.json
-Accept: application/json
+Accept: application/json
+
+###
+
+GET http://127.0.0.1:8000/stream/tv/mf1358a3be25.json
+Accept: application/json
+
+###

beamup.json

@@ -1,4 +1,4 @@
 {
     "projectName": "mediafusion",
-    "lastCommit": "65a6781"
+    "lastCommit": "8d1b505"
 }

db/crud.py

@@ -1,3 +1,4 @@
+import hashlib
 import logging
 from typing import Optional
 from uuid import uuid4
@@ -5,7 +6,7 @@ from uuid import uuid4
 from beanie import WriteRules
 from beanie.operators import In
 
-from db import schemas
+from db import schemas, models
 from db.config import settings
 from db.models import (
     MediaFusionMovieMetaData,
@@ -13,9 +14,15 @@ from db.models import (
     Streams,
     Season,
     Episode,
+    MediaFusionTVMetaData,
 )
 from db.schemas import Stream, MetaIdProjection
-from utils.parser import parse_stream_data, get_catalogs, search_imdb
+from utils.parser import (
+    parse_stream_data,
+    get_catalogs,
+    search_imdb,
+    parse_tv_stream_data,
+)
 
 
 async def get_meta_list(
@@ -43,6 +50,29 @@ async def get_meta_list(
     return meta_list
 
 
+async def get_tv_meta_list(
+    genre: Optional[str] = None, skip: int = 0, limit: int = 25
+) -> list[schemas.Meta]:
+    query = MediaFusionTVMetaData.find(
+        MediaFusionTVMetaData.is_approved == True, fetch_links=True
+    )
+    if genre:
+        query = query.find(In(MediaFusionTVMetaData.genres, [genre]))
+
+    tv_meta_list = (
+        await query.skip(skip)
+        .limit(limit)
+        .sort(-MediaFusionTVMetaData.streams.created_at)
+        .project(schemas.Meta)
+        .to_list()
+    )
+
+    for meta in tv_meta_list:
+        meta.poster = f"{settings.host_url}/poster/tv/{meta.id}.jpg"
+
+    return tv_meta_list
+
+
 async def get_movie_data_by_id(
     movie_id: str, fetch_links: bool = False
 ) -> Optional[MediaFusionMovieMetaData]:
@@ -59,6 +89,13 @@ async def get_series_data_by_id(
     return series_data
 
 
+async def get_tv_data_by_id(
+    tv_id: str, fetch_links: bool = False
+) -> Optional[MediaFusionTVMetaData]:
+    tv_data = await MediaFusionTVMetaData.get(tv_id, fetch_links=fetch_links)
+    return tv_data
+
+
 async def get_movie_streams(user_data, secret_str: str, video_id: str) -> list[Stream]:
     movie_data = await get_movie_data_by_id(video_id, True)
     if not movie_data:
@@ -83,6 +120,14 @@ async def get_series_streams(
     )
 
 
+async def get_tv_streams(video_id: str) -> list[Stream]:
+    tv_data = await get_tv_data_by_id(video_id, True)
+    if not tv_data:
+        return []
+
+    return parse_tv_stream_data(tv_data.streams)
+
+
 async def get_movie_meta(meta_id: str):
     movie_data = await get_movie_data_by_id(meta_id)
 
@@ -122,9 +167,23 @@ async def get_series_meta(meta_id: str):
         stream: Streams
         if stream.season:  # Ensure the stream has season data
             for episode in stream.season.episodes:
+                stream_id = (
+                    f"{meta_id}:{stream.season.season_number}:{episode.episode_number}"
+                )
+                # check if the stream is already in the list
+                if next(
+                    (
+                        video
+                        for video in metadata["meta"]["videos"]
+                        if video["id"] == stream_id
+                    ),
+                    None,
+                ):
+                    continue
+
                 metadata["meta"]["videos"].append(
                     {
-                        "id": f"{meta_id}:{stream.season.season_number}:{episode.episode_number}",
+                        "id": stream_id,
                         "name": f"S{stream.season.season_number} EP{episode.episode_number}",
                         "season": stream.season.season_number,
                         "episode": episode.episode_number,
@@ -135,6 +194,17 @@ async def get_series_meta(meta_id: str):
     return metadata
 
 
+async def get_tv_meta(meta_id: str):
+    tv_data = await get_tv_data_by_id(meta_id)
+
+    if not tv_data:
+        return {}
+
+    return {
+        "meta": {"_id": meta_id, **tv_data.model_dump()},
+    }
+
+
 async def save_movie_metadata(metadata: dict):
     # Try to get the existing movie
     existing_movie = await MediaFusionMovieMetaData.find_one(
@@ -313,6 +383,8 @@ async def save_series_metadata(metadata: dict):
 async def process_search_query(search_query: str, catalog_type: str) -> dict:
     if catalog_type == "movie":
         meta_class = MediaFusionMovieMetaData
+    elif catalog_type == "tv":
+        meta_class = MediaFusionTVMetaData
     else:
         meta_class = MediaFusionSeriesMetaData
 
@@ -332,6 +404,8 @@ async def process_search_query(search_query: str, catalog_type: str) -> dict:
         # Use the appropriate function to get the meta data
         if catalog_type == "movie":
             meta = await get_movie_meta(item.id)
+        elif catalog_type == "tv":
+            meta = await get_tv_meta(item.id)
         else:
             meta = await get_series_meta(item.id)
 
@@ -346,3 +420,65 @@ async def process_search_query(search_query: str, catalog_type: str) -> dict:
 async def get_stream_by_info_hash(info_hash: str) -> Streams | None:
     stream = await Streams.get(info_hash)
     return stream
+
+
+async def save_tv_channel_metadata(tv_metadata: schemas.TVMetaData) -> tuple[str, bool]:
+    # Try to get the existing TV channel
+    channel_id = "mf" + hashlib.sha256(tv_metadata.title.encode()).hexdigest()[:10]
+    logging.info(f"Processing TV channel id {channel_id}")
+    existing_channel = await models.MediaFusionTVMetaData.get(channel_id)
+
+    # Map the TVStreams schema to the TVStreams model
+    streams_models = []
+    for stream in tv_metadata.streams:
+        streams_models.append(
+            models.TVStreams(
+                url=stream.url,
+                name=stream.name,
+                behaviorHints=stream.behaviorHints.model_dump(exclude_none=True),
+                ytId=stream.ytId,
+                source=stream.source,
+            )
+        )
+
+    if existing_channel:
+        # Check for existing streams and update if necessary
+        await existing_channel.fetch_all_links()
+        for new_stream in streams_models:
+            matching_stream = next(
+                (
+                    s
+                    for s in existing_channel.streams
+                    if (s.url and s.url == new_stream.url)
+                    or (s.ytId and s.ytId == new_stream.ytId)
+                ),
+                None,
+            )
+            if not matching_stream:
+                existing_channel.streams.append(new_stream)
+        await existing_channel.save(link_rule=WriteRules.WRITE)
+        logging.info(f"Updated TV channel {existing_channel.title}")
+        is_new = False
+    else:
+        tv_metadata.genres.extend([tv_metadata.country, tv_metadata.tv_language])
+        genres = list(set(tv_metadata.genres))
+
+        # If the channel doesn't exist, create a new one
+        tv_channel_data = models.MediaFusionTVMetaData(
+            id=channel_id,
+            title=tv_metadata.title,
+            poster=tv_metadata.poster,
+            background=tv_metadata.background,
+            streams=streams_models,
+            type="tv",
+            country=tv_metadata.country,
+            tv_language=tv_metadata.tv_language,
+            logo=tv_metadata.logo,
+            genres=genres,
+            is_approved=False,
+        )
+        await tv_channel_data.insert(link_rule=WriteRules.WRITE)
+        logging.info(f"Added TV channel {tv_channel_data.title}")
+        is_new = True
+
+    return channel_id, is_new

db/database.py

@@ -2,7 +2,13 @@ import motor.motor_asyncio
 from beanie import init_beanie
 
 from db.config import settings
-from db.models import MediaFusionSeriesMetaData, MediaFusionMovieMetaData, Streams
+from db.models import (
+    MediaFusionSeriesMetaData,
+    MediaFusionMovieMetaData,
+    Streams,
+    TVStreams,
+    MediaFusionTVMetaData,
+)
 
 
 async def init():
@@ -10,4 +16,13 @@ async def init():
     client = motor.motor_asyncio.AsyncIOMotorClient(settings.mongo_uri)
 
     # Init beanie with the Product document class
-    await init_beanie(database=client.mediafusion, document_models=[MediaFusionMovieMetaData, MediaFusionSeriesMetaData, Streams])
+    await init_beanie(
+        database=client.mediafusion,
+        document_models=[
+            MediaFusionMovieMetaData,
+            MediaFusionSeriesMetaData,
+            Streams,
+            TVStreams,
+            MediaFusionTVMetaData,
+        ],
+    )

db/models.py

@@ -1,9 +1,10 @@
+import hashlib
 from datetime import datetime
-from typing import Optional
+from typing import Optional, Any
 
 import pymongo
 from beanie import Document, Link
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, computed_field
 from pymongo import IndexModel, ASCENDING
 
 
@@ -50,12 +51,21 @@ class Streams(Document):
         return None
 
 
+class TVStreams(Document):
+    name: str
+    url: str | None = None
+    ytId: str | None = None
+    source: str
+    behaviorHints: dict[str, Any] | None = None
+    created_at: datetime = Field(default_factory=datetime.now)
+
+
 class MediaFusionMetaData(Document):
     id: str
     title: str
-    year: Optional[int]
+    year: Optional[int] = None
     poster: str
-    background: str
+    background: Optional[str] = None
     streams: list[Link[Streams]]
     type: str
 
@@ -73,3 +83,13 @@ class MediaFusionMovieMetaData(MediaFusionMetaData):
 
 class MediaFusionSeriesMetaData(MediaFusionMetaData):
     type: str = "series"
+
+
+class MediaFusionTVMetaData(MediaFusionMetaData):
+    type: str = "tv"
+    country: str
+    tv_language: str
+    logo: Optional[str] = None
+    genres: Optional[list[str]] = None
+    is_approved: bool = False
+    streams: list[Link[TVStreams]]

db/schemas.py

@@ -1,6 +1,6 @@
-from typing import Optional, Any, Literal
+from typing import Optional, Literal
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
 
 from utils.const import CATALOG_ID_DATA
 
@@ -16,8 +16,12 @@ class Meta(BaseModel):
     name: str = Field(alias="title")
     type: str = Field(default="movie")
     poster: str
-    background: str
+    background: str | None = None
     videos: list | None = None
+    country: str | None = None
+    language: str | None = Field(None, alias="tv_language")
+    logo: Optional[str] = None
+    genres: Optional[list[str]] = None
 
 
 class MetaItem(BaseModel):
@@ -28,13 +32,20 @@ class Metas(BaseModel):
     metas: list[Meta] = []
 
 
+class StreamBehaviorHints(BaseModel):
+    notWebReady: Optional[bool] = None
+    bingeGroup: Optional[str] = None
+    proxyHeaders: Optional[dict[Literal["request", "response"], dict]] = None
+
+
 class Stream(BaseModel):
     name: str
     description: str
     infoHash: str | None = None
     fileIdx: int | None = None
     url: str | None = None
-    behaviorHints: dict[str, Any] | None = None
+    ytId: str | None = None
+    behaviorHints: StreamBehaviorHints | None = None
 
 
 class Streams(BaseModel):
@@ -63,3 +74,32 @@ class AuthorizeData(BaseModel):
 
 class MetaIdProjection(BaseModel):
     id: str = Field(alias="_id")
+
+
+class TVStreamsBehaviorHints(StreamBehaviorHints):
+    is_redirect: bool = False
+
+
+class TVStreams(BaseModel):
+    name: str
+    url: str | None = None
+    ytId: str | None = None
+    source: str
+    behaviorHints: TVStreamsBehaviorHints | None = None
+
+    @model_validator(mode="after")
+    def validate_url_or_yt_id(self) -> "TVStreams":
+        if not self.url and not self.ytId:
+            raise ValueError("Either url or ytId must be present")
+        return self
+
+
+class TVMetaData(BaseModel):
+    title: str
+    poster: str
+    background: Optional[str] = None
+    country: str
+    tv_language: str
+    logo: Optional[str] = None
+    genres: list[str] = []
+    streams: list[TVStreams]

resources/manifest.json

@@ -1,9 +1,9 @@
 {
   "id": "mhdzumair.addons.mediafusion",
-  "version": "3.4.5",
+  "version": "3.5.0",
   "name": "Media Fusion",
   "contactEmail": "mhdzumair@gmail",
-  "description": "Watch Tamil, Hindi, Malayalam, Kannada, English, and dubbed movies & series with the Media Fusion add-on. Source: https://github.com/mhdzumair/MediaFusion",
+  "description": "Watch Tamil, Hindi, Malayalam, Kannada, English, and dubbed movies, series & live TV with the Media Fusion add-on. Source: https://github.com/mhdzumair/MediaFusion",
   "logo": "https://882b9915d0fe-mediafusion.baby-beamup.club/static/images/mediafusion_logo.png",
   "behaviorHints": {
 	"configurable": true,
@@ -15,7 +15,8 @@
 	  "name": "stream",
 	  "types": [
 		"movie",
-		"series"
+		"series",
+		"tv"
 	  ],
 	  "idPrefixes": [
 		"tt",
@@ -26,7 +27,8 @@
 	  "name": "meta",
 	  "types": [
 		"movie",
-		"series"
+		"series",
+		"tv"
 	  ],
 	  "idPrefixes": [
 		"tt",
@@ -36,7 +38,8 @@
   ],
   "types": [
 	"movie",
-	"series"
+	"series",
+	"tv"
   ],
   "catalogs": [
 	{
@@ -368,6 +371,47 @@
 		  "isRequired": false
 		}
 	  ]
+	},
+	{
+	  "id": "live_tv",
+	  "type": "tv",
+	  "name": "Live TV",
+	  "extra": [
+		{
+		  "name": "skip",
+		  "isRequired": false
+		},
+		{
+		  "name": "genre",
+		  "isRequired": false,
+		  "options": [
+			"Tamil",
+			"Malayalam",
+			"English",
+			"Sports",
+			"Tamil Entertainment",
+			"India",
+			"Sri Lanka",
+			"Tamil Infotainment",
+			"Tamil Movies",
+			"Tamil Music",
+			"Tamil News",
+			"Tamil Kids",
+			"Hindi"
+		  ]
+		}
+	  ]
+	},
+	{
+	  "id": "mediafusion_search_tv",
+	  "type": "tv",
+	  "name": "Live TV",
+	  "extra": [
+		{
+		  "name": "search",
+		  "isRequired": true
+		}
+	  ]
 	}
   ]
 }

scrappers/tamilultra.py

@@ -0,0 +1,220 @@
+import argparse
+import asyncio
+import json
+import logging
+from urllib.parse import urlparse, urljoin
+
+import requests
+from playwright.async_api import async_playwright
+
+logging.basicConfig(
+    format="%(levelname)s::%(asctime)s - %(message)s", level=logging.INFO
+)
+BASE_URL = "https://tamilultra.in"
+MEDIAFUSION_URL = "http://127.0.0.1:8000"
+
+
+async def scrape_tv_channels(page):
+    # Scrape channel metadata
+    channels_data = []
+    channel_elements = await page.query_selector_all("article.item.movies")
+
+    # First, store all channel information in a list
+    channel_info_list = []
+    for channel_element in channel_elements:
+        title_element = await channel_element.query_selector("h3 > a")
+        title = (
+            (await title_element.text_content())
+            .replace("\u2013", "-")
+            .split("-")[0]
+            .strip()
+            .title()
+            if title_element
+            else "No Title"
+        )
+        poster_element = await channel_element.query_selector(".poster > img")
+        poster_url = (
+            await poster_element.get_attribute("src")
+            if poster_element
+            else "No Poster URL"
+        )
+        stream_page_link_element = await channel_element.query_selector(".poster > a")
+        stream_page_url = (
+            await stream_page_link_element.get_attribute("href")
+            if stream_page_link_element
+            else "No Stream Page URL"
+        )
+
+        channel_info_list.append((title, poster_url, stream_page_url))
+
+    # Then, navigate to each channel's stream page and capture the M3U8 URLs
+    for title, poster_url, stream_page_url in channel_info_list:
+        # Navigate to the stream page
+        await page.goto(stream_page_url)
+
+        m3u8_url_data = []
+
+        # Scrape genre tags
+        genre_elements = await page.query_selector_all(".sgeneros a[rel='tag']")
+        genres = [await genre.text_content() for genre in genre_elements]
+        genres = [genre.title() for genre in genres]
+
+        # Query for player option elements and click to load M3U8 URL
+        player_option_elements = await page.query_selector_all(
+            "#playeroptionsul > li.dooplay_player_option"
+        )
+        for player_option_element in player_option_elements:
+            # Click the player option element
+            await player_option_element.click()
+            # Wait for the iframe to load and get its 'src' attribute
+            iframe_element = await page.wait_for_selector("iframe.metaframe.rptss")
+            iframe_src = await iframe_element.get_attribute("src")
+            m3u8_url_part = iframe_src.replace("/player.php?", "")
+            # Check if iframe_src is a valid URL
+            parsed_src = urlparse(m3u8_url_part)
+            behavior_hints = {}
+            if parsed_src.scheme and parsed_src.netloc:
+                # If it's a valid URL, use it directly
+                m3u8_url = m3u8_url_part
+                if "jio.tamilultra.in" in m3u8_url:
+                    behavior_hints = {
+                        "notWebReady": True,
+                        "proxyHeaders": {
+                            "request": {
+                                "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
+                                "Referer": BASE_URL + "/",
+                            }
+                        },
+                    }
+
+            else:
+                # Otherwise, join with the BASE_URL
+                m3u8_url = urljoin(BASE_URL, m3u8_url_part)
+                behavior_hints = {
+                    "is_redirect": True,
+                    "proxyHeaders": {
+                        "request": {
+                            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
+                            "Referer": urljoin(BASE_URL, iframe_src),
+                        }
+                    },
+                }
+            m3u8_url_data.append((m3u8_url, behavior_hints))
+
+        channels_data.append(
+            {
+                "title": title.replace("Hd", "").strip(),
+                "poster": poster_url,
+                "genres": genres,
+                "country": "India",  # Set default country
+                "tv_language": genres[0],
+                "streams": [
+                    {
+                        "name": f"{title} - {index}",
+                        "url": m3u8_url,
+                        "source": "TamilUltra",
+                        "behaviorHints": behavior_hints,
+                    }
+                    for index, (m3u8_url, behavior_hints) in enumerate(m3u8_url_data, 1)
+                ],
+            }
+        )
+        logging.info("Scraped %s", title)
+
+    return channels_data
+
+
+async def scrape_category(category_url, page):
+    # Navigate to the category page
+    await page.goto(category_url)
+
+    # Scrape channels from the current page
+    channels_data = await scrape_tv_channels(
+        page
+    )  # Assuming scrape_tv_channels accepts a page argument
+
+    # Try to find a pagination control and collect all page URLs
+    pagination_links = await page.query_selector_all("div.pagination a.inactive")
+    page_urls = [category_url]  # Include the first page
+    for link in pagination_links:
+        page_url = await link.get_attribute("href")
+        if page_url:
+            page_urls.append(page_url)
+
+    logging.info("found %d pages", len(page_urls))
+
+    # Iterate over each page URL and scrape channels
+    for page_url in page_urls:
+        if page_url != category_url:  # We already scraped the first page
+            await page.goto(page_url)
+            # Scrape channels from this page
+            channels_data.extend(await scrape_tv_channels(page))
+
+    return channels_data
+
+
+async def scrape_all_categories():
+    async with async_playwright() as p:
+        browser = await p.firefox.launch(headless=True)
+        page = await browser.new_page()
+
+        # Extract category URLs
+        await page.goto(BASE_URL)
+        category_elements = await page.query_selector_all(".main-header a")
+        category_urls = [
+            urljoin(BASE_URL, await element.get_attribute("href"))
+            for element in category_elements
+        ]
+
+        # Scrape channels from each category
+        all_channels_data = []
+        for category_url in category_urls:
+            logging.info("Scraping %s", category_url)
+            all_channels_data.extend(await scrape_category(category_url, page))
+
+        await browser.close()
+
+        # remove duplicates
+        unique_channels = {channel["title"]: channel for channel in all_channels_data}
+        unique_channels_data = list(unique_channels.values())
+
+        logging.info("found %d channels", len(unique_channels_data))
+
+        with open("tamilultra.json", "w") as file:
+            json.dump({"channels": unique_channels_data}, file, indent=4)
+
+        logging.info(
+            "Done scraping TamilUltra. Manually verify the data & add it via /tv-metadata endpoint"
+        )
+
+
+def main(is_scraping: bool = True):
+    if is_scraping:
+        asyncio.run(scrape_all_categories())
+        return
+
+    with open("tamilultra.json") as file:
+        channels = json.load(file)["channels"]
+
+    for channel in channels:
+        logging.info("Adding %s", channel["title"])
+        response = requests.post(f"{MEDIAFUSION_URL}/tv-metadata", json=channel)
+
+        try:
+            response.raise_for_status()
+        except requests.HTTPError as err:
+            logging.info("Response data: %s", response.text)
+            continue
+
+        logging.info("Response data: %s", response.json())
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Scrape TamilUltra Live TV")
+    parser.add_argument(
+        "--no-scrape",
+        action="store_true",
+        help="Don't scrape TamilUltra. Use this option to add the data to MediaFusion",
+    )
+    args = parser.parse_args()
+    main(not args.no_scrape)

utils/const.py

@@ -29,6 +29,8 @@ CATALOG_ID_DATA = [
     "english_hdrip",
     "english_tcrip",
     "english_series",
+    "live_tv",
+    "mediafusion_search_tv",
 ]
 
 CATALOG_NAME_DATA = [
@@ -62,4 +64,6 @@ CATALOG_NAME_DATA = [
     "English HD Movies",
     "English TCRip/DVDScr/HDCAM Movies",
     "English Series",
+    "Live TV",
+    "MediaFusion Search TV",
 ]

utils/parser.py

@@ -5,7 +5,7 @@ import requests
 from imdb import Cinemagoer, IMDbDataAccessError
 
 from db.config import settings
-from db.models import Streams
+from db.models import Streams, TVStreams
 from db.schemas import Stream, UserData
 from streaming_providers.realdebrid.utils import (
     order_streams_by_instant_availability_and_date,
@@ -149,3 +149,27 @@ def search_imdb(title: str, year: int, retry: int = 5) -> dict:
                 "background": poster,
             }
     return {}
+
+
+def parse_tv_stream_data(stream: list[TVStreams]) -> list[Stream]:
+    stream_list = []
+    for stream in stream:
+        if stream.behaviorHints.get("is_redirect", False):
+            response = requests.get(
+                stream.url,
+                headers=stream.behaviorHints["proxyHeaders"]["request"],
+                allow_redirects=False,
+            )
+            if response.status_code == 302:
+                stream.url = response.headers["Location"]
+        stream_list.append(
+            Stream(
+                name="MediaFusion",
+                description=f"{stream.name}, {stream.source}",
+                url=stream.url,
+                ytId=stream.ytId,
+                behaviorHints=stream.behaviorHints,
+            )
+        )
+
+    return stream_list

utils/poster.py

@@ -14,7 +14,7 @@ async def create_poster(mediafusion_data: MediaFusionMetaData) -> BytesIO:
     response.raise_for_status()
 
     # Check if the response content type is an image
-    if not response.headers["Content-Type"].startswith("image/"):
+    if not response.headers["Content-Type"].lower().startswith("image/"):
         raise ValueError(
             f"Unexpected content type: {response.headers['Content-Type']} for URL: {mediafusion_data.poster}"
         )
@@ -24,7 +24,7 @@ async def create_poster(mediafusion_data: MediaFusionMetaData) -> BytesIO:
         raise ValueError(f"Empty content for URL: {mediafusion_data.poster}")
 
     try:
-        image = Image.open(BytesIO(response.content))
+        image = Image.open(BytesIO(response.content)).convert("RGBA")
     except UnidentifiedImageError:
         raise ValueError(f"Cannot identify image from URL: {mediafusion_data.poster}")
 

utils/validation_helper.py

@@ -0,0 +1,78 @@
+import logging
+
+import requests
+from urllib.parse import urlparse, quote
+from requests.exceptions import RequestException
+
+from db import schemas
+
+
+def is_valid_url(url: str) -> bool:
+    parsed_url = urlparse(url)
+    return all([parsed_url.scheme, parsed_url.netloc])
+
+
+def does_url_exist(url: str) -> bool:
+    try:
+        response = requests.head(url, allow_redirects=True, timeout=5)
+        return response.status_code == 200
+    except RequestException:
+        return False
+
+
+def validate_image_url(url: str) -> bool:
+    return is_valid_url(url) and does_url_exist(url)
+
+
+def validate_m3u8_url(url: str, behaviour_hint: dict) -> bool:
+    if not is_valid_url(url):
+        return False
+
+    try:
+        headers = behaviour_hint.get("proxyHeaders", {}).get("request", {})
+        response = requests.get(url, allow_redirects=True, headers=headers, timeout=30)
+        content_type = response.headers.get("Content-Type", "").lower()
+        return (
+            "application/vnd.apple.mpegurl" in content_type
+            or "application/x-mpegurl" in content_type
+        )
+    except RequestException as err:
+        logging.error(err)
+        return False
+
+
+class ValidationError(Exception):
+    pass
+
+
+def validate_yt_id(yt_id: str) -> bool:
+    image_url = f"https://img.youtube.com/vi/{yt_id}/mqdefault.jpg"
+    response = requests.head(image_url, allow_redirects=True, timeout=5)
+    return response.status_code == 200
+
+
+def validate_tv_metadata(metadata: schemas.TVMetaData) -> list[schemas.TVStreams]:
+    if (
+        not validate_image_url(metadata.poster)
+        or (metadata.logo and not validate_image_url(metadata.logo))
+        or (metadata.background and not validate_image_url(metadata.background))
+    ):
+        raise ValidationError("Invalid image URL provided.")
+
+    # Validate stream URLs at least 1 stream should be valid
+    valid_streams = [
+        stream
+        for stream in metadata.streams
+        if (
+            stream.url
+            and validate_m3u8_url(
+                stream.url, stream.behaviorHints.model_dump(exclude_none=True)
+            )
+        )
+        or (stream.ytId and validate_yt_id(stream.ytId))
+    ]
+    if not valid_streams:
+        raise ValidationError("Invalid stream URLs provided.")
+
+    unique_streams = {stream.url or stream.ytId: stream for stream in valid_streams}
+    return list(unique_streams.values())