From 0a6c69eaaa2e393d0ea6e39c23bde3ea5d518fe9 Mon Sep 17 00:00:00 2001 From: mhdzumair Date: Sun, 16 Oct 2022 17:02:22 +0530 Subject: [PATCH] Added series data support --- api/main.py | 24 ++++++++++- db/crud.py | 88 +++++++++++++++++++++++++++++++++++------ db/models.py | 6 ++- resources/manifest.json | 18 ++++++--- utils/scrap.py | 52 ++++++++++++++++++------ 5 files changed, 155 insertions(+), 33 deletions(-) diff --git a/api/main.py b/api/main.py index 50b56fe..8472246 100644 --- a/api/main.py +++ b/api/main.py @@ -58,7 +58,7 @@ async def get_home(request: Request): "request": request, "name": manifest.get("name"), "version": manifest.get("version"), "description": manifest.get("description"), "gives": [ "Tamil Movies", "Malayalam Movies", "Telugu Movies", "Hindi Movies", "Kannada Movies", "English Movies", - "Dubbed Movies" + "Dubbed Movies", "Series" ], "logo": "static/tamilblasters.png" }, @@ -75,6 +75,8 @@ async def get_manifest(response: Response): @app.get("/catalog/movie/{catalog_id}.json", response_model=schemas.Movie) @app.get("/catalog/movie/{catalog_id}/skip={skip}.json", response_model=schemas.Movie) +@app.get("/catalog/series/{catalog_id}.json", response_model=schemas.Movie) +@app.get("/catalog/series/{catalog_id}/skip={skip}.json", response_model=schemas.Movie) async def get_catalog(response: Response, catalog_id: str, skip: int = 0): response.headers.update({ "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Headers": "*" @@ -106,8 +108,26 @@ async def get_stream(video_id: str, response: Response): def run_scraper( background_tasks: BackgroundTasks, language: Literal["tamil", "malayalam", "telugu", "hindi", "kannada", "english"] = "tamil", - video_type: Literal["hdrip", "tcrip", "dubbed"] = "hdrip", pages: int = 1, start_page: int = 1, + video_type: Literal["hdrip", "tcrip", "dubbed", "series"] = "hdrip", pages: int = 1, start_page: int = 1, is_scrape_home: bool = False, ): background_tasks.add_task(scrap.run_scraper, language, video_type, pages, start_page, is_scrape_home) return {"message": "Scraping in background..."} + + +@app.get("/meta/series/{meta_id}.json") +async def get_series_meta(meta_id: str, response: Response): + response.headers.update({ + "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Headers": "*" + }) + return await crud.get_series_meta(meta_id) + + +@app.get("/stream/series/{video_id}:{season}:{episode}.json", response_model=schemas.Streams) +async def get_series_streams(video_id: str, season: int, episode: str, response: Response): + response.headers.update({ + "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Headers": "*" + }) + streams = schemas.Streams() + streams.streams.extend(await crud.get_series_streams(video_id, season, episode)) + return streams diff --git a/db/crud.py b/db/crud.py index 9d5a777..d496013 100644 --- a/db/crud.py +++ b/db/crud.py @@ -2,7 +2,7 @@ import logging from typing import Optional from uuid import uuid4 -from imdb import Cinemagoer +from imdb import Cinemagoer, IMDbDataAccessError from api import schemas from db.models import TamilBlasterMovie @@ -15,18 +15,24 @@ async def get_movies_meta(catalog: str, skip: int = 0, limit: int = 100): movies = await TamilBlasterMovie.find(TamilBlasterMovie.catalog == catalog).sort("-created_at").skip(skip).limit( limit).to_list() + unique_names = [] + for movie in movies: meta_data = schemas.Meta.parse_obj(movie) meta_data.id = movie.imdb_id if movie.imdb_id else movie.tamilblaster_id - movies_meta.append(meta_data) + if movie.name not in unique_names: + movies_meta.append(meta_data) + unique_names.append(movie.name) return movies_meta -async def get_movies_data(video_id: str) -> list[Optional[TamilBlasterMovie]]: +async def get_movies_data(video_id: str, video_type: str = "movie") -> list[Optional[TamilBlasterMovie]]: if video_id.startswith("tt"): - movie_data = await TamilBlasterMovie.find(TamilBlasterMovie.imdb_id == video_id).to_list() + movie_data = await TamilBlasterMovie.find(TamilBlasterMovie.imdb_id == video_id, + TamilBlasterMovie.type == video_type).to_list() else: - movie_data = await TamilBlasterMovie.find(TamilBlasterMovie.tamilblaster_id == video_id).to_list() + movie_data = await TamilBlasterMovie.find(TamilBlasterMovie.tamilblaster_id == video_id, + TamilBlasterMovie.type == video_type).to_list() return movie_data @@ -47,6 +53,23 @@ async def get_movie_streams(video_id: str): return stream_data +async def get_series_streams(video_id: str, season: int, episode: str): + series_data = await get_movies_data(video_id, video_type="series") + if not series_data: + return [] + + stream_data = [] + for series in series_data: + if series.episode == episode and series.season == season: + for name, info_hash in series.video_qualities.items(): + stream_data.append({ + "name": name, + "infoHash": info_hash, + }) + + return stream_data + + async def get_movie_meta(meta_id: str): movies_data = await get_movies_data(meta_id) if not movies_data: @@ -63,8 +86,38 @@ async def get_movie_meta(meta_id: str): } +async def get_series_meta(meta_id: str): + series_data = await get_movies_data(meta_id, video_type="series") + if not series_data: + return + + metadata = { + "meta": { + "id": meta_id, + "type": "series", + "name": series_data[0].name, + "poster": series_data[0].poster, + "background": series_data[0].poster, + "videos": [] + } + } + for series in series_data: + metadata["meta"]["videos"].append({ + "id": f"{meta_id}:{series.season}:{series.episode}", + "name": f"{series.name} S{series.season} EP{series.episode}", + "season": series.season, + "episode": series.episode, + "released": series.created_at + }) + + return metadata + + def search_imdb(title: str): - result = ia.search_movie(title) + try: + result = ia.search_movie(title) + except IMDbDataAccessError: + return search_imdb(title) for movie in result: if movie.get("title").lower() in title.lower(): return f"tt{movie.movieID}" @@ -72,7 +125,8 @@ def search_imdb(title: str): async def save_movie_metadata(metadata: dict): movie_data = await TamilBlasterMovie.find_one( - TamilBlasterMovie.name == metadata["name"], TamilBlasterMovie.catalog == metadata["catalog"] + TamilBlasterMovie.name == metadata["name"], TamilBlasterMovie.catalog == metadata["catalog"], + TamilBlasterMovie.season == metadata["season"], TamilBlasterMovie.episode == metadata["episode"] ) if movie_data: @@ -82,11 +136,23 @@ async def save_movie_metadata(metadata: dict): else: movie_data = TamilBlasterMovie.parse_obj(metadata) movie_data.video_qualities = metadata["video_qualities"] - imdb_id = search_imdb(movie_data.name) - if imdb_id: - movie_data.imdb_id = imdb_id + + series_data = await TamilBlasterMovie.find_one( + TamilBlasterMovie.name == metadata["name"], TamilBlasterMovie.catalog == metadata["catalog"], + TamilBlasterMovie.type == "series" + ) + if series_data: + movie_data.tamilblaster_id = series_data.tamilblaster_id + movie_data.imdb_id = series_data.imdb_id else: - movie_data.tamilblaster_id = f"tb{uuid4().fields[-1]}" + imdb_id = search_imdb(movie_data.name) + if any([ + metadata["type"] == "series" and metadata["episode"].isdigit() and imdb_id, + all([metadata["type"] == "movie", imdb_id]) + ]): + movie_data.imdb_id = imdb_id + else: + movie_data.tamilblaster_id = f"tb{uuid4().fields[-1]}" logging.info(f"new movie '{metadata['name']}' added.") diff --git a/db/models.py b/db/models.py index 34bb331..861ec72 100644 --- a/db/models.py +++ b/db/models.py @@ -10,6 +10,9 @@ from pymongo import IndexModel class TamilBlasterMovie(Document): name: str catalog: str + type: str + season: Optional[int] + episode: Optional[str] poster: str imdb_id: Optional[str] tamilblaster_id: Optional[str] @@ -19,7 +22,8 @@ class TamilBlasterMovie(Document): class Settings: indexes = [ IndexModel( - [("name", pymongo.ASCENDING), ("catalog", pymongo.ASCENDING)], + [("name", pymongo.ASCENDING), ("catalog", pymongo.ASCENDING), + ("season", pymongo.ASCENDING), ("episode", pymongo.ASCENDING)], unique=True, ) ] diff --git a/resources/manifest.json b/resources/manifest.json index 0c04de5..abef442 100644 --- a/resources/manifest.json +++ b/resources/manifest.json @@ -1,42 +1,48 @@ { "id": "mhdzumair.addons.tamilblasters", - "version": "1.2.2", + "version": "1.3.0", "name": "Tamil Blasters", "contactEmail": "mhdzumair@gmail", - "description" : "TamilBlasters add-on for watch tamil, hindi, malayalam, kannada, english movies & dubbed movies. Created By: Mohamed Zumair. Source: https://github.com/mhdzumair/tamilblasters_streamio_addon", + "description" : "TamilBlasters add-on for watch tamil, hindi, malayalam, kannada, english, dubbed movies & series . Created By: Mohamed Zumair. Source: https://github.com/mhdzumair/tamilblasters_streamio_addon", "logo": "https://882b9915d0fe-stremio-tamilblasters.baby-beamup.club/static/tamilblasters.png", "resources": [ "catalog", { "name": "stream", - "types": ["movie"], + "types": ["movie", "series"], "idPrefixes": ["tt", "tb"] }, { "name": "meta", - "types": ["movie"], + "types": ["movie", "series"], "idPrefixes": ["tt", "tb"] } ], - "types": ["movies"], + "types": ["movie", "series"], "catalogs": [ {"id": "any_any", "type": "movie", "name": "TamilBlaster New Movies"}, {"id": "tamil_hdrip", "type": "movie", "name": "Tamil HD Movies"}, {"id": "tamil_tcrip", "type": "movie", "name": "Tamil TCRip/DVDScr/HDCAM Movies"}, {"id": "tamil_dubbed", "type": "movie", "name": "Tamil Dubbed Movies"}, + {"id": "tamil_series", "type": "series", "name": "Tamil Series"}, {"id": "malayalam_tcrip", "type": "movie", "name": "Malayalam TCRip/DVDScr/HDCAM Movies"}, {"id": "malayalam_hdrip", "type": "movie", "name": "Malayalam HD Movies"}, {"id": "malayalam_dubbed", "type": "movie", "name": "Malayalam Dubbed Movies"}, + {"id": "malayalam_series", "type": "series", "name": "Malayalam Series"}, {"id": "telugu_tcrip", "type": "movie", "name": "Telugu TCRip/DVDScr/HDCAM Movies"}, {"id": "telugu_hdrip", "type": "movie", "name": "Telugu HD Movies"}, {"id": "telugu_dubbed", "type": "movie", "name": "Telugu Dubbed Movies"}, + {"id": "telugu_series", "type": "series", "name": "Telugu Series"}, {"id": "hindi_tcrip", "type": "movie", "name": "Hindi TCRip/DVDScr/HDCAM Movies"}, {"id": "hindi_hdrip", "type": "movie", "name": "Hindi HD Movies"}, {"id": "hindi_dubbed", "type": "movie", "name": "Hindi Dubbed Movies"}, + {"id": "hindi_series", "type": "series", "name": "Hindi Series"}, {"id": "kannada_tcrip", "type": "movie", "name": "Kannada TCRip/DVDScr/HDCAM Movies"}, {"id": "kannada_hdrip", "type": "movie", "name": "Kannada HD Movies"}, {"id": "kannada_dubbed", "type": "movie", "name": "Kannada Dubbed Movies"}, + {"id": "kannada_series", "type": "series", "name": "Kannada Series"}, {"id": "english_hdrip", "type": "movie", "name": "English HD Movies"}, - {"id": "english_tcrip", "type": "movie", "name": "English TCRip/DVDScr/HDCAM Movies"} + {"id": "english_tcrip", "type": "movie", "name": "English TCRip/DVDScr/HDCAM Movies"}, + {"id": "english_series", "type": "series", "name": "English Series"} ] } \ No newline at end of file diff --git a/utils/scrap.py b/utils/scrap.py index 6de848f..378f930 100755 --- a/utils/scrap.py +++ b/utils/scrap.py @@ -19,31 +19,37 @@ tamil_blaster_links = { "tamil": { "hdrip": f"{homepage}/index.php?/forums/forum/7-tamil-new-movies-hdrips-bdrips-dvdrips-hdtv", "tcrip": f"{homepage}/index.php?/forums/forum/8-tamil-new-movies-tcrip-dvdscr-hdcam-predvd", - "dubbed": f"{homepage}/index.php?/forums/forum/9-tamil-dubbed-movies-bdrips-hdrips-dvdscr-hdcam-in-multi-audios" + "dubbed": f"{homepage}/index.php?/forums/forum/9-tamil-dubbed-movies-bdrips-hdrips-dvdscr-hdcam-in-multi-audios", + "series": f"{homepage}/index.php?/forums/forum/63-tamil-new-web-series-tv-shows" }, "malayalam": { "tcrip": f"{homepage}/index.php?/forums/forum/75-malayalam-new-movies-tcrip-dvdscr-hdcam-predvd", "hdrip": f"{homepage}/index.php?/forums/forum/74-malayalam-new-movies-hdrips-bdrips-dvdrips-hdtv", - "dubbed": f"{homepage}/index.php?/forums/forum/76-malayalam-dubbed-movies-bdrips-hdrips-dvdscr-hdcam" + "dubbed": f"{homepage}/index.php?/forums/forum/76-malayalam-dubbed-movies-bdrips-hdrips-dvdscr-hdcam", + "series": f"{homepage}/index.php?/forums/forum/98-malayalam-new-web-series-tv-shows" }, "telugu": { "tcrip": f"{homepage}/index.php?/forums/forum/79-telugu-new-movies-tcrip-dvdscr-hdcam-predvd", "hdrip": f"{homepage}/index.php?/forums/forum/78-telugu-new-movies-hdrips-bdrips-dvdrips-hdtv", - "dubbed": f"{homepage}/index.php?/forums/forum/80-telugu-dubbed-movies-bdrips-hdrips-dvdscr-hdcam" + "dubbed": f"{homepage}/index.php?/forums/forum/80-telugu-dubbed-movies-bdrips-hdrips-dvdscr-hdcam", + "series": f"{homepage}/index.php?/forums/forum/96-telugu-new-web-series-tv-shows" }, "hindi": { "tcrip": f"{homepage}/index.php?/forums/forum/87-hindi-new-movies-tcrip-dvdscr-hdcam-predvd", "hdrip": f"{homepage}/index.php?/forums/forum/86-hindi-new-movies-hdrips-bdrips-dvdrips-hdtv", - "dubbed": f"{homepage}/index.php?/forums/forum/88-hindi-dubbed-movies-bdrips-hdrips-dvdscr-hdcam" + "dubbed": f"{homepage}/index.php?/forums/forum/88-hindi-dubbed-movies-bdrips-hdrips-dvdscr-hdcam", + "series": f"{homepage}/index.php?/forums/forum/89-hindi-new-web-series-tv-shows" }, "kannada": { "tcrip": f"{homepage}/index.php?/forums/forum/83-kannada-new-movies-tcrip-dvdscr-hdcam-predvd", "hdrip": f"{homepage}/index.php?/forums/forum/82-kannada-new-movies-hdrips-bdrips-dvdrips-hdtv", - "dubbed": f"{homepage}/index.php?/forums/forum/84-kannada-dubbed-movies-bdrips-hdrips-dvdscr-hdcam" + "dubbed": f"{homepage}/index.php?/forums/forum/84-kannada-dubbed-movies-bdrips-hdrips-dvdscr-hdcam", + "series": f"{homepage}/index.php?/forums/forum/103-kannada-new-web-series-tv-shows" }, "english": { "tcrip": f"{homepage}/index.php?/forums/forum/52-english-movies-hdcam-dvdscr-predvd", - "hdrip": f"{homepage}/index.php?/forums/forum/53-english-movies-hdrips-bdrips-dvdrips" + "hdrip": f"{homepage}/index.php?/forums/forum/53-english-movies-hdrips-bdrips-dvdrips", + "series": f"{homepage}/index.php?/forums/forum/92-english-web-series-tv-shows" } } @@ -72,15 +78,27 @@ async def scrap_page(url, language, video_type): tamil_blasters = BeautifulSoup(response.content, "html.parser") - movies = tamil_blasters.find("ol").select("li[data-rowid]") + try: + movies = tamil_blasters.find("ol").select("li[data-rowid]") + except AttributeError: + logging.error(f"No data found for {language}:{video_type}") + return for movie in movies: movie = movie.find("a") - data = re.search(r"^(.+\(\d{4}\))(.+)", movie.text.strip()) + season = episode = None try: - title, video_quality = re.sub(r'\s+', ' ', data[1].strip()), data[2].strip("[] ") + if video_type == "series": + data = re.search(r"^(.+\(\d{4}\)).*S(\d+).*EP?\s?(\d+|\(\d+\s?-\s?\d+\))(.+)", movie.text.strip()) + title, season, video_quality = data[1], int(data[2]), data[4].strip("[] ") + episode = str(int(data[3])) if data[3].isdigit() else data[3].strip("()") + metadata = {"type": "series"} + else: + data = re.search(r"^(.+\(\d{4}\))(.+)", movie.text.strip()) + title, video_quality = re.sub(r'\s+', ' ', data[1].strip()), data[2].strip("[] ") + metadata = {"type": "movie"} except TypeError: - logging.error(f"not able to parse: {movie.text}") + logging.error(f"not able to parse: {movie.text.strip()}") continue logging.info(f"getting movie data for '{title}'") @@ -100,9 +118,11 @@ async def scrap_page(url, language, video_type): poster = movie_page.select_one("img[data-src]").get("data-src") created_at = dateparser(movie_page.find("time").get("datetime")) - metadata = {"name": title, "catalog": f"{language}_{video_type}", - "video_qualities": {video_quality: info_hash}, - "poster": poster, "created_at": created_at} + metadata.update({ + "name": title, "catalog": f"{language}_{video_type}", + "video_qualities": {video_quality: info_hash}, + "poster": poster, "created_at": created_at, "season": season, "episode": episode + }) await crud.save_movie_metadata(metadata) @@ -115,6 +135,9 @@ async def scrap_homepage(): movie_list = movie_list_div.find_all("p")[2:-2] for movie in movie_list: + if re.search(r"S(\d+).*EP?\s?(\d+|\(\d+\s?-\s?\d+\))", movie.text.strip()): + continue + data = re.search(r"^(.+\(\d{4}\))", movie.text.strip()) try: title = re.sub(r'\s+', ' ', data[1].strip()) @@ -128,6 +151,9 @@ async def scrap_homepage(): "name": title, "catalog": "any_any", "video_qualities": {}, + "type": "movie", + "season": None, + "episode": None } for video_quality in video_qualities: