diff --git a/mediafusion_scrapy/pipelines/catalog_parse_pipeline.py b/mediafusion_scrapy/pipelines/catalog_parse_pipeline.py index 704e465..8051cf4 100644 --- a/mediafusion_scrapy/pipelines/catalog_parse_pipeline.py +++ b/mediafusion_scrapy/pipelines/catalog_parse_pipeline.py @@ -1,3 +1,6 @@ +from utils.const import CATALOG_DATA + + class CatalogParsePipeline: def process_item(self, item, spider): if "video_type" not in item or "languages" not in item: @@ -5,13 +8,17 @@ class CatalogParsePipeline: video_type = item["video_type"] languages = item["languages"] torrent_name = item["torrent_name"].lower() - catalogs = [] + catalogs = item.get("catalog", []) for language in languages: if language.lower() == "english" and "eng" not in torrent_name: # Fix for ESubs torrents continue if video_type == "dubbed" and language.lower() == "english": continue - catalogs.append(f"{language.lower()}_{video_type}") + catalog_name = f"{language.lower()}_{video_type}" + if catalog_name not in CATALOG_DATA: + continue + if catalog_name not in catalogs: + catalogs.append(catalog_name) item["catalog"] = catalogs return item diff --git a/mediafusion_scrapy/pipelines/moto_gp_parser_pipeline.py b/mediafusion_scrapy/pipelines/moto_gp_parser_pipeline.py index 16d70bc..900bbb6 100644 --- a/mediafusion_scrapy/pipelines/moto_gp_parser_pipeline.py +++ b/mediafusion_scrapy/pipelines/moto_gp_parser_pipeline.py @@ -7,7 +7,6 @@ from scrapy.exceptions import DropItem from db.models import ( Episode, ) -from utils.parser import convert_size_to_bytes from utils.runtime_const import SPORTS_ARTIFACTS @@ -119,18 +118,12 @@ class MotoGPParserPipeline: episodes = [] for index, file_detail in enumerate(torrent_data.get("file_data", [])): file_name = file_detail.get("filename") - file_size = file_detail.get("size") - size = ( - convert_size_to_bytes(file_size) - if isinstance(file_size, str) - else file_size - ) episodes.append( Episode( episode_number=index + 1, filename=file_name, - size=size, + size=file_detail.get("size"), file_index=index, title=" ".join(file_name.split(".")[1:-1]), released=torrent_data.get("created_at"), diff --git a/mediafusion_scrapy/pipelines/sports_parser_pipeline.py b/mediafusion_scrapy/pipelines/sports_parser_pipeline.py index f50c3e4..6c02f28 100644 --- a/mediafusion_scrapy/pipelines/sports_parser_pipeline.py +++ b/mediafusion_scrapy/pipelines/sports_parser_pipeline.py @@ -7,7 +7,6 @@ from scrapers.imdb_data import search_imdb from scrapy.exceptions import DropItem from scrapers.tmdb_data import search_tmdb -from utils.parser import convert_size_to_bytes from utils.runtime_const import SPORTS_ARTIFACTS from cinemagoerng import web @@ -109,11 +108,7 @@ class BaseParserPipeline: largest_file = max( torrent_data["file_data"], - key=lambda x: ( - convert_size_to_bytes(x["size"]) - if isinstance(x["size"], str) - else x["size"] - ), + key=lambda x: x["size"], ) largest_file_index = torrent_data["file_data"].index(largest_file) torrent_data["largest_file"] = { diff --git a/scrapers/tmdb_data.py b/scrapers/tmdb_data.py index 3d54716..eece935 100644 --- a/scrapers/tmdb_data.py +++ b/scrapers/tmdb_data.py @@ -39,7 +39,6 @@ async def get_tmdb_data(tmdb_id: str, media_type: str) -> Optional[Dict[str, Any headers=UA_HEADER, timeout=10, ) - logging.info(f"TMDB API response: {response.status_code}, {response.url}") response.raise_for_status() data = response.json() @@ -106,7 +105,7 @@ async def get_tmdb_data(tmdb_id: str, media_type: str) -> Optional[Dict[str, Any return formatted_data except Exception as e: - logging.exception(f"Error fetching TMDB data for ID {tmdb_id}: {e}") + logging.error(f"Error fetching TMDB data for ID {tmdb_id}: {e}") return None