Refactor pipelines and improve catalog parser pipeline handling logic.

Removed unnecessary size conversion logic to simplify code and reduce dependencies. Enhanced catalog processing with validation against predefined data and prevented duplication. Adjusted logging levels for better error visibility in TMDB integration.

mediafusion_scrapy/pipelines/catalog_parse_pipeline.py

@@ -1,3 +1,6 @@
+from utils.const import CATALOG_DATA
+
+
 class CatalogParsePipeline:
     def process_item(self, item, spider):
         if "video_type" not in item or "languages" not in item:
@@ -5,13 +8,17 @@ class CatalogParsePipeline:
         video_type = item["video_type"]
         languages = item["languages"]
         torrent_name = item["torrent_name"].lower()
-        catalogs = []
+        catalogs = item.get("catalog", [])
         for language in languages:
             if language.lower() == "english" and "eng" not in torrent_name:
                 # Fix for ESubs torrents
                 continue
             if video_type == "dubbed" and language.lower() == "english":
                 continue
-            catalogs.append(f"{language.lower()}_{video_type}")
+            catalog_name = f"{language.lower()}_{video_type}"
+            if catalog_name not in CATALOG_DATA:
+                continue
+            if catalog_name not in catalogs:
+                catalogs.append(catalog_name)
         item["catalog"] = catalogs
         return item

mediafusion_scrapy/pipelines/moto_gp_parser_pipeline.py

@@ -7,7 +7,6 @@ from scrapy.exceptions import DropItem
 from db.models import (
     Episode,
 )
-from utils.parser import convert_size_to_bytes
 from utils.runtime_const import SPORTS_ARTIFACTS
 
 
@@ -119,18 +118,12 @@ class MotoGPParserPipeline:
         episodes = []
         for index, file_detail in enumerate(torrent_data.get("file_data", [])):
             file_name = file_detail.get("filename")
-            file_size = file_detail.get("size")
-            size = (
-                convert_size_to_bytes(file_size)
-                if isinstance(file_size, str)
-                else file_size
-            )
 
             episodes.append(
                 Episode(
                     episode_number=index + 1,
                     filename=file_name,
-                    size=size,
+                    size=file_detail.get("size"),
                     file_index=index,
                     title=" ".join(file_name.split(".")[1:-1]),
                     released=torrent_data.get("created_at"),

mediafusion_scrapy/pipelines/sports_parser_pipeline.py

@@ -7,7 +7,6 @@ from scrapers.imdb_data import search_imdb
 from scrapy.exceptions import DropItem
 
 from scrapers.tmdb_data import search_tmdb
-from utils.parser import convert_size_to_bytes
 from utils.runtime_const import SPORTS_ARTIFACTS
 
 from cinemagoerng import web
@@ -109,11 +108,7 @@ class BaseParserPipeline:
 
         largest_file = max(
             torrent_data["file_data"],
-            key=lambda x: (
-                convert_size_to_bytes(x["size"])
-                if isinstance(x["size"], str)
-                else x["size"]
-            ),
+            key=lambda x: x["size"],
         )
         largest_file_index = torrent_data["file_data"].index(largest_file)
         torrent_data["largest_file"] = {

scrapers/tmdb_data.py

@@ -39,7 +39,6 @@ async def get_tmdb_data(tmdb_id: str, media_type: str) -> Optional[Dict[str, Any
                 headers=UA_HEADER,
                 timeout=10,
             )
-            logging.info(f"TMDB API response: {response.status_code}, {response.url}")
             response.raise_for_status()
             data = response.json()
 
@@ -106,7 +105,7 @@ async def get_tmdb_data(tmdb_id: str, media_type: str) -> Optional[Dict[str, Any
             return formatted_data
 
     except Exception as e:
-        logging.exception(f"Error fetching TMDB data for ID {tmdb_id}: {e}")
+        logging.error(f"Error fetching TMDB data for ID {tmdb_id}: {e}")
         return None