diff --git a/mediafusion_scrapy/pipelines/torrent_parser_pipeline.py b/mediafusion_scrapy/pipelines/torrent_parser_pipeline.py index 5838d8a..72ffd1a 100644 --- a/mediafusion_scrapy/pipelines/torrent_parser_pipeline.py +++ b/mediafusion_scrapy/pipelines/torrent_parser_pipeline.py @@ -44,13 +44,6 @@ class TorrentDownloadAndParsePipeline: response.body, item.get("parsed_data") ) - if settings.adult_content_filter_in_torrent_title and torrent_metadata.get( - "adult" - ): - raise DropItem( - f"Torrent name contains 18+ keywords: {torrent_metadata['torrent_name']}" - ) - if not torrent_metadata: return item diff --git a/scrapers/base_scraper.py b/scrapers/base_scraper.py index 2ee8788..e6f4046 100644 --- a/scrapers/base_scraper.py +++ b/scrapers/base_scraper.py @@ -541,13 +541,6 @@ class BaseScraper(abc.ABC): :return: True if valid, False otherwise """ - if settings.adult_content_filter_in_torrent_title and parsed_data.get("adult"): - self.metrics.record_skip("Adult content") - self.logger.debug( - f"Torrent title contains adult content: '{torrent_title}'" - ) - return False - # Check similarity ratios max_similarity_ratio = calculate_max_similarity_ratio( parsed_data["title"], metadata.title, metadata.aka_titles diff --git a/scrapers/feed_scraper.py b/scrapers/feed_scraper.py index 6edbc0c..e30491d 100644 --- a/scrapers/feed_scraper.py +++ b/scrapers/feed_scraper.py @@ -187,10 +187,7 @@ class FeedScraper(ABC): ) -> bool: """Validate feed item""" - if is_contain_18_plus_keywords(title) or ( - settings.adult_content_filter_in_torrent_title - and parsed_title_data.get("adult") - ): + if is_contain_18_plus_keywords(title): logger.info(f"Skipping adult content: {title}") scraper.metrics.record_skip("Adult Content") return False diff --git a/scrapers/routes.py b/scrapers/routes.py index 1c1b3cb..4d023d0 100644 --- a/scrapers/routes.py +++ b/scrapers/routes.py @@ -238,10 +238,6 @@ async def add_torrent( raise_error(str(e)) if not torrent_data: raise_error("Failed to extract torrent metadata.") - if settings.adult_content_filter_in_torrent_title and torrent_data.get("adult"): - raise_error( - f"Torrent name contains 18+ keywords: {torrent_data['torrent_name']}" - ) info_hash = torrent_data.get("info_hash") # Check if torrent already exists diff --git a/utils/parser.py b/utils/parser.py index 4bb4803..34b80d7 100644 --- a/utils/parser.py +++ b/utils/parser.py @@ -5,8 +5,8 @@ import logging import math import re from datetime import datetime, timezone -from typing import Optional, List, Any from os.path import basename +from typing import Optional, List, Any from urllib.parse import quote from thefuzz import fuzz @@ -24,7 +24,7 @@ from utils import const from utils.config import config_manager from utils.const import STREAMING_PROVIDERS_SHORT_NAMES, CERTIFICATION_MAPPING from utils.network import encode_mediaflow_proxy_url -from utils.runtime_const import ADULT_CONTENT_KEYWORDS, TRACKERS, MANIFEST_TEMPLATE +from utils.runtime_const import TRACKERS, MANIFEST_TEMPLATE, ADULT_PARSER from utils.validation_helper import validate_m3u8_or_mpd_url_with_cache @@ -660,7 +660,10 @@ def is_contain_18_plus_keywords(title: str) -> bool: """ Check if the title contains 18+ keywords to filter out adult content. """ - return ADULT_CONTENT_KEYWORDS.search(title) is not None + if not settings.adult_content_filter_in_torrent_title: + return False + + return ADULT_PARSER.parse(title).get("adult", False) def calculate_max_similarity_ratio( diff --git a/utils/runtime_const.py b/utils/runtime_const.py index a316671..f6d3c5d 100644 --- a/utils/runtime_const.py +++ b/utils/runtime_const.py @@ -1,15 +1,24 @@ import re from datetime import timedelta +import PTT from fastapi.templating import Jinja2Templates +from regex import regex from db import schemas from db.config import settings from utils import get_json_data, const -ADULT_CONTENT_KEYWORDS = re.compile( - settings.adult_content_regex_keywords, - re.IGNORECASE, +ADULT_PARSER = PTT.Parser() +ADULT_PARSER.add_handler( + "adult", + PTT.handlers.create_adult_pattern(), + PTT.transformers.boolean, +) +ADULT_PARSER.add_handler( + "adult", + regex.compile(settings.adult_content_regex_keywords, regex.IGNORECASE), + PTT.transformers.boolean, ) SPORTS_ARTIFACTS = get_json_data("resources/json/sports_artifacts.json") diff --git a/utils/torrent.py b/utils/torrent.py index d761e70..445d9fa 100644 --- a/utils/torrent.py +++ b/utils/torrent.py @@ -67,7 +67,7 @@ def extract_torrent_metadata( else: metadata.update(PTT.parse_title(torrent_name, True)) - if is_contain_18_plus_keywords(torrent_name) or metadata.get("adult"): + if is_contain_18_plus_keywords(torrent_name): logging.warning( f"Torrent name contains 18+ keywords: {torrent_name}. Skipping" )