mirror of
https://github.com/Viren070/MediaFusion.git
synced 2025-12-01 23:21:11 +01:00
Mohamed Zumair
52d8956034
* Refactor ZileanScraper to use parallel requests for searching and filtering streams with new endpoints * Fix DLHD scraping & enable DLHD without MediaFlow * Switch to httpx for async HTTP requests * Implement caching for PikPak token to reduce login error * Refactor torrent cleanup logic * Add inactivity monitor extension to close idle spiders Introduced `InactivityMonitor` to automatically close spiders that remain inactive for a specified time. This extension checks activity at regular intervals and uses configurable settings for check intervals and inactivity timeouts. If no items are scraped within the timeout period, the spider is closed to free resources. * Handle TypeError in dynamic sorting of streams * Refactor torrent info scraper to support pre-processing. Introduced a pre-processing function mapping to handle specific indexer requirements before parsing the HTML. Added a custom pre-processing function for "TheRARBG" to handle URL adjustments, improving modularity and readability in the `get_torrent_info` function. * Refine error logging and fix hash key in spider * Fix Prowlarr not stop on max process limit * #315: Integrate ScrapeOps logging into all scrapers & spiders Added ScrapeOps logging to Zilean, Torrentio, Prowlarr, and Prowlarr Feed scrapers to enhance request tracking and error handling. Configured ScrapeOps API key in settings and updated Pipfile/Pipfile.lock with scrapeops-python-requests and scrapeops-scrapy dependencies. * Refactor scraper cache status handler * verify torrent before parsing on prowlarr & prioritize magnet on badass_torrents * Add support for provide locally hosted mediaflow proxy public address and reduce the time leg on private ip address checking * handle RD exception cases * do not setup scrapeops when api key is none * Enhanced dynamic sorting of torrent streams Revised the dynamic_sort_key function to handle different key types more efficiently with match-case. Simplified error handling and improved logging to capture sorting data in the case of exceptions. * add missing last update date for metadata * update domain for nowmesports
131 lines
4.8 KiB
Python
131 lines
4.8 KiB
Python
from pydantic import model_validator, Field
|
|
from pydantic_settings import BaseSettings
|
|
|
|
|
|
class Settings(BaseSettings):
|
|
# Core Application Settings
|
|
addon_name: str = "MediaFusion"
|
|
host_url: str
|
|
secret_key: str = Field(..., max_length=32, min_length=32)
|
|
api_password: str
|
|
git_rev: str = "stable"
|
|
logging_level: str = "INFO"
|
|
logo_url: str = (
|
|
"https://raw.githubusercontent.com/mhdzumair/MediaFusion/main/resources/images/mediafusion_logo.png"
|
|
)
|
|
is_public_instance: bool = False
|
|
poster_host_url: str | None = None
|
|
|
|
# Database and Cache Settings
|
|
mongo_uri: str
|
|
db_max_connections: int = 50
|
|
redis_url: str = "redis://redis-service:6379"
|
|
|
|
# External Service URLs
|
|
scraper_proxy_url: str | None = None
|
|
playwright_cdp_url: str = "ws://browserless:3000?blockAds=true&stealth=true"
|
|
flaresolverr_url: str = "http://flaresolverr:8191/v1"
|
|
|
|
# External Service API Keys
|
|
scrapeops_api_key: str | None = None
|
|
|
|
# Prowlarr Settings
|
|
prowlarr_url: str = "http://prowlarr-service:9696"
|
|
prowlarr_api_key: str | None = None
|
|
prowlarr_live_title_search: bool = False
|
|
prowlarr_background_title_search: bool = True
|
|
prowlarr_search_query_timeout: int = 120
|
|
prowlarr_search_interval_hour: int = 24
|
|
prowlarr_immediate_max_process: int = 10
|
|
prowlarr_immediate_max_process_time: int = 15
|
|
prowlarr_feed_scrape_interval_hour: int = 3
|
|
|
|
# Torrentio Settings
|
|
torrentio_search_interval_days: int = 3
|
|
torrentio_url: str = "https://torrentio.strem.fun"
|
|
|
|
# Zilean Settings
|
|
zilean_search_interval_hour: int = 24
|
|
zilean_url: str = "http://zilean.zilean:8181"
|
|
|
|
# Premiumize Settings
|
|
premiumize_oauth_client_id: str | None = None
|
|
premiumize_oauth_client_secret: str | None = None
|
|
|
|
# Configuration Sources
|
|
remote_config_source: str = (
|
|
"https://raw.githubusercontent.com/mhdzumair/MediaFusion/main/resources/json/scraper_config.json"
|
|
)
|
|
local_config_path: str = "resources/json/scraper_config.json"
|
|
|
|
# Feature Toggles
|
|
is_scrap_from_torrentio: bool = False
|
|
is_scrap_from_zilean: bool = False
|
|
enable_rate_limit: bool = False
|
|
validate_m3u8_urls_liveness: bool = True
|
|
disable_download_via_browser: bool = False
|
|
|
|
# Content Filtering
|
|
adult_content_regex_keywords: str = (
|
|
r"(^|\b|\s|$|[\[._-])"
|
|
r"(18\s*\+|adults?|porn|sex|xxx|nude|boobs?|pussy|ass|bigass|bigtits?|blowjob|hardfuck|onlyfans?|naked|hot|milf|slut|doggy|anal|threesome|foursome|erotic|sexy|18\s*plus|trailer|RiffTrax)"
|
|
r"(\b|\s|$|[\]._-])"
|
|
)
|
|
|
|
# Time-related Settings
|
|
meta_cache_ttl: int = 1800 # 30 minutes in seconds
|
|
worker_max_tasks_per_child: int = 20
|
|
|
|
# Global Scheduler Settings
|
|
disable_all_scheduler: bool = False
|
|
|
|
# Individual Scheduler Settings
|
|
tamilmv_scheduler_crontab: str = "0 */3 * * *"
|
|
disable_tamilmv_scheduler: bool = False
|
|
tamil_blasters_scheduler_crontab: str = "0 */6 * * *"
|
|
disable_tamil_blasters_scheduler: bool = False
|
|
formula_tgx_scheduler_crontab: str = "*/30 * * * *"
|
|
disable_formula_tgx_scheduler: bool = False
|
|
nowmetv_scheduler_crontab: str = "0 0 * * *"
|
|
disable_nowmetv_scheduler: bool = False
|
|
nowsports_scheduler_crontab: str = "0 10 * * *"
|
|
disable_nowsports_scheduler: bool = False
|
|
tamilultra_scheduler_crontab: str = "0 8 * * *"
|
|
disable_tamilultra_scheduler: bool = False
|
|
validate_tv_streams_in_db_crontab: str = "0 */6 * * *"
|
|
disable_validate_tv_streams_in_db: bool = False
|
|
sport_video_scheduler_crontab: str = "*/20 * * * *"
|
|
disable_sport_video_scheduler: bool = False
|
|
streamed_scheduler_crontab: str = "*/30 * * * *"
|
|
disable_streamed_scheduler: bool = False
|
|
streambtw_scheduler_crontab: str = "*/15 * * * *"
|
|
disable_streambtw_scheduler: bool = True
|
|
dlhd_scheduler_crontab: str = "25 * * * *"
|
|
disable_dlhd_scheduler: bool = False
|
|
update_imdb_data_crontab: str = "0 2 * * *"
|
|
motogp_tgx_scheduler_crontab: str = "0 5 * * *"
|
|
disable_motogp_tgx_scheduler: bool = False
|
|
update_seeders_crontab: str = "0 0 * * *"
|
|
arab_torrents_scheduler_crontab: str = "0 0 * * *"
|
|
disable_arab_torrents_scheduler: bool = False
|
|
wwe_tgx_scheduler_crontab: str = "10 */3 * * *"
|
|
disable_wwe_tgx_scheduler: bool = False
|
|
ufc_tgx_scheduler_crontab: str = "30 */3 * * *"
|
|
disable_ufc_tgx_scheduler: bool = False
|
|
prowlarr_feed_scraper_crontab: str = "0 */3 * * *"
|
|
disable_prowlarr_feed_scraper: bool = False
|
|
cleanup_expired_scraper_task_crontab: str = "0 * * * *"
|
|
|
|
@model_validator(mode="after")
|
|
def default_poster_host_url(self) -> "Settings":
|
|
if not self.poster_host_url:
|
|
self.poster_host_url = self.host_url
|
|
return self
|
|
|
|
class Config:
|
|
env_file = ".env"
|
|
extra = "ignore"
|
|
|
|
|
|
settings = Settings()
|