From 035ae1a4c7a0ca281ade661d5aeae73b7738edf2 Mon Sep 17 00:00:00 2001 From: mhdzumair Date: Sat, 16 Nov 2024 10:40:06 +0530 Subject: [PATCH] Refactor proxy configuration to support advanced transport settings Changed the proxy configuration system to a more flexible transport configuration using HTTPX routing. Introduced new settings for global proxy enablement and robust pattern-based proxy and SSL configurations. Updated README.md with new examples and detailed explanations of the new configurations. --- README.md | 82 ++++++++++++++++++++++----- mediaflow_proxy/configs.py | 35 ++++++++---- mediaflow_proxy/extractors/base.py | 3 +- mediaflow_proxy/extractors/factory.py | 4 +- mediaflow_proxy/routes/extractor.py | 4 +- mediaflow_proxy/speedtest/service.py | 7 +-- mediaflow_proxy/utils/http_utils.py | 2 +- 7 files changed, 98 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 0dd7a74..50fc1dc 100644 --- a/README.md +++ b/README.md @@ -60,39 +60,93 @@ Set the following environment variables: - `API_PASSWORD`: Required. Protects against unauthorized access and API network abuses. - `ENABLE_STREAMING_PROGRESS`: Optional. Enable streaming progress logging. Default is `false`. -### Proxy Configuration Examples +### Transport Configuration -MediaFlow Proxy now supports advanced proxy routing using HTTPX's routing system. You can configure different proxy rules for different domains, protocols, and patterns. Here are some examples: +MediaFlow Proxy now supports advanced transport configuration using HTTPX's routing system. You can configure proxy and SSL verification settings for different domains and protocols. -1. Basic proxy configuration with a default proxy: +#### Basic Configuration + +Enable proxy for all routes: ```env -PROXY_DEFAULT_URL=http://default-proxy:8080 +PROXY_URL=http://proxy:8080 +ALL_PROXY=true ``` -2. Advanced routing with multiple rules: +#### Advanced Routing Configuration + +Configure different proxy settings for specific patterns: ```env -PROXY_ROUTES='{ - "all://*.debrid.com": { - "proxy_url": "socks5://debrid-proxy:8080" - }, +PROXY_URL=http://proxy:8080 +TRANSPORT_ROUTES='{ "https://internal.company.com": { - "proxy_url": null, - "verify_ssl": false + "proxy": false }, - "all://api.external.com": { - "proxy_url": "http://api-proxy:8080", + "all://streaming.service.com": { + "proxy_url": "socks5://streaming-proxy:1080", "verify_ssl": false } }' ``` -Proxy routing supports various patterns: +The routing system supports various patterns: - Domain routing: `"all://example.com"` - Subdomain routing: `"all://*.example.com"` - Protocol-specific routing: `"https://example.com"` - Port-specific routing: `"all://*:1234"` - Wildcard routing: `"all://"` +#### Route Configuration Options + +Each route can have the following settings: +- `proxy`: Boolean to enable/disable proxy for this route (default: true) +- `proxy_url`: Optional specific proxy URL for this route (overrides primary proxy_url) +- `verify_ssl`: Boolean to control SSL verification (default: true) + +#### Configuration Examples + +1. Simple proxy setup with SSL bypass for internal domain: +```env +PROXY_URL=http://main-proxy:8080 +TRANSPORT_ROUTES='{ + "https://internal.domain.com": { + "proxy": false, + "verify_ssl": false + } +}' +``` + +2. Different proxies for different services: +```env +PROXY_URL=http://default-proxy:8080 +TRANSPORT_ROUTES='{ + "all://*.streaming.com": { + "proxy": true, + "proxy_url": "socks5://streaming-proxy:1080" + }, + "all://*.internal.com": { + "proxy": false + }, + "https://api.service.com": { + "proxy": true, + "verify_ssl": false + } +}' +``` + +3. Global proxy with exceptions: +```env +PROXY_URL=http://main-proxy:8080 +ALL_PROXY=true +TRANSPORT_ROUTES='{ + "all://local.network": { + "proxy": false + }, + "all://*.trusted-service.com": { + "proxy": false + } +}' +``` + ### Speed Test Feature MediaFlow Proxy now includes a built-in speed test feature for testing RealDebrid and AllDebrid network speeds. To access the speed test: diff --git a/mediaflow_proxy/configs.py b/mediaflow_proxy/configs.py index 3d1d5d5..da6bfbd 100644 --- a/mediaflow_proxy/configs.py +++ b/mediaflow_proxy/configs.py @@ -5,14 +5,24 @@ from pydantic import BaseModel, Field from pydantic_settings import BaseSettings -class ProxyRoute(BaseModel): +class RouteConfig(BaseModel): + """Configuration for a specific route""" + + proxy: bool = True proxy_url: Optional[str] = None verify_ssl: bool = True -class ProxyConfig(BaseSettings): - default_url: Optional[str] = None - routes: Dict[str, ProxyRoute] = Field(default_factory=dict) +class TransportConfig(BaseSettings): + """Main proxy configuration""" + + proxy_url: Optional[str] = Field( + None, description="Primary proxy URL. Example: socks5://user:pass@proxy:1080 or http://proxy:8080" + ) + all_proxy: bool = Field(False, description="Enable proxy for all routes by default") + transport_routes: Dict[str, RouteConfig] = Field( + default_factory=dict, description="Pattern-based route configuration" + ) def get_mounts( self, async_http: bool = True @@ -23,25 +33,26 @@ class ProxyConfig(BaseSettings): mounts = {} transport_cls = httpx.AsyncHTTPTransport if async_http else httpx.HTTPTransport - # Add specific routes - for pattern, route in self.routes.items(): - mounts[pattern] = transport_cls(proxy=route.proxy_url, verify=route.verify_ssl) if route.proxy_url else None + # Configure specific routes + for pattern, route in self.transport_routes.items(): + mounts[pattern] = transport_cls( + verify=route.verify_ssl, proxy=route.proxy_url or self.proxy_url if route.proxy else None + ) - # Set default proxy if specified - if self.default_url: - mounts["all://"] = transport_cls(proxy=self.default_url) + # Set default proxy for all routes if enabled + if self.all_proxy: + mounts["all://"] = transport_cls(proxy=self.proxy_url) return mounts class Config: env_file = ".env" - env_prefix = "PROXY_" extra = "ignore" class Settings(BaseSettings): api_password: str # The password for accessing the API endpoints. - proxy_config: ProxyConfig = Field(default_factory=ProxyConfig) # Configuration for proxying requests. + transport_config: TransportConfig = Field(default_factory=TransportConfig) # Configuration for httpx transport. enable_streaming_progress: bool = False # Whether to enable streaming progress tracking. user_agent: str = ( diff --git a/mediaflow_proxy/extractors/base.py b/mediaflow_proxy/extractors/base.py index b9c1cc3..4cdd0f0 100644 --- a/mediaflow_proxy/extractors/base.py +++ b/mediaflow_proxy/extractors/base.py @@ -10,8 +10,7 @@ from mediaflow_proxy.utils.http_utils import create_httpx_client class BaseExtractor(ABC): """Base class for all URL extractors.""" - def __init__(self, proxy_enabled: bool, request_headers: dict): - self.proxy_url = settings.proxy_url if proxy_enabled else None + def __init__(self, request_headers: dict): self.base_headers = { "user-agent": settings.user_agent, "accept-language": "en-US,en;q=0.5", diff --git a/mediaflow_proxy/extractors/factory.py b/mediaflow_proxy/extractors/factory.py index 46ad441..f7a4f45 100644 --- a/mediaflow_proxy/extractors/factory.py +++ b/mediaflow_proxy/extractors/factory.py @@ -16,9 +16,9 @@ class ExtractorFactory: } @classmethod - def get_extractor(cls, host: str, proxy_enabled: bool, request_headers: dict) -> BaseExtractor: + def get_extractor(cls, host: str, request_headers: dict) -> BaseExtractor: """Get appropriate extractor instance for the given host.""" extractor_class = cls._extractors.get(host) if not extractor_class: raise ValueError(f"Unsupported host: {host}") - return extractor_class(proxy_enabled, request_headers) + return extractor_class(request_headers) diff --git a/mediaflow_proxy/routes/extractor.py b/mediaflow_proxy/routes/extractor.py index af8b273..cecd689 100644 --- a/mediaflow_proxy/routes/extractor.py +++ b/mediaflow_proxy/routes/extractor.py @@ -24,9 +24,7 @@ async def extract_url( ): """Extract clean links from various video hosting services.""" try: - extractor = ExtractorFactory.get_extractor( - extractor_params.host, extractor_params.use_request_proxy, proxy_headers.request - ) + extractor = ExtractorFactory.get_extractor(extractor_params.host, proxy_headers.request) final_url, headers = await extractor.extract(extractor_params.destination) if extractor_params.redirect_stream: diff --git a/mediaflow_proxy/speedtest/service.py b/mediaflow_proxy/speedtest/service.py index e0ae1c4..8f92222 100644 --- a/mediaflow_proxy/speedtest/service.py +++ b/mediaflow_proxy/speedtest/service.py @@ -3,11 +3,8 @@ import time from datetime import datetime, timezone from typing import Dict, Optional, Type -from httpx import AsyncClient - -from mediaflow_proxy.configs import settings -from mediaflow_proxy.utils.http_utils import Streamer from mediaflow_proxy.utils.cache_utils import get_cached_speedtest, set_cache_speedtest +from mediaflow_proxy.utils.http_utils import Streamer, create_httpx_client from .models import SpeedTestTask, LocationResult, SpeedTestResult, SpeedTestProvider from .providers.all_debrid import AllDebridSpeedTest from .providers.base import BaseSpeedTestProvider @@ -68,7 +65,7 @@ class SpeedTestService: provider_impl = self._get_provider(provider, api_key) config = await provider_impl.get_config() - async with AsyncClient(follow_redirects=True, timeout=10, proxy=settings.proxy_url) as client: + async with create_httpx_client() as client: streamer = Streamer(client) for location, url in config.test_urls.items(): diff --git a/mediaflow_proxy/utils/http_utils.py b/mediaflow_proxy/utils/http_utils.py index 06ecbac..5038fda 100644 --- a/mediaflow_proxy/utils/http_utils.py +++ b/mediaflow_proxy/utils/http_utils.py @@ -32,7 +32,7 @@ class DownloadError(Exception): def create_httpx_client(follow_redirects: bool = True, timeout: float = 30.0, **kwargs) -> httpx.AsyncClient: """Creates an HTTPX client with configured proxy routing""" - mounts = settings.proxy_config.get_mounts() + mounts = settings.transport_config.get_mounts() client = httpx.AsyncClient(mounts=mounts, follow_redirects=follow_redirects, timeout=timeout, **kwargs) return client