Refactor Redis client imports and configuration

Moved Redis client setup to a new module `redis_database` and updated all references across the codebase. Also added a new configuration setting for redis_max_connections for better resource management.

api/main.py

@@ -44,8 +44,8 @@ from utils.runtime_const import (
     DELETE_ALL_META,
     DELETE_ALL_META_ITEM,
     TEMPLATES,
-    REDIS_ASYNC_CLIENT,
 )
+from db.redis_database import REDIS_ASYNC_CLIENT
 from utils.validation_helper import (
     validate_mediaflow_proxy_credentials,
     validate_rpdb_token,

api/middleware.py

@@ -16,10 +16,10 @@ from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.routing import Match
 
 from db.config import settings
+from db.redis_database import REDIS_SYNC_CLIENT, REDIS_ASYNC_CLIENT
 from db.schemas import UserData
 from utils import crypto, const
 from utils.network import get_client_ip
-from utils.runtime_const import REDIS_ASYNC_CLIENT, REDIS_SYNC_CLIENT
 
 
 async def find_route_handler(app, request: Request) -> Optional[Callable]:

db/config.py

@@ -25,6 +25,7 @@ class Settings(BaseSettings):
     mongo_uri: str
     db_max_connections: int = 50
     redis_url: str = "redis://redis-service:6379"
+    redis_max_connections: int = 50
 
     # External Service URLs
     scraper_proxy_url: str | None = None

db/crud.py

@@ -37,7 +37,7 @@ from utils.parser import (
     parse_stream_data,
     parse_tv_stream_data,
 )
-from utils.runtime_const import REDIS_ASYNC_CLIENT
+from db.redis_database import REDIS_ASYNC_CLIENT
 from utils.validation_helper import (
     validate_parent_guide_nudity,
     get_filter_certification_values,

kodi/routes.py

@@ -13,7 +13,7 @@ from starlette.responses import StreamingResponse
 from db.config import settings
 from db.schemas import KodiConfig
 from utils import const
-from utils.runtime_const import REDIS_ASYNC_CLIENT
+from db.redis_database import REDIS_ASYNC_CLIENT
 
 kodi_router = APIRouter()
 

mediafusion_scrapy/custom_stats.py

@@ -4,7 +4,7 @@ import logging
 from scrapy import signals
 from scrapy.statscollectors import StatsCollector
 
-from utils.runtime_const import REDIS_SYNC_CLIENT
+from db.redis_database import REDIS_SYNC_CLIENT
 
 
 class RedisStatsCollector(StatsCollector):

mediafusion_scrapy/pipelines/redis_cache_pipeline.py

@@ -1,7 +1,7 @@
 from scrapy import signals
 from scrapy.exceptions import DropItem
 
-from utils.runtime_const import REDIS_ASYNC_CLIENT
+from db.redis_database import REDIS_ASYNC_CLIENT
 
 
 class RedisCacheURLPipeline:

mediafusion_scrapy/pipelines/store_pipelines.py

@@ -13,7 +13,7 @@ from db.models import (
     MediaFusionSeriesMetaData,
 )
 from db.schemas import TVMetaData
-from utils.runtime_const import REDIS_ASYNC_CLIENT
+from db.redis_database import REDIS_ASYNC_CLIENT
 
 
 class QueueBasedPipeline:

mediafusion_scrapy/spiders/common.py

@@ -4,7 +4,7 @@ import re
 import scrapy
 
 from utils.config import config_manager
-from utils.runtime_const import REDIS_SYNC_CLIENT
+from db.redis_database import REDIS_SYNC_CLIENT
 
 
 class CommonTamilSpider(scrapy.Spider):

mediafusion_scrapy/spiders/sport_video.py

@@ -3,7 +3,7 @@ from urllib.parse import urlparse
 import scrapy
 
 from utils.config import config_manager
-from utils.runtime_const import REDIS_SYNC_CLIENT
+from db.redis_database import REDIS_SYNC_CLIENT
 
 
 class SportVideoSpider(scrapy.Spider):

mediafusion_scrapy/spiders/streambtw.py

@@ -3,7 +3,8 @@ import re
 
 import scrapy
 
-from utils.runtime_const import SPORTS_ARTIFACTS, REDIS_SYNC_CLIENT
+from utils.runtime_const import SPORTS_ARTIFACTS
+from db.redis_database import REDIS_SYNC_CLIENT
 
 
 class StreamBTWSpider(scrapy.Spider):

mediafusion_scrapy/spiders/tgx.py

@@ -9,7 +9,8 @@ from db.config import settings
 from db.models import TorrentStreams
 from utils.config import config_manager
 from utils.parser import convert_size_to_bytes
-from utils.runtime_const import SPORTS_ARTIFACTS, REDIS_ASYNC_CLIENT
+from utils.runtime_const import SPORTS_ARTIFACTS
+from db.redis_database import REDIS_ASYNC_CLIENT
 from utils.torrent import parse_magnet
 
 

scrapers/base_scraper.py

@@ -15,7 +15,7 @@ from tenacity import retry, stop_after_attempt, wait_exponential
 
 from db.models import TorrentStreams, MediaFusionMetaData
 from utils.parser import calculate_max_similarity_ratio
-from utils.runtime_const import REDIS_ASYNC_CLIENT
+from db.redis_database import REDIS_ASYNC_CLIENT
 
 
 @dataclass

scrapers/prowlarr.py

@@ -21,7 +21,8 @@ from scrapers.base_scraper import BaseScraper
 from scrapers.imdb_data import get_episode_by_date, get_season_episodes
 from utils.network import CircuitBreaker, batch_process_with_circuit_breaker
 from utils.parser import is_contain_18_plus_keywords
-from utils.runtime_const import REDIS_ASYNC_CLIENT, PROWLARR_SEARCH_TTL
+from utils.runtime_const import PROWLARR_SEARCH_TTL
+from db.redis_database import REDIS_ASYNC_CLIENT
 from utils.torrent import extract_torrent_metadata
 from utils.wrappers import minimum_run_interval
 

scrapers/prowlarr_feed.py

@@ -20,7 +20,7 @@ from scrapers.prowlarr import (
 from utils.crypto import get_text_hash
 from utils.network import CircuitBreaker, batch_process_with_circuit_breaker
 from utils.parser import is_contain_18_plus_keywords
-from utils.runtime_const import REDIS_ASYNC_CLIENT
+from db.redis_database import REDIS_ASYNC_CLIENT
 from utils.wrappers import minimum_run_interval
 
 logger = logging.getLogger(__name__)

scrapers/routes.py

@@ -24,7 +24,8 @@ from scrapers.tv import add_tv_metadata, parse_m3u_playlist
 from utils import const, torrent
 from utils.network import get_request_namespace
 from utils.parser import calculate_max_similarity_ratio
-from utils.runtime_const import TEMPLATES, REDIS_ASYNC_CLIENT
+from utils.runtime_const import TEMPLATES
+from db.redis_database import REDIS_ASYNC_CLIENT
 
 router = APIRouter()
 

scrapers/rpdb.py

@@ -5,7 +5,7 @@ import time
 import httpx
 
 from db import schemas
-from utils.runtime_const import REDIS_ASYNC_CLIENT
+from db.redis_database import REDIS_ASYNC_CLIENT
 
 RPDB_SUPPORTED_SET = "rpdb_supported_ids"
 RPDB_UNSUPPORTED_HASH = "rpdb_unsupported_ids"

scrapers/tv.py

@@ -14,7 +14,7 @@ from db import schemas, crud
 from db.models import TVStreams, MediaFusionTVMetaData
 from utils import validation_helper
 from utils.parser import is_contain_18_plus_keywords
-from utils.runtime_const import REDIS_ASYNC_CLIENT
+from db.redis_database import REDIS_ASYNC_CLIENT
 from utils.validation_helper import validate_live_stream_url
 
 

streaming_providers/cache_helpers.py

@@ -2,7 +2,7 @@ import logging
 from datetime import datetime, timedelta, timezone
 from typing import List, Dict
 
-from utils.runtime_const import REDIS_ASYNC_CLIENT
+from db.redis_database import REDIS_ASYNC_CLIENT
 
 # Constants
 CACHE_KEY_PREFIX = "debrid_cache:"

streaming_providers/pikpak/utils.py

@@ -12,7 +12,7 @@ from db.schemas import UserData
 from streaming_providers.exceptions import ProviderException
 from streaming_providers.parser import select_file_index_from_torrent
 from utils import crypto
-from utils.runtime_const import REDIS_ASYNC_CLIENT
+from db.redis_database import REDIS_ASYNC_CLIENT
 
 
 async def get_torrent_file_by_info_hash(

streaming_providers/routes.py

@@ -24,7 +24,7 @@ from streaming_providers.seedr.api import router as seedr_router
 from utils import crypto, torrent, wrappers, const
 from utils.lock import acquire_redis_lock, release_redis_lock
 from utils.network import get_user_public_ip, get_user_data, encode_mediaflow_proxy_url
-from utils.runtime_const import REDIS_ASYNC_CLIENT
+from db.redis_database import REDIS_ASYNC_CLIENT
 
 # Seconds until when the Video URLs are cached
 URL_CACHE_EXP = 3600

utils/config.py

@@ -6,7 +6,7 @@ from typing import Any, Dict, Optional
 import httpx
 
 from db.config import settings
-from utils.runtime_const import REDIS_SYNC_CLIENT
+from db.redis_database import REDIS_SYNC_CLIENT
 
 logger = logging.getLogger(__name__)
 

utils/lock.py

@@ -5,7 +5,7 @@ import time
 from redis.asyncio import Redis
 from redis.exceptions import LockNotOwnedError
 
-from utils.runtime_const import REDIS_ASYNC_CLIENT
+from db.redis_database import REDIS_ASYNC_CLIENT
 
 scheduler_lock_key = "mediafusion_scheduler_lock"
 heartbeat_key = "mediafusion_scheduler_heartbeat"

utils/network.py

@@ -10,7 +10,8 @@ from fastapi.requests import Request
 from db.schemas import UserData
 from utils import crypto
 from utils.crypto import encrypt_data
-from utils.runtime_const import PRIVATE_CIDR, REDIS_ASYNC_CLIENT
+from utils.runtime_const import PRIVATE_CIDR
+from db.redis_database import REDIS_ASYNC_CLIENT
 
 
 class CircuitBreakerOpenException(Exception):

utils/poster.py

@@ -10,7 +10,7 @@ from imdb import Cinemagoer
 from db.models import MediaFusionMetaData
 from scrapers.imdb_data import get_imdb_rating
 from utils import const
-from utils.runtime_const import REDIS_ASYNC_CLIENT
+from db.redis_database import REDIS_ASYNC_CLIENT
 
 ia = Cinemagoer()
 font_cache = {}

utils/runtime_const.py

@@ -2,7 +2,6 @@ import re
 from datetime import timedelta
 
 from fastapi.templating import Jinja2Templates
-import redis
 
 from db import schemas
 from db.config import settings
@@ -35,16 +34,6 @@ TRACKERS = get_json_data("resources/json/trackers.json")
 
 SECRET_KEY = settings.secret_key.encode("utf-8")
 
-
-REDIS_SYNC_CLIENT: redis.Redis = redis.Redis(
-    connection_pool=redis.ConnectionPool.from_url(settings.redis_url)
-)
-
-REDIS_ASYNC_CLIENT: redis.asyncio.Redis = redis.asyncio.Redis(
-    connection_pool=redis.asyncio.ConnectionPool.from_url(settings.redis_url)
-)
-
-
 PROWLARR_SEARCH_TTL = int(
     timedelta(hours=settings.prowlarr_search_interval_hour).total_seconds()
 )

utils/validation_helper.py

@@ -7,7 +7,8 @@ import httpx
 
 from db import schemas
 from utils import const
-from utils.runtime_const import REDIS_ASYNC_CLIENT, PRIVATE_CIDR
+from utils.runtime_const import PRIVATE_CIDR
+from db.redis_database import REDIS_ASYNC_CLIENT
 
 
 def is_valid_url(url: str) -> bool: