diff --git a/api/main.py b/api/main.py index 3b7be88..cd83d11 100644 --- a/api/main.py +++ b/api/main.py @@ -44,8 +44,8 @@ from utils.runtime_const import ( DELETE_ALL_META, DELETE_ALL_META_ITEM, TEMPLATES, - REDIS_ASYNC_CLIENT, ) +from db.redis_database import REDIS_ASYNC_CLIENT from utils.validation_helper import ( validate_mediaflow_proxy_credentials, validate_rpdb_token, diff --git a/api/middleware.py b/api/middleware.py index 2463349..92861bb 100644 --- a/api/middleware.py +++ b/api/middleware.py @@ -16,10 +16,10 @@ from starlette.middleware.base import BaseHTTPMiddleware from starlette.routing import Match from db.config import settings +from db.redis_database import REDIS_SYNC_CLIENT, REDIS_ASYNC_CLIENT from db.schemas import UserData from utils import crypto, const from utils.network import get_client_ip -from utils.runtime_const import REDIS_ASYNC_CLIENT, REDIS_SYNC_CLIENT async def find_route_handler(app, request: Request) -> Optional[Callable]: diff --git a/db/config.py b/db/config.py index b6addbc..4658cec 100644 --- a/db/config.py +++ b/db/config.py @@ -25,6 +25,7 @@ class Settings(BaseSettings): mongo_uri: str db_max_connections: int = 50 redis_url: str = "redis://redis-service:6379" + redis_max_connections: int = 50 # External Service URLs scraper_proxy_url: str | None = None diff --git a/db/crud.py b/db/crud.py index 2d7c488..b2282fb 100644 --- a/db/crud.py +++ b/db/crud.py @@ -37,7 +37,7 @@ from utils.parser import ( parse_stream_data, parse_tv_stream_data, ) -from utils.runtime_const import REDIS_ASYNC_CLIENT +from db.redis_database import REDIS_ASYNC_CLIENT from utils.validation_helper import ( validate_parent_guide_nudity, get_filter_certification_values, diff --git a/kodi/routes.py b/kodi/routes.py index 0dbb9f0..aa0c190 100644 --- a/kodi/routes.py +++ b/kodi/routes.py @@ -13,7 +13,7 @@ from starlette.responses import StreamingResponse from db.config import settings from db.schemas import KodiConfig from utils import const -from utils.runtime_const import REDIS_ASYNC_CLIENT +from db.redis_database import REDIS_ASYNC_CLIENT kodi_router = APIRouter() diff --git a/mediafusion_scrapy/custom_stats.py b/mediafusion_scrapy/custom_stats.py index 2ee59b9..5ab4ee0 100644 --- a/mediafusion_scrapy/custom_stats.py +++ b/mediafusion_scrapy/custom_stats.py @@ -4,7 +4,7 @@ import logging from scrapy import signals from scrapy.statscollectors import StatsCollector -from utils.runtime_const import REDIS_SYNC_CLIENT +from db.redis_database import REDIS_SYNC_CLIENT class RedisStatsCollector(StatsCollector): diff --git a/mediafusion_scrapy/pipelines/redis_cache_pipeline.py b/mediafusion_scrapy/pipelines/redis_cache_pipeline.py index 924363d..b03e0a9 100644 --- a/mediafusion_scrapy/pipelines/redis_cache_pipeline.py +++ b/mediafusion_scrapy/pipelines/redis_cache_pipeline.py @@ -1,7 +1,7 @@ from scrapy import signals from scrapy.exceptions import DropItem -from utils.runtime_const import REDIS_ASYNC_CLIENT +from db.redis_database import REDIS_ASYNC_CLIENT class RedisCacheURLPipeline: diff --git a/mediafusion_scrapy/pipelines/store_pipelines.py b/mediafusion_scrapy/pipelines/store_pipelines.py index 948044a..8939356 100644 --- a/mediafusion_scrapy/pipelines/store_pipelines.py +++ b/mediafusion_scrapy/pipelines/store_pipelines.py @@ -13,7 +13,7 @@ from db.models import ( MediaFusionSeriesMetaData, ) from db.schemas import TVMetaData -from utils.runtime_const import REDIS_ASYNC_CLIENT +from db.redis_database import REDIS_ASYNC_CLIENT class QueueBasedPipeline: diff --git a/mediafusion_scrapy/spiders/common.py b/mediafusion_scrapy/spiders/common.py index 41661b1..8b58950 100644 --- a/mediafusion_scrapy/spiders/common.py +++ b/mediafusion_scrapy/spiders/common.py @@ -4,7 +4,7 @@ import re import scrapy from utils.config import config_manager -from utils.runtime_const import REDIS_SYNC_CLIENT +from db.redis_database import REDIS_SYNC_CLIENT class CommonTamilSpider(scrapy.Spider): diff --git a/mediafusion_scrapy/spiders/sport_video.py b/mediafusion_scrapy/spiders/sport_video.py index 2738140..5153e7f 100644 --- a/mediafusion_scrapy/spiders/sport_video.py +++ b/mediafusion_scrapy/spiders/sport_video.py @@ -3,7 +3,7 @@ from urllib.parse import urlparse import scrapy from utils.config import config_manager -from utils.runtime_const import REDIS_SYNC_CLIENT +from db.redis_database import REDIS_SYNC_CLIENT class SportVideoSpider(scrapy.Spider): diff --git a/mediafusion_scrapy/spiders/streambtw.py b/mediafusion_scrapy/spiders/streambtw.py index 01ad701..ef7d4f8 100644 --- a/mediafusion_scrapy/spiders/streambtw.py +++ b/mediafusion_scrapy/spiders/streambtw.py @@ -3,7 +3,8 @@ import re import scrapy -from utils.runtime_const import SPORTS_ARTIFACTS, REDIS_SYNC_CLIENT +from utils.runtime_const import SPORTS_ARTIFACTS +from db.redis_database import REDIS_SYNC_CLIENT class StreamBTWSpider(scrapy.Spider): diff --git a/mediafusion_scrapy/spiders/tgx.py b/mediafusion_scrapy/spiders/tgx.py index 79aa86d..33f7f1c 100644 --- a/mediafusion_scrapy/spiders/tgx.py +++ b/mediafusion_scrapy/spiders/tgx.py @@ -9,7 +9,8 @@ from db.config import settings from db.models import TorrentStreams from utils.config import config_manager from utils.parser import convert_size_to_bytes -from utils.runtime_const import SPORTS_ARTIFACTS, REDIS_ASYNC_CLIENT +from utils.runtime_const import SPORTS_ARTIFACTS +from db.redis_database import REDIS_ASYNC_CLIENT from utils.torrent import parse_magnet diff --git a/scrapers/base_scraper.py b/scrapers/base_scraper.py index ea5bff6..4c776bd 100644 --- a/scrapers/base_scraper.py +++ b/scrapers/base_scraper.py @@ -15,7 +15,7 @@ from tenacity import retry, stop_after_attempt, wait_exponential from db.models import TorrentStreams, MediaFusionMetaData from utils.parser import calculate_max_similarity_ratio -from utils.runtime_const import REDIS_ASYNC_CLIENT +from db.redis_database import REDIS_ASYNC_CLIENT @dataclass diff --git a/scrapers/prowlarr.py b/scrapers/prowlarr.py index d047279..0c0401c 100644 --- a/scrapers/prowlarr.py +++ b/scrapers/prowlarr.py @@ -21,7 +21,8 @@ from scrapers.base_scraper import BaseScraper from scrapers.imdb_data import get_episode_by_date, get_season_episodes from utils.network import CircuitBreaker, batch_process_with_circuit_breaker from utils.parser import is_contain_18_plus_keywords -from utils.runtime_const import REDIS_ASYNC_CLIENT, PROWLARR_SEARCH_TTL +from utils.runtime_const import PROWLARR_SEARCH_TTL +from db.redis_database import REDIS_ASYNC_CLIENT from utils.torrent import extract_torrent_metadata from utils.wrappers import minimum_run_interval diff --git a/scrapers/prowlarr_feed.py b/scrapers/prowlarr_feed.py index b1357c3..7d1dde8 100644 --- a/scrapers/prowlarr_feed.py +++ b/scrapers/prowlarr_feed.py @@ -20,7 +20,7 @@ from scrapers.prowlarr import ( from utils.crypto import get_text_hash from utils.network import CircuitBreaker, batch_process_with_circuit_breaker from utils.parser import is_contain_18_plus_keywords -from utils.runtime_const import REDIS_ASYNC_CLIENT +from db.redis_database import REDIS_ASYNC_CLIENT from utils.wrappers import minimum_run_interval logger = logging.getLogger(__name__) diff --git a/scrapers/routes.py b/scrapers/routes.py index 5fbc66a..fbf75fe 100644 --- a/scrapers/routes.py +++ b/scrapers/routes.py @@ -24,7 +24,8 @@ from scrapers.tv import add_tv_metadata, parse_m3u_playlist from utils import const, torrent from utils.network import get_request_namespace from utils.parser import calculate_max_similarity_ratio -from utils.runtime_const import TEMPLATES, REDIS_ASYNC_CLIENT +from utils.runtime_const import TEMPLATES +from db.redis_database import REDIS_ASYNC_CLIENT router = APIRouter() diff --git a/scrapers/rpdb.py b/scrapers/rpdb.py index 8c32101..1708c7e 100644 --- a/scrapers/rpdb.py +++ b/scrapers/rpdb.py @@ -5,7 +5,7 @@ import time import httpx from db import schemas -from utils.runtime_const import REDIS_ASYNC_CLIENT +from db.redis_database import REDIS_ASYNC_CLIENT RPDB_SUPPORTED_SET = "rpdb_supported_ids" RPDB_UNSUPPORTED_HASH = "rpdb_unsupported_ids" diff --git a/scrapers/tv.py b/scrapers/tv.py index 51eefac..dc7bd4c 100644 --- a/scrapers/tv.py +++ b/scrapers/tv.py @@ -14,7 +14,7 @@ from db import schemas, crud from db.models import TVStreams, MediaFusionTVMetaData from utils import validation_helper from utils.parser import is_contain_18_plus_keywords -from utils.runtime_const import REDIS_ASYNC_CLIENT +from db.redis_database import REDIS_ASYNC_CLIENT from utils.validation_helper import validate_live_stream_url diff --git a/streaming_providers/cache_helpers.py b/streaming_providers/cache_helpers.py index 3682aae..060ecbc 100644 --- a/streaming_providers/cache_helpers.py +++ b/streaming_providers/cache_helpers.py @@ -2,7 +2,7 @@ import logging from datetime import datetime, timedelta, timezone from typing import List, Dict -from utils.runtime_const import REDIS_ASYNC_CLIENT +from db.redis_database import REDIS_ASYNC_CLIENT # Constants CACHE_KEY_PREFIX = "debrid_cache:" diff --git a/streaming_providers/pikpak/utils.py b/streaming_providers/pikpak/utils.py index a2a554a..c99fc80 100644 --- a/streaming_providers/pikpak/utils.py +++ b/streaming_providers/pikpak/utils.py @@ -12,7 +12,7 @@ from db.schemas import UserData from streaming_providers.exceptions import ProviderException from streaming_providers.parser import select_file_index_from_torrent from utils import crypto -from utils.runtime_const import REDIS_ASYNC_CLIENT +from db.redis_database import REDIS_ASYNC_CLIENT async def get_torrent_file_by_info_hash( diff --git a/streaming_providers/routes.py b/streaming_providers/routes.py index e7e51bd..caf1781 100644 --- a/streaming_providers/routes.py +++ b/streaming_providers/routes.py @@ -24,7 +24,7 @@ from streaming_providers.seedr.api import router as seedr_router from utils import crypto, torrent, wrappers, const from utils.lock import acquire_redis_lock, release_redis_lock from utils.network import get_user_public_ip, get_user_data, encode_mediaflow_proxy_url -from utils.runtime_const import REDIS_ASYNC_CLIENT +from db.redis_database import REDIS_ASYNC_CLIENT # Seconds until when the Video URLs are cached URL_CACHE_EXP = 3600 diff --git a/utils/config.py b/utils/config.py index 76c619e..fe2b912 100644 --- a/utils/config.py +++ b/utils/config.py @@ -6,7 +6,7 @@ from typing import Any, Dict, Optional import httpx from db.config import settings -from utils.runtime_const import REDIS_SYNC_CLIENT +from db.redis_database import REDIS_SYNC_CLIENT logger = logging.getLogger(__name__) diff --git a/utils/lock.py b/utils/lock.py index 6c21de8..c6a951d 100644 --- a/utils/lock.py +++ b/utils/lock.py @@ -5,7 +5,7 @@ import time from redis.asyncio import Redis from redis.exceptions import LockNotOwnedError -from utils.runtime_const import REDIS_ASYNC_CLIENT +from db.redis_database import REDIS_ASYNC_CLIENT scheduler_lock_key = "mediafusion_scheduler_lock" heartbeat_key = "mediafusion_scheduler_heartbeat" diff --git a/utils/network.py b/utils/network.py index 2b72988..bf5bed3 100644 --- a/utils/network.py +++ b/utils/network.py @@ -10,7 +10,8 @@ from fastapi.requests import Request from db.schemas import UserData from utils import crypto from utils.crypto import encrypt_data -from utils.runtime_const import PRIVATE_CIDR, REDIS_ASYNC_CLIENT +from utils.runtime_const import PRIVATE_CIDR +from db.redis_database import REDIS_ASYNC_CLIENT class CircuitBreakerOpenException(Exception): diff --git a/utils/poster.py b/utils/poster.py index 6a28131..a52d2b0 100644 --- a/utils/poster.py +++ b/utils/poster.py @@ -10,7 +10,7 @@ from imdb import Cinemagoer from db.models import MediaFusionMetaData from scrapers.imdb_data import get_imdb_rating from utils import const -from utils.runtime_const import REDIS_ASYNC_CLIENT +from db.redis_database import REDIS_ASYNC_CLIENT ia = Cinemagoer() font_cache = {} diff --git a/utils/runtime_const.py b/utils/runtime_const.py index caf37e5..2905fd5 100644 --- a/utils/runtime_const.py +++ b/utils/runtime_const.py @@ -2,7 +2,6 @@ import re from datetime import timedelta from fastapi.templating import Jinja2Templates -import redis from db import schemas from db.config import settings @@ -35,16 +34,6 @@ TRACKERS = get_json_data("resources/json/trackers.json") SECRET_KEY = settings.secret_key.encode("utf-8") - -REDIS_SYNC_CLIENT: redis.Redis = redis.Redis( - connection_pool=redis.ConnectionPool.from_url(settings.redis_url) -) - -REDIS_ASYNC_CLIENT: redis.asyncio.Redis = redis.asyncio.Redis( - connection_pool=redis.asyncio.ConnectionPool.from_url(settings.redis_url) -) - - PROWLARR_SEARCH_TTL = int( timedelta(hours=settings.prowlarr_search_interval_hour).total_seconds() ) diff --git a/utils/validation_helper.py b/utils/validation_helper.py index 17ad699..7e81de2 100644 --- a/utils/validation_helper.py +++ b/utils/validation_helper.py @@ -7,7 +7,8 @@ import httpx from db import schemas from utils import const -from utils.runtime_const import REDIS_ASYNC_CLIENT, PRIVATE_CIDR +from utils.runtime_const import PRIVATE_CIDR +from db.redis_database import REDIS_ASYNC_CLIENT def is_valid_url(url: str) -> bool: