Viren070/MediaFusion
1
0
Fork 0
mirror of https://github.com/Viren070/MediaFusion.git synced 2025-12-01 23:21:11 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactoring and bugfixes

Browse Source
This commit is contained in:
mhdzumair
2024-02-08 12:55:11 +05:30
parent 54ce758495
commit 95db6bf207
6 changed files with 69 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/main.py
+3 -12
View File
@@ -85,24 +85,15 @@ async def start_scheduler():
scheduler.add_job(
crud.delete_search_history, CronTrigger(day="*/1"), name="delete_search_history"
)
if settings.enable_scrapper:
scheduler.add_job(
tamil_blasters.run_schedule_scrape,
CronTrigger(hour="*/6"),
name="tamil_blasters",
)
scheduler.add_job(
tamilmv.run_schedule_scrape, CronTrigger(hour="*/3"), name="tamilmv"
)
scheduler.start()
app.state.scheduler = scheduler
@app.on_event("shutdown")
async def stop_scheduler():
if settings.enable_scrapper:
try:
app.state.scheduler.shutdown(wait=False)
except AttributeError:
pass
@app.on_event("shutdown")
db/config.py
+2 -2
View File
@@ -8,11 +8,11 @@ class Settings(BaseSettings):
secret_key: str
host_url: str = "https://mediafusion.fun"
logging_level: str = "INFO"
enable_scrapper: bool = False
enable_search_scrapper: bool = False
enable_tamilmv_search_scrapper: bool = False
scrapper_proxy_url: str | None = None
torrentio_url: str = "https://torrentio.strem.fun"
is_scrap_from_torrentio: bool = False
torrentio_search_interval_days: int = 3
premiumize_oauth_client_id: str | None = None
premiumize_oauth_client_secret: str | None = None
prowlarr_url: str = "http://prowlarr-service:9696"
db/crud.py
+20 -13
View File
@@ -129,13 +129,16 @@ async def get_movie_streams(
.to_list()
)
if settings.is_scrap_from_torrentio:
streams = await get_streams_from_torrentio(
user_data, streams, video_id, "movie"
)
if settings.prowlarr_api_key:
if video_id.startswith("tt"):
if (
video_id.startswith("tt")
settings.is_scrap_from_torrentio
and "torrentio_streams" in user_data.selected_catalogs
):
streams = await get_streams_from_torrentio(
redis, streams, video_id, "movie"
)
if (
settings.prowlarr_api_key
and "prowlarr_streams" in user_data.selected_catalogs
):
movie_metadata = await get_movie_data_by_id(video_id, False)
@@ -168,13 +171,17 @@ async def get_series_streams(
.to_list()
)
if settings.is_scrap_from_torrentio:
streams = await get_streams_from_torrentio(
user_data, streams, video_id, "series", season, episode
)
if settings.prowlarr_api_key:
if video_id.startswith("tt"):
if (
video_id.startswith("tt")
settings.is_scrap_from_torrentio
and "torrentio_streams" in user_data.selected_catalogs
):
streams = await get_streams_from_torrentio(
redis, streams, video_id, "series", season, episode
)
if (
settings.prowlarr_api_key
and "prowlarr_streams" in user_data.selected_catalogs
):
series_metadata = await get_series_data_by_id(video_id, False)
@@ -464,7 +471,7 @@ async def save_series_metadata(metadata: dict):
async def process_search_query(search_query: str, catalog_type: str) -> dict:
if settings.enable_search_scrapper and catalog_type in ["movie", "series"]:
if settings.enable_tamilmv_search_scrapper and catalog_type in ["movie", "series"]:
# check if the search query is already searched in for last 24 hours or not
last_search = await models.SearchHistory.find_one(
{
scrappers/helpers.py
+6
View File
@@ -130,6 +130,9 @@ def get_scrapper_config(site_name: str, get_key: str) -> dict:
@dramatiq.actor(time_limit=30 * 60 * 1000)
async def update_torrent_movie_streams_metadata(info_hashes: list[str]):
"""Update torrent streams metadata."""
if not info_hashes:
return
streams_metadata = await info_hashes_to_torrent_metadata(info_hashes, [])
for stream_metadata in streams_metadata:
@@ -151,6 +154,9 @@ async def update_torrent_movie_streams_metadata(info_hashes: list[str]):
@dramatiq.actor(time_limit=30 * 60 * 1000)
async def update_torrent_series_streams_metadata(info_hashes: list[str]):
"""Update torrent streams metadata."""
if not info_hashes:
return
streams_metadata = await info_hashes_to_torrent_metadata(info_hashes, [])
for stream_metadata in streams_metadata:
scrappers/prowlarr.py
+12 -4
View File
@@ -7,7 +7,7 @@ import dramatiq
import httpx
from redis.asyncio import Redis
from thefuzz import fuzz
from torf import Magnet
from torf import Magnet, MagnetError
from db.config import settings
from db.models import TorrentStreams, Season, Episode
@@ -240,7 +240,13 @@ async def prowlarr_data_parser(meta_data: dict) -> tuple[dict, bool]:
)
except Exception as e:
if meta_data.get("magnetUrl"):
magnet = Magnet.from_string(meta_data.get("magnetUrl"))
try:
magnet = Magnet.from_string(meta_data.get("magnetUrl"))
except MagnetError:
logging.error(
f"Error parsing {meta_data.get('indexer')} magnet link: {meta_data.get('magnetUrl')}",
)
return {}, False
torrent_data = {
"info_hash": magnet.infohash,
"announce_list": magnet.tr,
@@ -518,7 +524,8 @@ async def parse_and_store_movie_stream_data(
]
# Continue to use background tasks for demagnetization
update_torrent_movie_streams_metadata.send(info_hashes)
if info_hashes:
update_torrent_movie_streams_metadata.send(info_hashes)
return streams
@@ -563,7 +570,8 @@ async def parse_and_store_series_stream_data(
]
# Use background tasks for metadata updates
update_torrent_series_streams_metadata.send(info_hashes)
if info_hashes:
update_torrent_series_streams_metadata.send(info_hashes)
return streams
scrappers/torrentio.py
+26 -19
View File
@@ -5,6 +5,7 @@ from os import path
import PTN
import httpx
from redis.asyncio import Redis
from db.config import settings
from db.models import TorrentStreams, Season, Episode
@@ -19,30 +20,34 @@ from utils.validation_helper import is_video_file
async def get_streams_from_torrentio(
user_data: UserData,
redis: Redis,
streams: list[TorrentStreams],
video_id: str,
catalog_type: str,
season: int = None,
episode: int = None,
):
if video_id.startswith("tt") and "torrentio_streams" in user_data.selected_catalogs:
last_stream = next(
(stream for stream in streams if stream.source == "Torrentio"), None
cache_key = f"{catalog_type}_{video_id}_{season}_{episode}_torrentio_streams"
cached_data = await redis.get(cache_key)
if cached_data:
return streams
if catalog_type == "movie":
streams.extend(await scrap_movie_streams_from_torrentio(video_id, catalog_type))
elif catalog_type == "series":
streams.extend(
await scrap_series_streams_from_torrentio(
video_id, catalog_type, season, episode
)
)
if last_stream is None or last_stream.updated_at < datetime.now() - timedelta(
days=3
):
if catalog_type == "movie":
streams.extend(
await scrap_movie_streams_from_torrentio(video_id, catalog_type)
)
elif catalog_type == "series":
streams.extend(
await scrap_series_streams_from_torrentio(
video_id, catalog_type, season, episode
)
)
# Cache the data for 24 hours
await redis.set(
cache_key,
"True",
ex=int(timedelta(days=settings.torrentio_search_interval_days).total_seconds()),
)
return streams
@@ -156,7 +161,8 @@ async def store_and_parse_movie_stream_data(
if torrent_stream.filename is None:
info_hashes.append(stream["infoHash"])
update_torrent_movie_streams_metadata.send(info_hashes)
if info_hashes:
update_torrent_movie_streams_metadata.send(info_hashes)
return streams
@@ -255,7 +261,8 @@ async def store_and_parse_series_stream_data(
if episode_item and episode_item.size is None:
info_hashes.append(stream["infoHash"])
update_torrent_series_streams_metadata.send(info_hashes)
if info_hashes:
update_torrent_series_streams_metadata.send(info_hashes)
return streams