Viren070/MediaFusion
1
0
Fork 0
mirror of https://github.com/Viren070/MediaFusion.git synced 2025-12-01 23:21:11 +01:00
Code Issues Packages Projects Releases Wiki Activity

Remove ScrapeOps integration

Browse Source
This commit is contained in:
mhdzumair
2024-11-01 16:31:58 +05:30
parent 98efbe59a0
commit fde61f7dfd
9 changed files with 2 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Pipfile
-2
View File
@@ -46,8 +46,6 @@ parsett = "*"
tenacity = "*"
ratelimit = "*"
qrcode = "*"
scrapeops-scrapy = "*"
scrapeops-python-requests = "*"
aioseedrcc = "*"
[dev-packages]
Pipfile.lock
Generated
+1 -40
View File
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "60522a8f5bb55e1ea28080bf09231118086caab9be5a91e6a4b15335f57142d0"
"sha256": "1bfd1eeb0ef28abd4907e5b1e1046ab6912f9b37f4b1b66001a91d41d2f2a115"
},
"pipfile-spec": 6,
"requires": {
@@ -1066,14 +1066,6 @@
"markers": "python_version >= '3.7'",
"version": "==1.0.1"
},
"json5": {
"hashes": [
"sha256:34ed7d834b1341a86987ed52f3f76cd8ee184394906b6e22a1e0deb9ab294e8f",
"sha256:548e41b9be043f9426776f05df8635a00fe06104ea51ed24b67f908856e151ae"
],
"markers": "python_version >= '3.8'",
"version": "==0.9.25"
},
"jsonschema": {
"hashes": [
"sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4",
@@ -2456,29 +2448,6 @@
"markers": "python_version >= '3.8'",
"version": "==0.20.1"
},
"scrapeops-python-logger": {
"hashes": [
"sha256:faf58d35a6f043e90cd9680c070046f6617ceff725528ab66a98657c7cdef8f3"
],
"markers": "python_version >= '3.6'",
"version": "==0.4.7"
},
"scrapeops-python-requests": {
"hashes": [
"sha256:2c7b6e2ecd7690efff1f0b0f506df1f189f07534bf9f9eb6f5b8e8cf604e5198"
],
"index": "pypi",
"markers": "python_version >= '3.6'",
"version": "==0.4.7"
},
"scrapeops-scrapy": {
"hashes": [
"sha256:42a78c2e48c395895a2145d57ee912dbdf14c675738187fa08c191fb637f6a2a"
],
"index": "pypi",
"markers": "python_version >= '3.8'",
"version": "==0.5.6"
},
"scrapy": {
"hashes": [
"sha256:4be353d6abbb942a9f7e7614ca8b5f3d9037381176ac8d8859c8cac676e74fa0",
@@ -2626,14 +2595,6 @@
"markers": "python_version >= '3.8'",
"version": "==0.22.1"
},
"tld": {
"hashes": [
"sha256:93dde5e1c04bdf1844976eae440706379d21f4ab235b73c05d7483e074fb5629",
"sha256:f75b2be080f767ed17c2338a339eaa4fab5792586319ca819119da252f9f3749"
],
"markers": "python_version >= '3.7' and python_version < '4'",
"version": "==0.13"
},
"tldextract": {
"hashes": [
"sha256:4dfc4c277b6b97fa053899fcdb892d2dc27295851ab5fac4e07797b6a21b2e46",
db/config.py
-3
View File
@@ -31,9 +31,6 @@ class Settings(BaseSettings):
playwright_cdp_url: str = "ws://browserless:3000?blockAds=true&stealth=true"
flaresolverr_url: str = "http://flaresolverr:8191/v1"
# External Service API Keys
scrapeops_api_key: str | None = None
# Prowlarr Settings
prowlarr_url: str = "http://prowlarr-service:9696"
prowlarr_api_key: str | None = None
docs/configuration.md
-3
View File
@@ -36,9 +36,6 @@ These URLs define the locations of various external services used by MediaFusion
- **playwright_cdp_url** (default: `"ws://browserless:3000?blockAds=true&stealth=true"`): The URL for the Playwright CDP (Chrome DevTools Protocol) service.
- **flaresolverr_url** (default: `"http://flaresolverr:8191/v1"`): The URL for the FlareSolverr service.
## External Service API Keys
- **scrapeops_api_key** (Optional): The API key for the ScrapeOps monitoring service.
## Prowlarr Settings
These settings are specific to the Prowlarr integration.
mediafusion_scrapy/settings.py
-14
View File
@@ -62,20 +62,6 @@ EXTENSIONS = {
"mediafusion_scrapy.extensions.InactivityMonitor": 100,
}
if settings.scrapeops_api_key:
SCRAPEOPS_API_KEY = settings.scrapeops_api_key
DOWNLOADER_MIDDLEWARES.update(
{
"scrapeops_scrapy.middleware.retry.RetryMiddleware": 550,
"scrapy.downloadermiddlewares.retry.RetryMiddleware": None,
}
)
EXTENSIONS.update(
{
"scrapeops_scrapy.extension.ScrapeOpsMonitor": 500, # ScrapeOps Monitor
}
)
# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# ITEM_PIPELINES = {
scrapers/prowlarr.py
-49
View File
@@ -5,7 +5,6 @@ from typing import List, Dict, Any, AsyncGenerator, Literal, AsyncIterable
import PTT
import dramatiq
import httpx
from scrapeops_python_requests.scrapeops_requests import ScrapeOpsRequests
from torf import Magnet, MagnetError
from db.config import settings
@@ -72,8 +71,6 @@ class ProwlarrScraper(BaseScraper):
cache_key_prefix=self.cache_key_prefix, logger_name=self.__class__.__name__
)
self.base_url = f"{settings.prowlarr_url}/api/v1/search"
self.scrapeops_logger = None
self.scrape_response = None
@BaseScraper.cache(ttl=PROWLARR_SEARCH_TTL)
@BaseScraper.rate_limit(calls=5, period=timedelta(seconds=1))
@@ -90,13 +87,6 @@ class ProwlarrScraper(BaseScraper):
if catalog_type == "series":
job_name += f":{season}:{episode}"
if settings.scrapeops_api_key:
self.scrapeops_logger = ScrapeOpsRequests(
scrapeops_api_key=settings.scrapeops_api_key,
spider_name="Prowlarr Scraper",
job_name=job_name,
)
try:
async for stream in self._scrape_and_parse(
processed_info_hashes,
@@ -106,11 +96,6 @@ class ProwlarrScraper(BaseScraper):
episode,
):
results.append(stream)
if settings.scrapeops_api_key:
self.scrapeops_logger.item_scraped(
item=stream.model_dump(include={"id"}),
response=self.scrape_response,
)
except httpx.ReadTimeout:
self.logger.warning("Timeout while fetching search results")
except httpx.HTTPStatusError as e:
@@ -119,9 +104,6 @@ class ProwlarrScraper(BaseScraper):
)
except Exception as e:
self.logger.exception(f"An error occurred during scraping: {str(e)}")
finally:
if settings.scrapeops_api_key:
self.scrapeops_logger.logger.close_sdk()
self.logger.info(
f"Returning {len(results)} scraped streams for {metadata.title}"
@@ -329,7 +311,6 @@ class ProwlarrScraper(BaseScraper):
params=params,
timeout=settings.prowlarr_search_query_timeout,
)
self.scrape_response = response
response.raise_for_status()
return response.json()
@@ -753,13 +734,6 @@ async def background_movie_title_search(
if not metadata:
return
if settings.scrapeops_api_key:
scraper.scrapeops_logger = ScrapeOpsRequests(
scrapeops_api_key=settings.scrapeops_api_key,
spider_name="Prowlarr Scraper",
job_name=f"background:{metadata.title}:{metadata.id}",
)
title_streams_generators = [
scraper.scrape_movie_by_title(
processed_info_hashes,
@@ -772,11 +746,6 @@ async def background_movie_title_search(
try:
async for stream in scraper.process_streams(*title_streams_generators):
await scraper.store_streams([stream])
if settings.scrapeops_api_key:
scraper.scrapeops_logger.item_scraped(
item=stream.model_dump(include={"id"}),
response=scraper.scrape_response,
)
except httpx.ReadTimeout:
scraper.logger.warning(
f"Timeout while fetching search results for movie {metadata.title} ({metadata.year}), retrying later"
@@ -790,9 +759,6 @@ async def background_movie_title_search(
scraper.logger.error(
f"Error fetching search results: {e.response.text}, status code: {e.response.status_code}"
)
finally:
if settings.scrapeops_api_key:
scraper.scrapeops_logger.logger.close_sdk()
scraper.logger.info(
f"Background title search completed for {metadata.title} ({metadata.year})"
@@ -817,13 +783,6 @@ async def background_series_title_search(
if not metadata:
return
if settings.scrapeops_api_key:
scraper.scrapeops_logger = ScrapeOpsRequests(
scrapeops_api_key=settings.scrapeops_api_key,
spider_name="Prowlarr Scraper",
job_name=f"background:{metadata.title}:{metadata.id}:{season}:{episode}",
)
title_streams_generators = [
scraper.scrape_series_by_title(
processed_info_hashes,
@@ -840,11 +799,6 @@ async def background_series_title_search(
try:
async for stream in scraper.process_streams(*title_streams_generators):
await scraper.store_streams([stream])
if settings.scrapeops_api_key:
scraper.scrapeops_logger.item_scraped(
item=stream.model_dump(include={"id"}),
response=scraper.scrape_response,
)
except httpx.ReadTimeout:
scraper.logger.warning(
f"Timeout while fetching search results for {metadata.title} S{season}E{episode}, retrying later"
@@ -859,9 +813,6 @@ async def background_series_title_search(
scraper.logger.error(
f"Error fetching search results: {e.response.text}, status code: {e.response.status_code}"
)
finally:
if settings.scrapeops_api_key:
scraper.scrapeops_logger.logger.close_sdk()
scraper.logger.info(
f"Background title search completed for {metadata.title} S{season}E{episode}"
scrapers/prowlarr_feed.py
-45
View File
@@ -2,7 +2,6 @@ import logging
import dramatiq
import httpx
from scrapeops_python_requests.scrapeops_requests import ScrapeOpsRequests
from db.config import settings
from db.crud import (
@@ -41,13 +40,6 @@ async def mark_item_as_processed(item_id: str):
async def scrape_prowlarr_feed():
scraper = ProwlarrScraper()
if settings.scrapeops_api_key:
scraper.scrapeops_logger = ScrapeOpsRequests(
scrapeops_api_key=settings.scrapeops_api_key,
spider_name="Prowlarr Scraper",
job_name="Prowlarr Feed Scraper",
)
params = {
"type": "search",
"categories": [2000, 5000, 8000], # Movies, TV, and Other categories
@@ -78,9 +70,6 @@ async def scrape_prowlarr_feed():
except Exception as e:
logger.exception(f"Error scraping Prowlarr feed: {e}")
finally:
if scraper.scrapeops_logger:
scraper.scrapeops_logger.logger.close_sdk()
async def process_feed_item(item: dict, scraper: ProwlarrScraper):
@@ -94,12 +83,6 @@ async def process_feed_item(item: dict, scraper: ProwlarrScraper):
parsed_title_data = scraper.parse_title_data(item["title"])
if is_contain_18_plus_keywords(item["title"]):
logger.warning(f"Item {item['title']} contains black listed keywords")
if settings.scrapeops_api_key:
scraper.scrapeops_logger.logger.item_dropped(
item={"info_hash": item_id},
response=scraper.scrape_response,
message="Contains blacklisted keywords",
)
return item_id
# Determine media type
@@ -114,22 +97,10 @@ async def process_feed_item(item: dict, scraper: ProwlarrScraper):
logger.warning(
f"Category 8000 item {item['title']} does not match expected format"
)
if settings.scrapeops_api_key:
scraper.scrapeops_logger.logger.item_dropped(
item={"info_hash": item_id},
response=scraper.scrape_response,
message="Does not match expected format",
)
return item_id
media_type = "series" if parsed_title_data.get("seasons") else "movie"
else:
logger.warning(f"Unsupported category {category_ids} for item {item['title']}")
if settings.scrapeops_api_key:
scraper.scrapeops_logger.logger.item_dropped(
item={"info_hash": item_id},
response=scraper.scrape_response,
message="Unsupported category",
)
return item_id
# Fetch or create metadata
@@ -140,12 +111,6 @@ async def process_feed_item(item: dict, scraper: ProwlarrScraper):
if not metadata:
logger.warning(f"Unable to find or create metadata for {item['title']}")
if settings.scrapeops_api_key:
scraper.scrapeops_logger.logger.item_dropped(
item={"info_hash": item_id},
response=scraper.scrape_response,
message="Unable to find or create metadata",
)
return item_id
# Process the stream
@@ -154,19 +119,9 @@ async def process_feed_item(item: dict, scraper: ProwlarrScraper):
)
if stream:
await store_new_torrent_streams([stream])
if settings.scrapeops_api_key:
scraper.scrapeops_logger.item_scraped(
item={"info_hash": item_id}, response=scraper.scrape_response
)
return item_id
else:
logger.warning(f"Failed to process stream for {item['title']}")
if settings.scrapeops_api_key:
scraper.scrapeops_logger.logger.item_dropped(
item={"info_hash": item_id},
response=scraper.scrape_response,
message="Failed to process stream",
)
return item_id
scrapers/torrentio.py
+1 -19
View File
@@ -5,7 +5,6 @@ from os import path
from typing import List, Dict, Any
import PTT
from scrapeops_python_requests.scrapeops_requests import ScrapeOpsRequests
from tenacity import RetryError
from db.config import settings
@@ -44,14 +43,6 @@ class TorrentioScraper(BaseScraper):
url = f"{self.base_url}/stream/{catalog_type}/{metadata.id}:{season}:{episode}.json"
job_name += f":{season}:{episode}"
scrapeops_logger = None
if settings.scrapeops_api_key:
scrapeops_logger = ScrapeOpsRequests(
scrapeops_api_key=settings.scrapeops_api_key,
spider_name="Torrentio Scraper",
job_name=job_name,
)
try:
response = await self.make_request(url)
response.raise_for_status()
@@ -61,23 +52,14 @@ class TorrentioScraper(BaseScraper):
self.logger.warning(f"Invalid response received for {url}")
return []
stream_data = await self.parse_response(
return await self.parse_response(
data, metadata, catalog_type, season, episode
)
for stream in stream_data:
if scrapeops_logger:
scrapeops_logger.item_scraped(
item=stream.model_dump(include={"id"}), response=response
)
return stream_data
except (ScraperError, RetryError):
return []
except Exception as e:
self.logger.exception(f"Error occurred while fetching {url}: {e}")
return []
finally:
if scrapeops_logger:
scrapeops_logger.logger.close_sdk()
def validate_response(self, response: Dict[str, Any]) -> bool:
return "streams" in response and isinstance(response["streams"], list)
scrapers/zilean.py
-23
View File
@@ -5,7 +5,6 @@ from typing import List, Dict, Any
import PTT
from httpx import Response
from tenacity import RetryError
from scrapeops_python_requests.scrapeops_requests import ScrapeOpsRequests
from db.config import settings
from db.models import TorrentStreams, Season, Episode, MediaFusionMetaData
from scrapers.base_scraper import BaseScraper, ScraperError
@@ -37,14 +36,6 @@ class ZileanScraper(BaseScraper):
if catalog_type == "series":
job_name += f":{season}:{episode}"
scrapeops_logger = None
if settings.scrapeops_api_key:
scrapeops_logger = ScrapeOpsRequests(
scrapeops_api_key=settings.scrapeops_api_key,
spider_name="Zilean Scraper",
job_name=job_name,
)
search_task = asyncio.create_task(
self.make_request(
f"{settings.zilean_url}/dmm/search",
@@ -80,16 +71,13 @@ class ZileanScraper(BaseScraper):
)
stream_data = []
response = None
if isinstance(search_response, Response):
response = search_response
stream_data.extend(search_response.json())
else:
self.logger.error(
f"Error occurred while search {metadata.title}: {search_response}"
)
if isinstance(filtered_response, Response):
response = filtered_response
stream_data.extend(filtered_response.json())
else:
self.logger.error(
@@ -98,20 +86,12 @@ class ZileanScraper(BaseScraper):
if not self.validate_response(stream_data):
self.logger.error(f"No valid streams found for {metadata.title}")
if scrapeops_logger:
scrapeops_logger.logger.close_sdk()
return []
try:
streams = await self.parse_response(
stream_data, metadata, catalog_type, season, episode
)
if scrapeops_logger:
for stream in streams:
scrapeops_logger.item_scraped(
item=stream.model_dump(include={"id"}),
response=response,
)
return streams
except (ScraperError, RetryError):
return []
@@ -120,9 +100,6 @@ class ZileanScraper(BaseScraper):
f"Error occurred while fetching {metadata.title}: {e}"
)
return []
finally:
if scrapeops_logger:
scrapeops_logger.logger.close_sdk()
async def parse_response(
self,