diff --git a/Pipfile b/Pipfile index 8eebcde..3d2cac2 100644 --- a/Pipfile +++ b/Pipfile @@ -46,8 +46,6 @@ parsett = "*" tenacity = "*" ratelimit = "*" qrcode = "*" -scrapeops-scrapy = "*" -scrapeops-python-requests = "*" aioseedrcc = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index c137fbf..4ad125a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "60522a8f5bb55e1ea28080bf09231118086caab9be5a91e6a4b15335f57142d0" + "sha256": "1bfd1eeb0ef28abd4907e5b1e1046ab6912f9b37f4b1b66001a91d41d2f2a115" }, "pipfile-spec": 6, "requires": { @@ -1066,14 +1066,6 @@ "markers": "python_version >= '3.7'", "version": "==1.0.1" }, - "json5": { - "hashes": [ - "sha256:34ed7d834b1341a86987ed52f3f76cd8ee184394906b6e22a1e0deb9ab294e8f", - "sha256:548e41b9be043f9426776f05df8635a00fe06104ea51ed24b67f908856e151ae" - ], - "markers": "python_version >= '3.8'", - "version": "==0.9.25" - }, "jsonschema": { "hashes": [ "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4", @@ -2456,29 +2448,6 @@ "markers": "python_version >= '3.8'", "version": "==0.20.1" }, - "scrapeops-python-logger": { - "hashes": [ - "sha256:faf58d35a6f043e90cd9680c070046f6617ceff725528ab66a98657c7cdef8f3" - ], - "markers": "python_version >= '3.6'", - "version": "==0.4.7" - }, - "scrapeops-python-requests": { - "hashes": [ - "sha256:2c7b6e2ecd7690efff1f0b0f506df1f189f07534bf9f9eb6f5b8e8cf604e5198" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==0.4.7" - }, - "scrapeops-scrapy": { - "hashes": [ - "sha256:42a78c2e48c395895a2145d57ee912dbdf14c675738187fa08c191fb637f6a2a" - ], - "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==0.5.6" - }, "scrapy": { "hashes": [ "sha256:4be353d6abbb942a9f7e7614ca8b5f3d9037381176ac8d8859c8cac676e74fa0", @@ -2626,14 +2595,6 @@ "markers": "python_version >= '3.8'", "version": "==0.22.1" }, - "tld": { - "hashes": [ - "sha256:93dde5e1c04bdf1844976eae440706379d21f4ab235b73c05d7483e074fb5629", - "sha256:f75b2be080f767ed17c2338a339eaa4fab5792586319ca819119da252f9f3749" - ], - "markers": "python_version >= '3.7' and python_version < '4'", - "version": "==0.13" - }, "tldextract": { "hashes": [ "sha256:4dfc4c277b6b97fa053899fcdb892d2dc27295851ab5fac4e07797b6a21b2e46", diff --git a/db/config.py b/db/config.py index c39b297..d2a5f37 100644 --- a/db/config.py +++ b/db/config.py @@ -31,9 +31,6 @@ class Settings(BaseSettings): playwright_cdp_url: str = "ws://browserless:3000?blockAds=true&stealth=true" flaresolverr_url: str = "http://flaresolverr:8191/v1" - # External Service API Keys - scrapeops_api_key: str | None = None - # Prowlarr Settings prowlarr_url: str = "http://prowlarr-service:9696" prowlarr_api_key: str | None = None diff --git a/docs/configuration.md b/docs/configuration.md index 3ea3fdb..290c85d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -36,9 +36,6 @@ These URLs define the locations of various external services used by MediaFusion - **playwright_cdp_url** (default: `"ws://browserless:3000?blockAds=true&stealth=true"`): The URL for the Playwright CDP (Chrome DevTools Protocol) service. - **flaresolverr_url** (default: `"http://flaresolverr:8191/v1"`): The URL for the FlareSolverr service. -## External Service API Keys -- **scrapeops_api_key** (Optional): The API key for the ScrapeOps monitoring service. - ## Prowlarr Settings These settings are specific to the Prowlarr integration. diff --git a/mediafusion_scrapy/settings.py b/mediafusion_scrapy/settings.py index dd67e18..76f5f74 100644 --- a/mediafusion_scrapy/settings.py +++ b/mediafusion_scrapy/settings.py @@ -62,20 +62,6 @@ EXTENSIONS = { "mediafusion_scrapy.extensions.InactivityMonitor": 100, } -if settings.scrapeops_api_key: - SCRAPEOPS_API_KEY = settings.scrapeops_api_key - DOWNLOADER_MIDDLEWARES.update( - { - "scrapeops_scrapy.middleware.retry.RetryMiddleware": 550, - "scrapy.downloadermiddlewares.retry.RetryMiddleware": None, - } - ) - EXTENSIONS.update( - { - "scrapeops_scrapy.extension.ScrapeOpsMonitor": 500, # ScrapeOps Monitor - } - ) - # Configure item pipelines # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html # ITEM_PIPELINES = { diff --git a/scrapers/prowlarr.py b/scrapers/prowlarr.py index f85b2a1..5e1e0bc 100644 --- a/scrapers/prowlarr.py +++ b/scrapers/prowlarr.py @@ -5,7 +5,6 @@ from typing import List, Dict, Any, AsyncGenerator, Literal, AsyncIterable import PTT import dramatiq import httpx -from scrapeops_python_requests.scrapeops_requests import ScrapeOpsRequests from torf import Magnet, MagnetError from db.config import settings @@ -72,8 +71,6 @@ class ProwlarrScraper(BaseScraper): cache_key_prefix=self.cache_key_prefix, logger_name=self.__class__.__name__ ) self.base_url = f"{settings.prowlarr_url}/api/v1/search" - self.scrapeops_logger = None - self.scrape_response = None @BaseScraper.cache(ttl=PROWLARR_SEARCH_TTL) @BaseScraper.rate_limit(calls=5, period=timedelta(seconds=1)) @@ -90,13 +87,6 @@ class ProwlarrScraper(BaseScraper): if catalog_type == "series": job_name += f":{season}:{episode}" - if settings.scrapeops_api_key: - self.scrapeops_logger = ScrapeOpsRequests( - scrapeops_api_key=settings.scrapeops_api_key, - spider_name="Prowlarr Scraper", - job_name=job_name, - ) - try: async for stream in self._scrape_and_parse( processed_info_hashes, @@ -106,11 +96,6 @@ class ProwlarrScraper(BaseScraper): episode, ): results.append(stream) - if settings.scrapeops_api_key: - self.scrapeops_logger.item_scraped( - item=stream.model_dump(include={"id"}), - response=self.scrape_response, - ) except httpx.ReadTimeout: self.logger.warning("Timeout while fetching search results") except httpx.HTTPStatusError as e: @@ -119,9 +104,6 @@ class ProwlarrScraper(BaseScraper): ) except Exception as e: self.logger.exception(f"An error occurred during scraping: {str(e)}") - finally: - if settings.scrapeops_api_key: - self.scrapeops_logger.logger.close_sdk() self.logger.info( f"Returning {len(results)} scraped streams for {metadata.title}" @@ -329,7 +311,6 @@ class ProwlarrScraper(BaseScraper): params=params, timeout=settings.prowlarr_search_query_timeout, ) - self.scrape_response = response response.raise_for_status() return response.json() @@ -753,13 +734,6 @@ async def background_movie_title_search( if not metadata: return - if settings.scrapeops_api_key: - scraper.scrapeops_logger = ScrapeOpsRequests( - scrapeops_api_key=settings.scrapeops_api_key, - spider_name="Prowlarr Scraper", - job_name=f"background:{metadata.title}:{metadata.id}", - ) - title_streams_generators = [ scraper.scrape_movie_by_title( processed_info_hashes, @@ -772,11 +746,6 @@ async def background_movie_title_search( try: async for stream in scraper.process_streams(*title_streams_generators): await scraper.store_streams([stream]) - if settings.scrapeops_api_key: - scraper.scrapeops_logger.item_scraped( - item=stream.model_dump(include={"id"}), - response=scraper.scrape_response, - ) except httpx.ReadTimeout: scraper.logger.warning( f"Timeout while fetching search results for movie {metadata.title} ({metadata.year}), retrying later" @@ -790,9 +759,6 @@ async def background_movie_title_search( scraper.logger.error( f"Error fetching search results: {e.response.text}, status code: {e.response.status_code}" ) - finally: - if settings.scrapeops_api_key: - scraper.scrapeops_logger.logger.close_sdk() scraper.logger.info( f"Background title search completed for {metadata.title} ({metadata.year})" @@ -817,13 +783,6 @@ async def background_series_title_search( if not metadata: return - if settings.scrapeops_api_key: - scraper.scrapeops_logger = ScrapeOpsRequests( - scrapeops_api_key=settings.scrapeops_api_key, - spider_name="Prowlarr Scraper", - job_name=f"background:{metadata.title}:{metadata.id}:{season}:{episode}", - ) - title_streams_generators = [ scraper.scrape_series_by_title( processed_info_hashes, @@ -840,11 +799,6 @@ async def background_series_title_search( try: async for stream in scraper.process_streams(*title_streams_generators): await scraper.store_streams([stream]) - if settings.scrapeops_api_key: - scraper.scrapeops_logger.item_scraped( - item=stream.model_dump(include={"id"}), - response=scraper.scrape_response, - ) except httpx.ReadTimeout: scraper.logger.warning( f"Timeout while fetching search results for {metadata.title} S{season}E{episode}, retrying later" @@ -859,9 +813,6 @@ async def background_series_title_search( scraper.logger.error( f"Error fetching search results: {e.response.text}, status code: {e.response.status_code}" ) - finally: - if settings.scrapeops_api_key: - scraper.scrapeops_logger.logger.close_sdk() scraper.logger.info( f"Background title search completed for {metadata.title} S{season}E{episode}" diff --git a/scrapers/prowlarr_feed.py b/scrapers/prowlarr_feed.py index 7cccf6e..a7a1e0b 100644 --- a/scrapers/prowlarr_feed.py +++ b/scrapers/prowlarr_feed.py @@ -2,7 +2,6 @@ import logging import dramatiq import httpx -from scrapeops_python_requests.scrapeops_requests import ScrapeOpsRequests from db.config import settings from db.crud import ( @@ -41,13 +40,6 @@ async def mark_item_as_processed(item_id: str): async def scrape_prowlarr_feed(): scraper = ProwlarrScraper() - if settings.scrapeops_api_key: - scraper.scrapeops_logger = ScrapeOpsRequests( - scrapeops_api_key=settings.scrapeops_api_key, - spider_name="Prowlarr Scraper", - job_name="Prowlarr Feed Scraper", - ) - params = { "type": "search", "categories": [2000, 5000, 8000], # Movies, TV, and Other categories @@ -78,9 +70,6 @@ async def scrape_prowlarr_feed(): except Exception as e: logger.exception(f"Error scraping Prowlarr feed: {e}") - finally: - if scraper.scrapeops_logger: - scraper.scrapeops_logger.logger.close_sdk() async def process_feed_item(item: dict, scraper: ProwlarrScraper): @@ -94,12 +83,6 @@ async def process_feed_item(item: dict, scraper: ProwlarrScraper): parsed_title_data = scraper.parse_title_data(item["title"]) if is_contain_18_plus_keywords(item["title"]): logger.warning(f"Item {item['title']} contains black listed keywords") - if settings.scrapeops_api_key: - scraper.scrapeops_logger.logger.item_dropped( - item={"info_hash": item_id}, - response=scraper.scrape_response, - message="Contains blacklisted keywords", - ) return item_id # Determine media type @@ -114,22 +97,10 @@ async def process_feed_item(item: dict, scraper: ProwlarrScraper): logger.warning( f"Category 8000 item {item['title']} does not match expected format" ) - if settings.scrapeops_api_key: - scraper.scrapeops_logger.logger.item_dropped( - item={"info_hash": item_id}, - response=scraper.scrape_response, - message="Does not match expected format", - ) return item_id media_type = "series" if parsed_title_data.get("seasons") else "movie" else: logger.warning(f"Unsupported category {category_ids} for item {item['title']}") - if settings.scrapeops_api_key: - scraper.scrapeops_logger.logger.item_dropped( - item={"info_hash": item_id}, - response=scraper.scrape_response, - message="Unsupported category", - ) return item_id # Fetch or create metadata @@ -140,12 +111,6 @@ async def process_feed_item(item: dict, scraper: ProwlarrScraper): if not metadata: logger.warning(f"Unable to find or create metadata for {item['title']}") - if settings.scrapeops_api_key: - scraper.scrapeops_logger.logger.item_dropped( - item={"info_hash": item_id}, - response=scraper.scrape_response, - message="Unable to find or create metadata", - ) return item_id # Process the stream @@ -154,19 +119,9 @@ async def process_feed_item(item: dict, scraper: ProwlarrScraper): ) if stream: await store_new_torrent_streams([stream]) - if settings.scrapeops_api_key: - scraper.scrapeops_logger.item_scraped( - item={"info_hash": item_id}, response=scraper.scrape_response - ) return item_id else: logger.warning(f"Failed to process stream for {item['title']}") - if settings.scrapeops_api_key: - scraper.scrapeops_logger.logger.item_dropped( - item={"info_hash": item_id}, - response=scraper.scrape_response, - message="Failed to process stream", - ) return item_id diff --git a/scrapers/torrentio.py b/scrapers/torrentio.py index c04bbf8..c2ec8ff 100644 --- a/scrapers/torrentio.py +++ b/scrapers/torrentio.py @@ -5,7 +5,6 @@ from os import path from typing import List, Dict, Any import PTT -from scrapeops_python_requests.scrapeops_requests import ScrapeOpsRequests from tenacity import RetryError from db.config import settings @@ -44,14 +43,6 @@ class TorrentioScraper(BaseScraper): url = f"{self.base_url}/stream/{catalog_type}/{metadata.id}:{season}:{episode}.json" job_name += f":{season}:{episode}" - scrapeops_logger = None - if settings.scrapeops_api_key: - scrapeops_logger = ScrapeOpsRequests( - scrapeops_api_key=settings.scrapeops_api_key, - spider_name="Torrentio Scraper", - job_name=job_name, - ) - try: response = await self.make_request(url) response.raise_for_status() @@ -61,23 +52,14 @@ class TorrentioScraper(BaseScraper): self.logger.warning(f"Invalid response received for {url}") return [] - stream_data = await self.parse_response( + return await self.parse_response( data, metadata, catalog_type, season, episode ) - for stream in stream_data: - if scrapeops_logger: - scrapeops_logger.item_scraped( - item=stream.model_dump(include={"id"}), response=response - ) - return stream_data except (ScraperError, RetryError): return [] except Exception as e: self.logger.exception(f"Error occurred while fetching {url}: {e}") return [] - finally: - if scrapeops_logger: - scrapeops_logger.logger.close_sdk() def validate_response(self, response: Dict[str, Any]) -> bool: return "streams" in response and isinstance(response["streams"], list) diff --git a/scrapers/zilean.py b/scrapers/zilean.py index 2970655..97f1598 100644 --- a/scrapers/zilean.py +++ b/scrapers/zilean.py @@ -5,7 +5,6 @@ from typing import List, Dict, Any import PTT from httpx import Response from tenacity import RetryError -from scrapeops_python_requests.scrapeops_requests import ScrapeOpsRequests from db.config import settings from db.models import TorrentStreams, Season, Episode, MediaFusionMetaData from scrapers.base_scraper import BaseScraper, ScraperError @@ -37,14 +36,6 @@ class ZileanScraper(BaseScraper): if catalog_type == "series": job_name += f":{season}:{episode}" - scrapeops_logger = None - if settings.scrapeops_api_key: - scrapeops_logger = ScrapeOpsRequests( - scrapeops_api_key=settings.scrapeops_api_key, - spider_name="Zilean Scraper", - job_name=job_name, - ) - search_task = asyncio.create_task( self.make_request( f"{settings.zilean_url}/dmm/search", @@ -80,16 +71,13 @@ class ZileanScraper(BaseScraper): ) stream_data = [] - response = None if isinstance(search_response, Response): - response = search_response stream_data.extend(search_response.json()) else: self.logger.error( f"Error occurred while search {metadata.title}: {search_response}" ) if isinstance(filtered_response, Response): - response = filtered_response stream_data.extend(filtered_response.json()) else: self.logger.error( @@ -98,20 +86,12 @@ class ZileanScraper(BaseScraper): if not self.validate_response(stream_data): self.logger.error(f"No valid streams found for {metadata.title}") - if scrapeops_logger: - scrapeops_logger.logger.close_sdk() return [] try: streams = await self.parse_response( stream_data, metadata, catalog_type, season, episode ) - if scrapeops_logger: - for stream in streams: - scrapeops_logger.item_scraped( - item=stream.model_dump(include={"id"}), - response=response, - ) return streams except (ScraperError, RetryError): return [] @@ -120,9 +100,6 @@ class ZileanScraper(BaseScraper): f"Error occurred while fetching {metadata.title}: {e}" ) return [] - finally: - if scrapeops_logger: - scrapeops_logger.logger.close_sdk() async def parse_response( self,