diff --git a/mediafusion_scrapy/settings.py b/mediafusion_scrapy/settings.py index 6c68087..7e92a15 100644 --- a/mediafusion_scrapy/settings.py +++ b/mediafusion_scrapy/settings.py @@ -21,14 +21,14 @@ USER_AGENT = "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWeb ROBOTSTXT_OBEY = False # Configure maximum concurrent requests performed by Scrapy (default: 16) -# CONCURRENT_REQUESTS = 32 +CONCURRENT_REQUESTS = 10 # Configure a delay for requests for the same website (default: 0) # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs -# DOWNLOAD_DELAY = 3 +DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: -# CONCURRENT_REQUESTS_PER_DOMAIN = 16 +CONCURRENT_REQUESTS_PER_DOMAIN = 16 # CONCURRENT_REQUESTS_PER_IP = 16 # Disable cookies (enabled by default) @@ -93,10 +93,10 @@ HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" # HTTPCACHE_POLICY = "scrapy.extensions.httpcache.RFC2616Policy" # Set settings whose default value is deprecated to a future-proof value -REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" +REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.12" TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" FEED_EXPORT_ENCODING = "utf-8" -LOG_LEVEL = "DEBUG" +LOG_LEVEL = "INFO" STATS_CLASS = "mediafusion_scrapy.custom_stats.RedisStatsCollector" diff --git a/mediafusion_scrapy/task.py b/mediafusion_scrapy/task.py index dcb9e8d..a2b8972 100644 --- a/mediafusion_scrapy/task.py +++ b/mediafusion_scrapy/task.py @@ -9,7 +9,10 @@ def run_spider_in_process(spider_name, *args, **kwargs): """ Function to start a scrapy spider in a new process. """ - process = CrawlerProcess(get_project_settings()) + # Load the scrapy settings + settings = get_project_settings() + settings.set("LOG_LEVEL", "INFO") + process = CrawlerProcess(settings) process.crawl(spider_name, *args, **kwargs) process.start() @@ -25,4 +28,4 @@ def run_spider(spider_name: str, *args, **kwargs): if __name__ == "__main__": - run_spider_in_process("sport_video") + run_spider_in_process("movies_tv_tgx", scrape_all="true", total_pages=5)