Update Scrapy settings

mediafusion_scrapy/settings.py

@@ -21,14 +21,14 @@ USER_AGENT = "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWeb
 ROBOTSTXT_OBEY = False
 
 # Configure maximum concurrent requests performed by Scrapy (default: 16)
-# CONCURRENT_REQUESTS = 32
+CONCURRENT_REQUESTS = 10
 
 # Configure a delay for requests for the same website (default: 0)
 # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
 # See also autothrottle settings and docs
-# DOWNLOAD_DELAY = 3
+DOWNLOAD_DELAY = 3
 # The download delay setting will honor only one of:
-# CONCURRENT_REQUESTS_PER_DOMAIN = 16
+CONCURRENT_REQUESTS_PER_DOMAIN = 16
 # CONCURRENT_REQUESTS_PER_IP = 16
 
 # Disable cookies (enabled by default)
@@ -93,10 +93,10 @@ HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
 # HTTPCACHE_POLICY = "scrapy.extensions.httpcache.RFC2616Policy"
 
 # Set settings whose default value is deprecated to a future-proof value
-REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
+REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.12"
 TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
 FEED_EXPORT_ENCODING = "utf-8"
-LOG_LEVEL = "DEBUG"
+LOG_LEVEL = "INFO"
 
 STATS_CLASS = "mediafusion_scrapy.custom_stats.RedisStatsCollector"
 

mediafusion_scrapy/task.py

@@ -9,7 +9,10 @@ def run_spider_in_process(spider_name, *args, **kwargs):
     """
     Function to start a scrapy spider in a new process.
     """
-    process = CrawlerProcess(get_project_settings())
+    # Load the scrapy settings
+    settings = get_project_settings()
+    settings.set("LOG_LEVEL", "INFO")
+    process = CrawlerProcess(settings)
     process.crawl(spider_name, *args, **kwargs)
     process.start()
 
@@ -25,4 +28,4 @@ def run_spider(spider_name: str, *args, **kwargs):
 
 
 if __name__ == "__main__":
-    run_spider_in_process("sport_video")
+    run_spider_in_process("movies_tv_tgx", scrape_all="true", total_pages=5)