Add shutdown signal count to force close scrapy spider after retries

mediafusion_scrapy/extensions.py

@@ -5,6 +5,9 @@ from scrapy.extensions.closespider import CloseSpider
 
 
 class CloseSpiderExtended(CloseSpider):
+    def __init__(self, crawler):
+        super().__init__(crawler)
+        self.shutdown_signal_count = 0
 
     def _count_items_produced(self, spider: Spider) -> None:
         if self.items_in_period >= 1:
@@ -14,6 +17,7 @@ class CloseSpiderExtended(CloseSpider):
                 f"Closing spider since no items were produced in the last "
                 f"{self.timeout_no_item} seconds."
             )
+            self.shutdown_signal_count += 1
             dramatiq_message = CurrentMessage.get_current_message()
             if dramatiq_message:
                 spider.logger.info(
@@ -22,3 +26,10 @@ class CloseSpiderExtended(CloseSpider):
                 abort(dramatiq_message.message_id, abort_ttl=0)
             assert self.crawler.engine
             self.crawler.engine.close_spider(spider, "closespider_timeout_no_item")
+
+        if self.shutdown_signal_count >= 3:
+            spider.logger.info(
+                f"Closing spider since no items were produced in the last "
+                f"{self.timeout_no_item} seconds for 3 consecutive times. Forcefully closing the spider."
+            )
+            raise SystemExit(0)