mirror of
https://github.com/Viren070/MediaFusion.git
synced 2025-12-01 23:21:11 +01:00
mhdzumair
863247018d
Integrate `dramatiq-abort` with Redis backend for task abortion and enhance the `CloseSpider` extension to abort jobs with no recent activity. These changes improve task management and error handling for idle spiders in the scraping process. Updated dependencies and project settings accordingly.
37 lines
932 B
Python
37 lines
932 B
Python
# Desc: Setup the dramatiq broker and middleware.
|
|
import dramatiq
|
|
from dramatiq.brokers.redis import RedisBroker
|
|
from dramatiq.middleware import (
|
|
AsyncIO,
|
|
AgeLimit,
|
|
TimeLimit,
|
|
ShutdownNotifications,
|
|
Callbacks,
|
|
Pipelines,
|
|
Prometheus,
|
|
CurrentMessage,
|
|
)
|
|
from dramatiq_abort import Abortable
|
|
from dramatiq_abort.backends import RedisBackend
|
|
|
|
from api.middleware import MaxTasksPerChild, Retries, TaskManager
|
|
from db.config import settings
|
|
|
|
# Setup the broker and the middleware
|
|
redis_broker = RedisBroker(url=settings.redis_url)
|
|
redis_broker.middleware = [
|
|
Prometheus(),
|
|
AgeLimit(),
|
|
TimeLimit(),
|
|
ShutdownNotifications(),
|
|
Callbacks(),
|
|
Pipelines(),
|
|
Retries(),
|
|
AsyncIO(),
|
|
MaxTasksPerChild(settings.worker_max_tasks_per_child),
|
|
TaskManager(),
|
|
CurrentMessage(),
|
|
Abortable(backend=RedisBackend.from_url(settings.redis_url)),
|
|
]
|
|
dramatiq.set_broker(redis_broker)
|