feat: allow configuration of ProcessPoolExecutor max workers with auto-detection and logging

.env-sample

@@ -27,6 +27,7 @@ FASTAPI_PORT=8000
 FASTAPI_WORKERS=1 # DO NOT change this if you don't know what you are doing. Setting this to -1 will spawn a worker for each CPU core, which can consume several GBs of RAM and cause high CPU usage.
 USE_GUNICORN=True # Will use uvicorn if False or if on Windows
 GUNICORN_PRELOAD_APP=True # Set to False to start workers without preloading the app (reduces startup cost but requires schema to exist)
+EXECUTOR_MAX_WORKERS= # Max workers for ProcessPoolExecutor (handles CPU-intensive tasks like RTN parsing). Leave empty for auto (min(cpu_count, 4)). Increase for higher concurrency on CPU-bound workloads.
 
 # ============================== #
 # Playback Settings              #

comet/core/execution.py

@@ -4,6 +4,8 @@ import os
 import signal
 from concurrent.futures import ProcessPoolExecutor
 
+from comet.core.models import settings
+
 _mp_context = None
 try:
     _mp_context = multiprocessing.get_context("forkserver")
@@ -17,9 +19,11 @@ def worker_initializer():
     signal.signal(signal.SIGINT, signal.SIG_IGN)
 
 
-def setup_executor(max_workers: int | None = None):
+def setup_executor():
     global app_executor
 
+    max_workers = settings.EXECUTOR_MAX_WORKERS
+
     if max_workers is None:
         cpu_count = os.cpu_count() or 1
         max_workers = min(cpu_count, 4)

comet/core/logger.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import re
 import sys
 import time
@@ -176,6 +177,15 @@ def log_startup_info(settings):
     )
     logger.log("COMET", f"Gunicorn Preload App: {settings.GUNICORN_PRELOAD_APP}")
 
+    executor_workers = settings.EXECUTOR_MAX_WORKERS
+    if executor_workers is None:
+        cpu_count = os.cpu_count() or 1
+        executor_workers = min(cpu_count, 4)
+    logger.log(
+        "COMET",
+        f"ProcessPoolExecutor: {executor_workers} workers {'(auto)' if settings.EXECUTOR_MAX_WORKERS is None else ''}",
+    )
+
     if settings.PUBLIC_BASE_URL:
         logger.log("COMET", f"Public Base URL: {settings.PUBLIC_BASE_URL}")
 

comet/core/models.py

@@ -28,6 +28,7 @@ class AppSettings(BaseSettings):
     FASTAPI_WORKERS: Optional[int] = 1
     USE_GUNICORN: Optional[bool] = True
     GUNICORN_PRELOAD_APP: Optional[bool] = True
+    EXECUTOR_MAX_WORKERS: Optional[int] = None
     ADMIN_DASHBOARD_PASSWORD: Optional[str] = "".join(
         random.choices(string.ascii_letters + string.digits, k=16)
     )