From 7e98d1f067b864261c40a871bdc6f98f38c7e0d5 Mon Sep 17 00:00:00 2001
From: g0ldyy <153996346+g0ldyy@users.noreply.github.com>
Date: Sun, 4 Jan 2026 23:39:01 +0100
Subject: [PATCH] feat: allow configuration of ProcessPoolExecutor max workers
 with auto-detection and logging

---
 .env-sample             |  1 +
 comet/core/execution.py |  6 +++++-
 comet/core/logger.py    | 10 ++++++++++
 comet/core/models.py    |  1 +
 4 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/.env-sample b/.env-sample
index 177b4ee..a28a6d6 100644
--- a/.env-sample
+++ b/.env-sample
@@ -27,6 +27,7 @@ FASTAPI_PORT=8000
 FASTAPI_WORKERS=1 # DO NOT change this if you don't know what you are doing. Setting this to -1 will spawn a worker for each CPU core, which can consume several GBs of RAM and cause high CPU usage.
 USE_GUNICORN=True # Will use uvicorn if False or if on Windows
 GUNICORN_PRELOAD_APP=True # Set to False to start workers without preloading the app (reduces startup cost but requires schema to exist)
+EXECUTOR_MAX_WORKERS= # Max workers for ProcessPoolExecutor (handles CPU-intensive tasks like RTN parsing). Leave empty for auto (min(cpu_count, 4)). Increase for higher concurrency on CPU-bound workloads.
 
 # ============================== #
 # Playback Settings              #
diff --git a/comet/core/execution.py b/comet/core/execution.py
index 52821f9..83e8ed6 100644
--- a/comet/core/execution.py
+++ b/comet/core/execution.py
@@ -4,6 +4,8 @@ import os
 import signal
 from concurrent.futures import ProcessPoolExecutor
 
+from comet.core.models import settings
+
 _mp_context = None
 try:
     _mp_context = multiprocessing.get_context("forkserver")
@@ -17,9 +19,11 @@ def worker_initializer():
     signal.signal(signal.SIGINT, signal.SIG_IGN)
 
 
-def setup_executor(max_workers: int | None = None):
+def setup_executor():
     global app_executor
 
+    max_workers = settings.EXECUTOR_MAX_WORKERS
+
     if max_workers is None:
         cpu_count = os.cpu_count() or 1
         max_workers = min(cpu_count, 4)
diff --git a/comet/core/logger.py b/comet/core/logger.py
index 0bbca1e..955a967 100644
--- a/comet/core/logger.py
+++ b/comet/core/logger.py
@@ -1,4 +1,5 @@
 import logging
+import os
 import re
 import sys
 import time
@@ -176,6 +177,15 @@ def log_startup_info(settings):
     )
     logger.log("COMET", f"Gunicorn Preload App: {settings.GUNICORN_PRELOAD_APP}")
 
+    executor_workers = settings.EXECUTOR_MAX_WORKERS
+    if executor_workers is None:
+        cpu_count = os.cpu_count() or 1
+        executor_workers = min(cpu_count, 4)
+    logger.log(
+        "COMET",
+        f"ProcessPoolExecutor: {executor_workers} workers {'(auto)' if settings.EXECUTOR_MAX_WORKERS is None else ''}",
+    )
+
     if settings.PUBLIC_BASE_URL:
         logger.log("COMET", f"Public Base URL: {settings.PUBLIC_BASE_URL}")
 
diff --git a/comet/core/models.py b/comet/core/models.py
index cdd5cf6..b7b5a29 100644
--- a/comet/core/models.py
+++ b/comet/core/models.py
@@ -28,6 +28,7 @@ class AppSettings(BaseSettings):
     FASTAPI_WORKERS: Optional[int] = 1
     USE_GUNICORN: Optional[bool] = True
     GUNICORN_PRELOAD_APP: Optional[bool] = True
+    EXECUTOR_MAX_WORKERS: Optional[int] = None
     ADMIN_DASHBOARD_PASSWORD: Optional[str] = "".join(
         random.choices(string.ascii_letters + string.digits, k=16)
     )