From e3c8f74a4fa5cc7fde04788b2dc5a95fc55ffe22 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 17 Sep 2026 05:45:33 +0000 Subject: [PATCH] fix(proxy): default prompt injection heuristics executor to a single worker SequenceMatcher holds the GIL, so extra heuristic threads add contention with the event loop without adding throughput. One worker drains scans in arrival order and keeps the loop responsive; PROMPT_INJECTION_HEURISTICS_MAX_THREADS remains an env override Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 4 +--- .../proxy/hooks/test_prompt_injection_detection.py | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 02413ee97ab..e7cb332e712 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -603,9 +603,7 @@ LOGGING_EXECUTOR_MAX_THREADS: Final = get_env_int("LOGGING_EXECUTOR_MAX_THREADS" LOGGING_EXECUTOR_MAX_PENDING_TASKS: Final = get_env_int("LOGGING_EXECUTOR_MAX_PENDING_TASKS", 10_000) LOGGING_EXECUTOR_DROPPED_TASK_LOG_INTERVAL_SECONDS: Final = 30.0 AWS_SIGNING_MAX_THREADS: Final = 16 -PROMPT_INJECTION_HEURISTICS_MAX_THREADS: Final = get_env_int( - "PROMPT_INJECTION_HEURISTICS_MAX_THREADS", os.cpu_count() or 1 -) +PROMPT_INJECTION_HEURISTICS_MAX_THREADS: Final = get_env_int("PROMPT_INJECTION_HEURISTICS_MAX_THREADS", 1) DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE: Final = os.getenv( "DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE", "streaming.chunk.yield" ) diff --git a/tests/test_litellm/proxy/hooks/test_prompt_injection_detection.py b/tests/test_litellm/proxy/hooks/test_prompt_injection_detection.py index b189ee740fe..919914b6a0b 100644 --- a/tests/test_litellm/proxy/hooks/test_prompt_injection_detection.py +++ b/tests/test_litellm/proxy/hooks/test_prompt_injection_detection.py @@ -1,6 +1,5 @@ import asyncio import importlib -import os import time from concurrent.futures import ThreadPoolExecutor @@ -156,7 +155,7 @@ async def test_heuristics_check_does_not_occupy_default_executor(): @pytest.mark.parametrize( ("configured", "expected"), - [("3", 3), ("not-an-int", os.cpu_count() or 1)], + [("3", 3), ("not-an-int", 1)], ) def test_heuristics_thread_count_config_is_honoured(monkeypatch: pytest.MonkeyPatch, configured: str, expected: int): monkeypatch.setenv("PROMPT_INJECTION_HEURISTICS_MAX_THREADS", configured)