scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-22 00:31:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 51

fix(proxy): default prompt injection heuristics executor to a single worker

Browse source 
SequenceMatcher holds the GIL, so extra heuristic threads add contention with the event loop without adding throughput. One worker drains scans in arrival order and keeps the loop responsive; PROMPT_INJECTION_HEURISTICS_MAX_THREADS remains an env override

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-17 05:45:33 +00:00
parent d6f6f64c0f
commit e3c8f74a4f
2 changed files with 2 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/constants.py
View file

@ -603,9 +603,7 @@ LOGGING_EXECUTOR_MAX_THREADS: Final = get_env_int("LOGGING_EXECUTOR_MAX_THREADS"
LOGGING_EXECUTOR_MAX_PENDING_TASKS: Final = get_env_int("LOGGING_EXECUTOR_MAX_PENDING_TASKS", 10_000)
LOGGING_EXECUTOR_DROPPED_TASK_LOG_INTERVAL_SECONDS: Final = 30.0
AWS_SIGNING_MAX_THREADS: Final = 16
PROMPT_INJECTION_HEURISTICS_MAX_THREADS: Final = get_env_int(
"PROMPT_INJECTION_HEURISTICS_MAX_THREADS", os.cpu_count() or 1
)
PROMPT_INJECTION_HEURISTICS_MAX_THREADS: Final = get_env_int("PROMPT_INJECTION_HEURISTICS_MAX_THREADS", 1)
DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE: Final = os.getenv(
"DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE", "streaming.chunk.yield"
)

3
tests/test_litellm/proxy/hooks/test_prompt_injection_detection.py
View file

@ -1,6 +1,5 @@
import asyncio
import importlib
import os
import time
from concurrent.futures import ThreadPoolExecutor
@ -156,7 +155,7 @@ async def test_heuristics_check_does_not_occupy_default_executor():
@pytest.mark.parametrize(
("configured", "expected"),
[("3", 3), ("not-an-int", os.cpu_count() or 1)],
[("3", 3), ("not-an-int", 1)],
)
def test_heuristics_thread_count_config_is_honoured(monkeypatch: pytest.MonkeyPatch, configured: str, expected: int):
monkeypatch.setenv("PROMPT_INJECTION_HEURISTICS_MAX_THREADS", configured)