From 87f3d1be784807bac193252c8f27f5e0cf4fc997 Mon Sep 17 00:00:00 2001 From: gvisco Date: Tue, 2 Jun 2026 19:22:52 +0200 Subject: [PATCH] harden uvloop configuration: raise UV_THREADPOOL_SIZE and add LITELLM_DISABLE_UVLOOP escape hatch UV_THREADPOOL_SIZE defaults to 4 in libuv. Under concurrent DNS and filesystem activity, the resolver threads can queue behind other work; a localhost lookup that takes microseconds under idle conditions can stall for seconds when the pool is saturated. Raising the default to 16 in all production Dockerfiles gives the process 4x more headroom without significant memory overhead. Adds LITELLM_DISABLE_UVLOOP env var: when set to "1", "true", or "yes", _get_loop_type returns "asyncio" instead of "uvloop". This lets operators switch event loops without rebuilding the image when they need a workaround for uvloop-specific issues. --- Dockerfile | 3 ++- docker/Dockerfile.database | 3 ++- docker/Dockerfile.non_root | 3 ++- litellm/proxy/proxy_cli.py | 9 +++++++-- tests/test_litellm/proxy/test_proxy_cli.py | 22 ++++++++++++++++++++-- 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9ad9ab31b65..00a0856a69a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,7 +81,8 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \ { apk del --no-cache npm 2>/dev/null || true; } WORKDIR /app -ENV PATH="/app/.venv/bin:${PATH}" +ENV PATH="/app/.venv/bin:${PATH}" \ + UV_THREADPOOL_SIZE=16 COPY --from=builder /app /app # Prisma binaries live in $HOME/.cache (default prisma-python location), diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index c84003a065f..49caa27e461 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -88,7 +88,8 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \ { apk del --no-cache npm 2>/dev/null || true; } WORKDIR /app -ENV PATH="/app/.venv/bin:${PATH}" +ENV PATH="/app/.venv/bin:${PATH}" \ + UV_THREADPOOL_SIZE=16 COPY --from=builder /app /app # Prisma binaries live in $HOME/.cache (default prisma-python location), diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 8717e5b3fcd..9187a3b0f6a 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -107,7 +107,8 @@ ENV PATH="/app/.venv/bin:${PATH}" \ PRISMA_SKIP_POSTINSTALL_GENERATE=1 \ PRISMA_HIDE_UPDATE_MESSAGE=1 \ PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \ - PRISMA_OFFLINE_MODE=true + PRISMA_OFFLINE_MODE=true \ + UV_THREADPOOL_SIZE=16 RUN mkdir -p /nonexistent /var/lib/litellm/assets /var/lib/litellm/ui && \ chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent && \ diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index c0246f234a8..38e177eaf00 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -479,9 +479,14 @@ class ProxyInitializationHelpers: @staticmethod def _get_loop_type(): - """Helper function to determine the event loop type based on platform""" if sys.platform in ("win32", "cygwin", "cli"): - return None # Let uvicorn choose the default loop on Windows + return None + if os.environ.get("LITELLM_DISABLE_UVLOOP", "").strip().lower() in ( + "1", + "true", + "yes", + ): + return "asyncio" return "uvloop" @staticmethod diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 580ed95062b..7988628e5a9 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -354,11 +354,29 @@ class TestProxyInitializationHelpers: assert ProxyInitializationHelpers._is_port_in_use(8000) is False def test_get_loop_type(self): - # Test on Windows with patch("sys.platform", "win32"): assert ProxyInitializationHelpers._get_loop_type() is None - # Test on Linux + with patch("sys.platform", "linux"): + assert ProxyInitializationHelpers._get_loop_type() == "uvloop" + + @patch.dict(os.environ, {"LITELLM_DISABLE_UVLOOP": "1"}) + def test_get_loop_type_disable_uvloop_numeric(self): + with patch("sys.platform", "linux"): + assert ProxyInitializationHelpers._get_loop_type() == "asyncio" + + @patch.dict(os.environ, {"LITELLM_DISABLE_UVLOOP": "true"}) + def test_get_loop_type_disable_uvloop_string(self): + with patch("sys.platform", "linux"): + assert ProxyInitializationHelpers._get_loop_type() == "asyncio" + + @patch.dict(os.environ, {"LITELLM_DISABLE_UVLOOP": "yes"}) + def test_get_loop_type_disable_uvloop_yes(self): + with patch("sys.platform", "linux"): + assert ProxyInitializationHelpers._get_loop_type() == "asyncio" + + @patch.dict(os.environ, {"LITELLM_DISABLE_UVLOOP": "0"}) + def test_get_loop_type_disable_uvloop_false(self): with patch("sys.platform", "linux"): assert ProxyInitializationHelpers._get_loop_type() == "uvloop"