From 539dd77a6aa8c245601b01d43012008b5a46a619 Mon Sep 17 00:00:00 2001 From: Zhang Wenhao <126763931+wenhaoone@users.noreply.github.com> Date: Sun, 23 Aug 2026 12:33:31 +0800 Subject: [PATCH] fix(proxy): propagate multi-worker startup failures --- litellm/proxy/proxy_cli.py | 45 ++++++++++++++++-- pyproject.toml | 2 +- tests/test_litellm/proxy/test_proxy_cli.py | 53 +++++++++++++++++++++- uv.lock | 2 +- 4 files changed, 94 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 0449802abae..b419031bdfc 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -281,6 +281,42 @@ class ProxyInitializationHelpers: ) return uvicorn_args + @staticmethod + def _run_uvicorn_server( + uvicorn_args: dict, + num_workers: int, + reload: bool, + ) -> None: + """Run Uvicorn and preserve startup failures from multi-process workers.""" + import inspect + + import uvicorn + + if num_workers <= 1 or reload: + uvicorn.run(**uvicorn_args, workers=num_workers) + return + + from uvicorn.supervisors import Multiprocess + + config = uvicorn.Config(**uvicorn_args, workers=num_workers) + server = uvicorn.Server(config=config) + server_socket = config.bind_socket() + try: + if "target" in inspect.signature(Multiprocess).parameters: + supervisor = Multiprocess(config=config, target=server.run, sockets=[server_socket]) + else: + supervisor = Multiprocess(config=config, sockets=[server_socket]) + supervisor.run() + + startup_failure_code: Final = getattr(uvicorn.config, "STARTUP_FAILURE", 3) + worker_exit_codes: Final = tuple( + process.exitcode for process in supervisor.processes if process.exitcode is not None + ) + if startup_failure_code in worker_exit_codes: + raise SystemExit(startup_failure_code) + finally: + server_socket.close() + @staticmethod def _apply_uvicorn_max_requests_jitter( uvicorn_args: dict, @@ -1072,7 +1108,7 @@ def run_server( ) from e else: try: - import uvicorn + importlib.import_module("uvicorn") except Exception: raise ImportError("uvicorn, gunicorn needs to be imported. Run - `pip install 'litellm[proxy]'`") @@ -1405,9 +1441,10 @@ def run_server( if num_workers > 1: start_query_engine_reaper() - uvicorn.run( - **uvicorn_args, - workers=num_workers, + ProxyInitializationHelpers._run_uvicorn_server( + uvicorn_args=uvicorn_args, + num_workers=num_workers, + reload=reload, ) elif run_gunicorn is True: ProxyInitializationHelpers._run_gunicorn_server( diff --git a/pyproject.toml b/pyproject.toml index fca5c7da1e2..3271aace501 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ Documentation = "https://docs.litellm.ai" [project.optional-dependencies] proxy = [ "gunicorn>=23.0.0,<24.0", - "uvicorn>=0.33.0,<1.0", + "uvicorn>=0.51.0,<1.0", "granian>=2.7.4,<3.0", "uvloop>=0.21.0,<1.0; sys_platform != 'win32'", "fastapi>=0.136.3,<1.0", diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 6ea6f208bb5..f75e74cd33d 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -136,6 +136,53 @@ class TestProxyInitializationHelpers: ) assert args["timeout_worker_healthcheck"] == 15 + def test_run_uvicorn_server_propagates_multi_worker_startup_failure(self): + class _FakeSocket: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + class _FakeConfig: + instances = [] + + def __init__(self, **kwargs): + self.kwargs = kwargs + self.socket = _FakeSocket() + self.__class__.instances.append(self) + + def bind_socket(self): + return self.socket + + class _FakeMultiprocess: + instances = [] + + def __init__(self, config, sockets): + self.processes = [SimpleNamespace(exitcode=3)] + self.ran = False + self.__class__.instances.append(self) + + def run(self): + self.ran = True + + with ( + patch("uvicorn.Config", _FakeConfig), + patch("uvicorn.Server", return_value=MagicMock()), + patch("uvicorn.supervisors.Multiprocess", _FakeMultiprocess), + patch("uvicorn.config.STARTUP_FAILURE", 3, create=True), + ): + with pytest.raises(SystemExit) as exc_info: + ProxyInitializationHelpers._run_uvicorn_server( + uvicorn_args={"app": "litellm.proxy.proxy_server:app"}, + num_workers=2, + reload=False, + ) + + assert exc_info.value.code == 3 + assert _FakeMultiprocess.instances[0].ran is True + assert _FakeConfig.instances[0].socket.closed is True + def test_installed_uvicorn_supports_worker_flags(self): params = inspect.signature(uvicorn.Config.__init__).parameters assert "timeout_worker_healthcheck" in params @@ -1667,7 +1714,9 @@ class TestQueryEngineReaperWiring: ) }, ), - patch("uvicorn.run") as mock_uvicorn_run, + patch( + "litellm.proxy.proxy_cli.ProxyInitializationHelpers._run_uvicorn_server" + ) as mock_run_uvicorn, patch( "litellm.proxy.proxy_cli.start_query_engine_reaper" ) as mock_start_reaper, @@ -1681,7 +1730,7 @@ class TestQueryEngineReaperWiring: "port": 8000, } result = runner.invoke(run_server, args) - return result, mock_uvicorn_run, mock_start_reaper + return result, mock_run_uvicorn, mock_start_reaper def test_multi_worker_uvicorn_starts_reaper(self): result, mock_uvicorn_run, mock_start_reaper = self._invoke_run_server( diff --git a/uv.lock b/uv.lock index 628483c0117..942bcaf13f0 100644 --- a/uv.lock +++ b/uv.lock @@ -4565,7 +4565,7 @@ requires-dist = [ { name = "starlette", marker = "extra == 'proxy'", specifier = ">=1.0.1,<2.0" }, { name = "tiktoken", specifier = ">=0.8.0,<1.0" }, { name = "tokenizers", specifier = ">=0.21.0,<1.0" }, - { name = "uvicorn", marker = "extra == 'proxy'", specifier = ">=0.33.0,<1.0" }, + { name = "uvicorn", marker = "extra == 'proxy'", specifier = ">=0.51.0,<1.0" }, { name = "uvloop", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.21.0,<1.0" }, { name = "websockets", marker = "extra == 'proxy'", specifier = ">=15.0.1,<16.0" }, ]