From de53d9c0c6595a839733283771db334eb2ed9dba Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:25:15 -0700 Subject: [PATCH 1/2] fix(proxy): default the uvicorn worker healthcheck timeout to 60 s so multi-worker boots survive a slow import --- litellm/constants.py | 1 + litellm/proxy/proxy_cli.py | 34 +++++++++++++++++++----------- tests/unit/proxy/test_proxy_cli.py | 20 ++++++++++++++++++ 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 6343c0675e2..8581b0d22d6 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -78,6 +78,7 @@ S3_PARTITION_GRANULARITY_ENV_VAR: Final = "S3_PARTITION_GRANULARITY" MAX_FILE_LIST_LIMIT: Final = 10000 DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)) DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)) +DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS: Final = 60 budget_reservation_disabled_info_emitted = False DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = int(os.getenv("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1)) DEFAULT_SQS_BATCH_SIZE: Final = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512)) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 53a447bc6ec..76207005059 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -18,7 +18,7 @@ from dotenv import load_dotenv from pydantic import ConfigDict import litellm -from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY +from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY, DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS from litellm.proxy.db.pgbouncer import ( PgBouncerError, PgBouncerSettings, @@ -317,15 +317,22 @@ class ProxyInitializationHelpers: uvicorn_args["log_level"] = resolve_log_level(litellm_log) if keepalive_timeout is not None: uvicorn_args["timeout_keep_alive"] = keepalive_timeout - if timeout_worker_healthcheck is not None: - if "timeout_worker_healthcheck" in inspect.signature(uvicorn.Config.__init__).parameters: - uvicorn_args["timeout_worker_healthcheck"] = timeout_worker_healthcheck - else: - print( - f"\033[1;33mLiteLLM Proxy: --timeout_worker_healthcheck " - f"requires uvicorn>=0.37.0, but installed uvicorn=={uvicorn.__version__}. " - f"Ignoring the flag.\033[0m" - ) + resolved_worker_healthcheck_timeout: Final = ( + DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS + if timeout_worker_healthcheck is None + else timeout_worker_healthcheck + ) + uvicorn_supports_worker_healthcheck: Final = ( + "timeout_worker_healthcheck" in inspect.signature(uvicorn.Config.__init__).parameters + ) + if uvicorn_supports_worker_healthcheck: + uvicorn_args["timeout_worker_healthcheck"] = resolved_worker_healthcheck_timeout + elif timeout_worker_healthcheck is not None: + print( + f"\033[1;33mLiteLLM Proxy: --timeout_worker_healthcheck " + f"requires uvicorn>=0.37.0, but installed uvicorn=={uvicorn.__version__}. " + f"Ignoring the flag.\033[0m" + ) return uvicorn_args @staticmethod @@ -931,8 +938,11 @@ class ProxyInitializationHelpers: type=int, help=( "Set the uvicorn worker health-check timeout in seconds (uvicorn timeout_worker_healthcheck parameter). " - "Requires uvicorn>=0.37.0. Only applies when running uvicorn directly with --num_workers>1; " - "ignored under --run_gunicorn / --run_hypercorn." + f"Defaults to {DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS} instead of uvicorn's 5: a spawned worker is " + "killed when it misses one healthcheck ping, and importing the proxy takes longer than 5 s on a loaded " + "host, so the lower default crash-loops with 'Child process died'. The same value is the deadline for a " + "replacement worker to finish starting on SIGHUP. Requires uvicorn>=0.37.0. Only applies when running " + "uvicorn directly with --num_workers>1; ignored under --run_gunicorn / --run_hypercorn." ), envvar="TIMEOUT_WORKER_HEALTHCHECK", ) diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index 8fc807cd8a0..d81ac57df3c 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -22,6 +22,7 @@ import yaml from uvicorn.config import LOOP_FACTORIES from uvicorn.importer import import_from_string +from litellm.constants import DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS from litellm.proxy import proxy_cli from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server @@ -153,6 +154,25 @@ class TestProxyInitializationHelpers: ) assert args["timeout_worker_healthcheck"] == 15 + args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000) + assert args["timeout_worker_healthcheck"] == DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS + + def test_worker_healthcheck_timeout_on_uvicorn_without_the_parameter(self, capsys): + class _OldUvicornConfig: + def __init__(self, timeout_keep_alive=None): + pass + + with patch("uvicorn.Config", _OldUvicornConfig): + args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000) + assert "timeout_worker_healthcheck" not in args + assert capsys.readouterr().out == "" + + args = ProxyInitializationHelpers._get_default_unvicorn_init_args( + "localhost", 8000, timeout_worker_healthcheck=15 + ) + assert "timeout_worker_healthcheck" not in args + assert "requires uvicorn>=0.37.0" in capsys.readouterr().out + @staticmethod def _uvicorn_access_info_enabled(args: dict) -> bool: import logging From cf5e091f7bfd2efab96b5d6cbb07be0db7d9c439 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:08:47 -0700 Subject: [PATCH 2/2] fix(proxy): warn on uvicorn below 0.37 when multi-worker boots cannot get the longer healthcheck uvicorn 0.33 to 0.36 has no timeout_worker_healthcheck, so the 60 s default never reaches it and a slow worker start still crash-loops with no hint. Print an upgrade warning when the proxy runs more than one uvicorn worker on such a version, keep granian out of the uvicorn-only path, and type the new tests. --- litellm/proxy/proxy_cli.py | 10 ++++- tests/unit/proxy/test_proxy_cli.py | 69 +++++++++++++++++++++++++++--- 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 4f5d9ef5376..7de80ecb51f 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -340,6 +340,7 @@ class ProxyInitializationHelpers: log_config: str | None = None, keepalive_timeout: int | None = None, timeout_worker_healthcheck: int | None = None, + num_workers: int = 1, ) -> dict: """ Get the arguments for `uvicorn` worker @@ -385,6 +386,12 @@ class ProxyInitializationHelpers: f"requires uvicorn>=0.37.0, but installed uvicorn=={uvicorn.__version__}. " f"Ignoring the flag.\033[0m", ) + elif num_workers > 1: + print_startup_message( + f"\033[1;33mLiteLLM Proxy: uvicorn=={uvicorn.__version__} gives each worker a fixed 5 s to answer " + f"its health ping, so a worker that starts slower is killed with 'Child process died'. Install " + f"uvicorn>=0.37.0 to get the {DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS} s default.\033[0m", + ) return uvicorn_args _get_default_unvicorn_init_args = get_default_unvicorn_init_args @@ -1667,13 +1674,14 @@ def run_server( f"(pid {metrics_process.pid})\033[0m" ) - running_uvicorn: Final = run_gunicorn is False and run_hypercorn is False + running_uvicorn: Final = run_gunicorn is False and run_hypercorn is False and run_granian is False uvicorn_args: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( host=host, port=port, log_config=log_config, keepalive_timeout=keepalive_timeout, timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None), + num_workers=(num_workers if running_uvicorn else 1), ) # Optional: recycle uvicorn workers after N requests if max_requests_before_restart is not None: diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index 3a921057949..cdfd3536973 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -164,28 +164,40 @@ class TestProxyInitializationHelpers: ("flag_value", "expected"), [(None, DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS), (15, 15)], ) - def test_uvicorn_worker_healthcheck_timeout_reaches_uvicorn(self, flag_value, expected): + def test_uvicorn_worker_healthcheck_timeout_reaches_uvicorn(self, flag_value: int | None, expected: int): args: Final = ProxyInitializationHelpers._get_default_unvicorn_init_args( "localhost", 8000, timeout_worker_healthcheck=flag_value ) assert uvicorn.Config(**args).timeout_worker_healthcheck == expected - def test_worker_healthcheck_timeout_on_uvicorn_without_the_parameter(self, capsys): + def test_worker_healthcheck_timeout_on_uvicorn_without_the_parameter( + self, capsys: pytest.CaptureFixture[str] + ): class _OldUvicornConfig: - def __init__(self, timeout_keep_alive=None): + def __init__(self, timeout_keep_alive: int | None = None): pass with patch("uvicorn.Config", _OldUvicornConfig): - default_args: Final = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000) - assert "timeout_worker_healthcheck" not in default_args + single_worker_args: Final = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000) + assert "timeout_worker_healthcheck" not in single_worker_args assert capsys.readouterr().out == "" + multi_worker_args: Final = ProxyInitializationHelpers._get_default_unvicorn_init_args( + "localhost", 8000, num_workers=2 + ) + assert "timeout_worker_healthcheck" not in multi_worker_args + multi_worker_output: Final = capsys.readouterr().out + assert "fixed 5 s" in multi_worker_output + assert "uvicorn>=0.37.0" in multi_worker_output + flagged_args: Final = ProxyInitializationHelpers._get_default_unvicorn_init_args( - "localhost", 8000, timeout_worker_healthcheck=15 + "localhost", 8000, timeout_worker_healthcheck=15, num_workers=2 ) assert "timeout_worker_healthcheck" not in flagged_args - assert "requires uvicorn>=0.37.0" in capsys.readouterr().out + flagged_output: Final = capsys.readouterr().out + assert "requires uvicorn>=0.37.0" in flagged_output + assert "fixed 5 s" not in flagged_output @pytest.mark.parametrize( ("raw_timeout", "expected"), @@ -1557,6 +1569,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=30, timeout_worker_healthcheck=None, + num_workers=1, ) mock_uvicorn_run.assert_called_once() @@ -1633,6 +1646,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=None, timeout_worker_healthcheck=15, + num_workers=1, ) @patch("uvicorn.run") @@ -2442,6 +2456,47 @@ class TestQueryEngineReaperWiring: result = runner.invoke(run_server, args) return result, mock_uvicorn_run, mock_start_reaper + @pytest.mark.parametrize( + ("args", "warns"), + [ + (["--local", "--num_workers", "2"], True), + (["--local", "--num_workers", "1"], False), + (["--local", "--num_workers", "2", "--run_granian"], False), + ], + ) + def test_old_uvicorn_warns_only_for_multi_worker_uvicorn(self, args: list[str], warns: bool): + from click.testing import CliRunner + + from litellm.proxy.proxy_cli import run_server + + class _OldUvicornConfig: + def __init__(self, timeout_keep_alive: int | None = None): + pass + + clean_env: Final = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.dict( + "sys.modules", + { + "proxy_server": MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + }, + ), + patch("uvicorn.Config", _OldUvicornConfig), + patch("uvicorn.run"), + patch("litellm.proxy.proxy_cli.start_query_engine_reaper"), + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.init_granian_server"), + ): + result: Final = CliRunner().invoke(run_server, args) + + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" + assert ("uvicorn>=0.37.0" in result.output) is warns + def test_multi_worker_uvicorn_starts_reaper(self): result, mock_uvicorn_run, mock_start_reaper = self._invoke_run_server( ["--local", "--num_workers", "2"]