fix(proxy): default the uvicorn worker healthcheck timeout to 60 s so multi-worker boots survive a slow import

This commit is contained in:
mateo-berri 2026-10-07 05:25:15 -07:00
parent cb138ba92f
commit de53d9c0c6
3 changed files with 43 additions and 12 deletions

View file

@ -78,6 +78,7 @@ S3_PARTITION_GRANULARITY_ENV_VAR: Final = "S3_PARTITION_GRANULARITY"
MAX_FILE_LIST_LIMIT: Final = 10000
DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10))
DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1))
DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS: Final = 60
budget_reservation_disabled_info_emitted = False
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = int(os.getenv("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1))
DEFAULT_SQS_BATCH_SIZE: Final = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512))

View file

@ -18,7 +18,7 @@ from dotenv import load_dotenv
from pydantic import ConfigDict
import litellm
from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY
from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY, DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS
from litellm.proxy.db.pgbouncer import (
PgBouncerError,
PgBouncerSettings,
@ -317,15 +317,22 @@ class ProxyInitializationHelpers:
uvicorn_args["log_level"] = resolve_log_level(litellm_log)
if keepalive_timeout is not None:
uvicorn_args["timeout_keep_alive"] = keepalive_timeout
if timeout_worker_healthcheck is not None:
if "timeout_worker_healthcheck" in inspect.signature(uvicorn.Config.__init__).parameters:
uvicorn_args["timeout_worker_healthcheck"] = timeout_worker_healthcheck
else:
print(
f"\033[1;33mLiteLLM Proxy: --timeout_worker_healthcheck "
f"requires uvicorn>=0.37.0, but installed uvicorn=={uvicorn.__version__}. "
f"Ignoring the flag.\033[0m"
)
resolved_worker_healthcheck_timeout: Final = (
DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS
if timeout_worker_healthcheck is None
else timeout_worker_healthcheck
)
uvicorn_supports_worker_healthcheck: Final = (
"timeout_worker_healthcheck" in inspect.signature(uvicorn.Config.__init__).parameters
)
if uvicorn_supports_worker_healthcheck:
uvicorn_args["timeout_worker_healthcheck"] = resolved_worker_healthcheck_timeout
elif timeout_worker_healthcheck is not None:
print(
f"\033[1;33mLiteLLM Proxy: --timeout_worker_healthcheck "
f"requires uvicorn>=0.37.0, but installed uvicorn=={uvicorn.__version__}. "
f"Ignoring the flag.\033[0m"
)
return uvicorn_args
@staticmethod
@ -931,8 +938,11 @@ class ProxyInitializationHelpers:
type=int,
help=(
"Set the uvicorn worker health-check timeout in seconds (uvicorn timeout_worker_healthcheck parameter). "
"Requires uvicorn>=0.37.0. Only applies when running uvicorn directly with --num_workers>1; "
"ignored under --run_gunicorn / --run_hypercorn."
f"Defaults to {DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS} instead of uvicorn's 5: a spawned worker is "
"killed when it misses one healthcheck ping, and importing the proxy takes longer than 5 s on a loaded "
"host, so the lower default crash-loops with 'Child process died'. The same value is the deadline for a "
"replacement worker to finish starting on SIGHUP. Requires uvicorn>=0.37.0. Only applies when running "
"uvicorn directly with --num_workers>1; ignored under --run_gunicorn / --run_hypercorn."
),
envvar="TIMEOUT_WORKER_HEALTHCHECK",
)

View file

@ -22,6 +22,7 @@ import yaml
from uvicorn.config import LOOP_FACTORIES
from uvicorn.importer import import_from_string
from litellm.constants import DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS
from litellm.proxy import proxy_cli
from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server
@ -153,6 +154,25 @@ class TestProxyInitializationHelpers:
)
assert args["timeout_worker_healthcheck"] == 15
args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000)
assert args["timeout_worker_healthcheck"] == DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS
def test_worker_healthcheck_timeout_on_uvicorn_without_the_parameter(self, capsys):
class _OldUvicornConfig:
def __init__(self, timeout_keep_alive=None):
pass
with patch("uvicorn.Config", _OldUvicornConfig):
args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000)
assert "timeout_worker_healthcheck" not in args
assert capsys.readouterr().out == ""
args = ProxyInitializationHelpers._get_default_unvicorn_init_args(
"localhost", 8000, timeout_worker_healthcheck=15
)
assert "timeout_worker_healthcheck" not in args
assert "requires uvicorn>=0.37.0" in capsys.readouterr().out
@staticmethod
def _uvicorn_access_info_enabled(args: dict) -> bool:
import logging