mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): default the uvicorn worker healthcheck timeout to 60 s so multi-worker boots survive a slow import
This commit is contained in:
parent
cb138ba92f
commit
de53d9c0c6
3 changed files with 43 additions and 12 deletions
|
|
@ -78,6 +78,7 @@ S3_PARTITION_GRANULARITY_ENV_VAR: Final = "S3_PARTITION_GRANULARITY"
|
|||
MAX_FILE_LIST_LIMIT: Final = 10000
|
||||
DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10))
|
||||
DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1))
|
||||
DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS: Final = 60
|
||||
budget_reservation_disabled_info_emitted = False
|
||||
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = int(os.getenv("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1))
|
||||
DEFAULT_SQS_BATCH_SIZE: Final = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512))
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from dotenv import load_dotenv
|
|||
from pydantic import ConfigDict
|
||||
|
||||
import litellm
|
||||
from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY
|
||||
from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY, DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS
|
||||
from litellm.proxy.db.pgbouncer import (
|
||||
PgBouncerError,
|
||||
PgBouncerSettings,
|
||||
|
|
@ -317,15 +317,22 @@ class ProxyInitializationHelpers:
|
|||
uvicorn_args["log_level"] = resolve_log_level(litellm_log)
|
||||
if keepalive_timeout is not None:
|
||||
uvicorn_args["timeout_keep_alive"] = keepalive_timeout
|
||||
if timeout_worker_healthcheck is not None:
|
||||
if "timeout_worker_healthcheck" in inspect.signature(uvicorn.Config.__init__).parameters:
|
||||
uvicorn_args["timeout_worker_healthcheck"] = timeout_worker_healthcheck
|
||||
else:
|
||||
print(
|
||||
f"\033[1;33mLiteLLM Proxy: --timeout_worker_healthcheck "
|
||||
f"requires uvicorn>=0.37.0, but installed uvicorn=={uvicorn.__version__}. "
|
||||
f"Ignoring the flag.\033[0m"
|
||||
)
|
||||
resolved_worker_healthcheck_timeout: Final = (
|
||||
DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS
|
||||
if timeout_worker_healthcheck is None
|
||||
else timeout_worker_healthcheck
|
||||
)
|
||||
uvicorn_supports_worker_healthcheck: Final = (
|
||||
"timeout_worker_healthcheck" in inspect.signature(uvicorn.Config.__init__).parameters
|
||||
)
|
||||
if uvicorn_supports_worker_healthcheck:
|
||||
uvicorn_args["timeout_worker_healthcheck"] = resolved_worker_healthcheck_timeout
|
||||
elif timeout_worker_healthcheck is not None:
|
||||
print(
|
||||
f"\033[1;33mLiteLLM Proxy: --timeout_worker_healthcheck "
|
||||
f"requires uvicorn>=0.37.0, but installed uvicorn=={uvicorn.__version__}. "
|
||||
f"Ignoring the flag.\033[0m"
|
||||
)
|
||||
return uvicorn_args
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -931,8 +938,11 @@ class ProxyInitializationHelpers:
|
|||
type=int,
|
||||
help=(
|
||||
"Set the uvicorn worker health-check timeout in seconds (uvicorn timeout_worker_healthcheck parameter). "
|
||||
"Requires uvicorn>=0.37.0. Only applies when running uvicorn directly with --num_workers>1; "
|
||||
"ignored under --run_gunicorn / --run_hypercorn."
|
||||
f"Defaults to {DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS} instead of uvicorn's 5: a spawned worker is "
|
||||
"killed when it misses one healthcheck ping, and importing the proxy takes longer than 5 s on a loaded "
|
||||
"host, so the lower default crash-loops with 'Child process died'. The same value is the deadline for a "
|
||||
"replacement worker to finish starting on SIGHUP. Requires uvicorn>=0.37.0. Only applies when running "
|
||||
"uvicorn directly with --num_workers>1; ignored under --run_gunicorn / --run_hypercorn."
|
||||
),
|
||||
envvar="TIMEOUT_WORKER_HEALTHCHECK",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import yaml
|
|||
from uvicorn.config import LOOP_FACTORIES
|
||||
from uvicorn.importer import import_from_string
|
||||
|
||||
from litellm.constants import DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS
|
||||
from litellm.proxy import proxy_cli
|
||||
from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server
|
||||
|
||||
|
|
@ -153,6 +154,25 @@ class TestProxyInitializationHelpers:
|
|||
)
|
||||
assert args["timeout_worker_healthcheck"] == 15
|
||||
|
||||
args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000)
|
||||
assert args["timeout_worker_healthcheck"] == DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS
|
||||
|
||||
def test_worker_healthcheck_timeout_on_uvicorn_without_the_parameter(self, capsys):
|
||||
class _OldUvicornConfig:
|
||||
def __init__(self, timeout_keep_alive=None):
|
||||
pass
|
||||
|
||||
with patch("uvicorn.Config", _OldUvicornConfig):
|
||||
args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000)
|
||||
assert "timeout_worker_healthcheck" not in args
|
||||
assert capsys.readouterr().out == ""
|
||||
|
||||
args = ProxyInitializationHelpers._get_default_unvicorn_init_args(
|
||||
"localhost", 8000, timeout_worker_healthcheck=15
|
||||
)
|
||||
assert "timeout_worker_healthcheck" not in args
|
||||
assert "requires uvicorn>=0.37.0" in capsys.readouterr().out
|
||||
|
||||
@staticmethod
|
||||
def _uvicorn_access_info_enabled(args: dict) -> bool:
|
||||
import logging
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue