mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge cf5e091f7b into d9c9d84298
This commit is contained in:
commit
c8652ee9c6
3 changed files with 115 additions and 14 deletions
|
|
@ -132,6 +132,7 @@ MAX_FILE_LIST_LIMIT: Final = 10000
|
|||
ANTHROPIC_DEFAULT_FILE_LIST_LIMIT: Final = 20
|
||||
DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
|
||||
DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = get_env_int("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)
|
||||
DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS: Final = 60
|
||||
GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS: Final = 10
|
||||
budget_reservation_disabled_info_emitted = False
|
||||
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = get_env_int("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1)
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from pydantic import ConfigDict
|
|||
|
||||
import litellm
|
||||
from litellm._logging import print_startup_message
|
||||
from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY
|
||||
from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY, DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS
|
||||
from litellm.proxy.db.pgbouncer import (
|
||||
PgBouncerError,
|
||||
PgBouncerSettings,
|
||||
|
|
@ -340,6 +340,7 @@ class ProxyInitializationHelpers:
|
|||
log_config: str | None = None,
|
||||
keepalive_timeout: int | None = None,
|
||||
timeout_worker_healthcheck: int | None = None,
|
||||
num_workers: int = 1,
|
||||
) -> dict:
|
||||
"""
|
||||
Get the arguments for `uvicorn` worker
|
||||
|
|
@ -369,15 +370,28 @@ class ProxyInitializationHelpers:
|
|||
uvicorn_args["log_level"] = resolve_log_level(litellm_log)
|
||||
if keepalive_timeout is not None:
|
||||
uvicorn_args["timeout_keep_alive"] = keepalive_timeout
|
||||
if timeout_worker_healthcheck is not None:
|
||||
if "timeout_worker_healthcheck" in inspect.signature(uvicorn.Config.__init__).parameters:
|
||||
uvicorn_args["timeout_worker_healthcheck"] = timeout_worker_healthcheck
|
||||
else:
|
||||
print_startup_message(
|
||||
f"\033[1;33mLiteLLM Proxy: --timeout_worker_healthcheck "
|
||||
f"requires uvicorn>=0.37.0, but installed uvicorn=={uvicorn.__version__}. "
|
||||
f"Ignoring the flag.\033[0m",
|
||||
)
|
||||
resolved_worker_healthcheck_timeout: Final = (
|
||||
DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS
|
||||
if timeout_worker_healthcheck is None
|
||||
else timeout_worker_healthcheck
|
||||
)
|
||||
uvicorn_supports_worker_healthcheck: Final = (
|
||||
"timeout_worker_healthcheck" in inspect.signature(uvicorn.Config.__init__).parameters
|
||||
)
|
||||
if uvicorn_supports_worker_healthcheck:
|
||||
uvicorn_args["timeout_worker_healthcheck"] = resolved_worker_healthcheck_timeout
|
||||
elif timeout_worker_healthcheck is not None:
|
||||
print_startup_message(
|
||||
f"\033[1;33mLiteLLM Proxy: --timeout_worker_healthcheck "
|
||||
f"requires uvicorn>=0.37.0, but installed uvicorn=={uvicorn.__version__}. "
|
||||
f"Ignoring the flag.\033[0m",
|
||||
)
|
||||
elif num_workers > 1:
|
||||
print_startup_message(
|
||||
f"\033[1;33mLiteLLM Proxy: uvicorn=={uvicorn.__version__} gives each worker a fixed 5 s to answer "
|
||||
f"its health ping, so a worker that starts slower is killed with 'Child process died'. Install "
|
||||
f"uvicorn>=0.37.0 to get the {DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS} s default.\033[0m",
|
||||
)
|
||||
return uvicorn_args
|
||||
|
||||
_get_default_unvicorn_init_args = get_default_unvicorn_init_args
|
||||
|
|
@ -1044,8 +1058,11 @@ class ProxyInitializationHelpers:
|
|||
type=int,
|
||||
help=(
|
||||
"Set the uvicorn worker health-check timeout in seconds (uvicorn timeout_worker_healthcheck parameter). "
|
||||
"Requires uvicorn>=0.37.0. Only applies when running uvicorn directly with --num_workers>1; "
|
||||
"ignored under --run_gunicorn / --run_hypercorn."
|
||||
f"Defaults to {DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS} instead of uvicorn's 5: a spawned worker is "
|
||||
"killed when it misses one healthcheck ping, and importing the proxy takes longer than 5 s on a loaded "
|
||||
"host, so the lower default crash-loops with 'Child process died'. The same value is the deadline for a "
|
||||
"replacement worker to finish starting on SIGHUP. Requires uvicorn>=0.37.0. Only applies when running "
|
||||
"uvicorn directly with --num_workers>1; ignored under --run_gunicorn / --run_hypercorn."
|
||||
),
|
||||
envvar="TIMEOUT_WORKER_HEALTHCHECK",
|
||||
)
|
||||
|
|
@ -1657,13 +1674,14 @@ def run_server(
|
|||
f"(pid {metrics_process.pid})\033[0m"
|
||||
)
|
||||
|
||||
running_uvicorn: Final = run_gunicorn is False and run_hypercorn is False
|
||||
running_uvicorn: Final = run_gunicorn is False and run_hypercorn is False and run_granian is False
|
||||
uvicorn_args: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args(
|
||||
host=host,
|
||||
port=port,
|
||||
log_config=log_config,
|
||||
keepalive_timeout=keepalive_timeout,
|
||||
timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None),
|
||||
num_workers=(num_workers if running_uvicorn else 1),
|
||||
)
|
||||
# Optional: recycle uvicorn workers after N requests
|
||||
if max_requests_before_restart is not None:
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ from uvicorn.importer import import_from_string
|
|||
|
||||
import litellm
|
||||
from litellm._logging import turn_on_json
|
||||
from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
|
||||
from litellm.constants import DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS, GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
|
||||
from litellm.proxy import proxy_cli
|
||||
from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server
|
||||
from litellm.proxy.shutdown.graceful_shutdown_manager import DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT
|
||||
|
|
@ -160,6 +160,45 @@ class TestProxyInitializationHelpers:
|
|||
)
|
||||
assert args["timeout_worker_healthcheck"] == 15
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("flag_value", "expected"),
|
||||
[(None, DEFAULT_WORKER_HEALTHCHECK_TIMEOUT_SECONDS), (15, 15)],
|
||||
)
|
||||
def test_uvicorn_worker_healthcheck_timeout_reaches_uvicorn(self, flag_value: int | None, expected: int):
|
||||
args: Final = ProxyInitializationHelpers._get_default_unvicorn_init_args(
|
||||
"localhost", 8000, timeout_worker_healthcheck=flag_value
|
||||
)
|
||||
|
||||
assert uvicorn.Config(**args).timeout_worker_healthcheck == expected
|
||||
|
||||
def test_worker_healthcheck_timeout_on_uvicorn_without_the_parameter(
|
||||
self, capsys: pytest.CaptureFixture[str]
|
||||
):
|
||||
class _OldUvicornConfig:
|
||||
def __init__(self, timeout_keep_alive: int | None = None):
|
||||
pass
|
||||
|
||||
with patch("uvicorn.Config", _OldUvicornConfig):
|
||||
single_worker_args: Final = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000)
|
||||
assert "timeout_worker_healthcheck" not in single_worker_args
|
||||
assert capsys.readouterr().out == ""
|
||||
|
||||
multi_worker_args: Final = ProxyInitializationHelpers._get_default_unvicorn_init_args(
|
||||
"localhost", 8000, num_workers=2
|
||||
)
|
||||
assert "timeout_worker_healthcheck" not in multi_worker_args
|
||||
multi_worker_output: Final = capsys.readouterr().out
|
||||
assert "fixed 5 s" in multi_worker_output
|
||||
assert "uvicorn>=0.37.0" in multi_worker_output
|
||||
|
||||
flagged_args: Final = ProxyInitializationHelpers._get_default_unvicorn_init_args(
|
||||
"localhost", 8000, timeout_worker_healthcheck=15, num_workers=2
|
||||
)
|
||||
assert "timeout_worker_healthcheck" not in flagged_args
|
||||
flagged_output: Final = capsys.readouterr().out
|
||||
assert "requires uvicorn>=0.37.0" in flagged_output
|
||||
assert "fixed 5 s" not in flagged_output
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw_timeout", "expected"),
|
||||
[("7.5", 7.5), ("0", 0.0), (None, DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT)],
|
||||
|
|
@ -1530,6 +1569,7 @@ class TestProxyInitializationHelpers:
|
|||
log_config=None,
|
||||
keepalive_timeout=30,
|
||||
timeout_worker_healthcheck=None,
|
||||
num_workers=1,
|
||||
)
|
||||
mock_uvicorn_run.assert_called_once()
|
||||
|
||||
|
|
@ -1606,6 +1646,7 @@ class TestProxyInitializationHelpers:
|
|||
log_config=None,
|
||||
keepalive_timeout=None,
|
||||
timeout_worker_healthcheck=15,
|
||||
num_workers=1,
|
||||
)
|
||||
|
||||
@patch("uvicorn.run")
|
||||
|
|
@ -2415,6 +2456,47 @@ class TestQueryEngineReaperWiring:
|
|||
result = runner.invoke(run_server, args)
|
||||
return result, mock_uvicorn_run, mock_start_reaper
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("args", "warns"),
|
||||
[
|
||||
(["--local", "--num_workers", "2"], True),
|
||||
(["--local", "--num_workers", "1"], False),
|
||||
(["--local", "--num_workers", "2", "--run_granian"], False),
|
||||
],
|
||||
)
|
||||
def test_old_uvicorn_warns_only_for_multi_worker_uvicorn(self, args: list[str], warns: bool):
|
||||
from click.testing import CliRunner
|
||||
|
||||
from litellm.proxy.proxy_cli import run_server
|
||||
|
||||
class _OldUvicornConfig:
|
||||
def __init__(self, timeout_keep_alive: int | None = None):
|
||||
pass
|
||||
|
||||
clean_env: Final = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")}
|
||||
with (
|
||||
patch.dict(os.environ, clean_env, clear=True),
|
||||
patch.dict(
|
||||
"sys.modules",
|
||||
{
|
||||
"proxy_server": MagicMock(
|
||||
app=MagicMock(),
|
||||
ProxyConfig=MagicMock(),
|
||||
KeyManagementSettings=MagicMock(),
|
||||
save_worker_config=MagicMock(),
|
||||
)
|
||||
},
|
||||
),
|
||||
patch("uvicorn.Config", _OldUvicornConfig),
|
||||
patch("uvicorn.run"),
|
||||
patch("litellm.proxy.proxy_cli.start_query_engine_reaper"),
|
||||
patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.init_granian_server"),
|
||||
):
|
||||
result: Final = CliRunner().invoke(run_server, args)
|
||||
|
||||
assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}"
|
||||
assert ("uvicorn>=0.37.0" in result.output) is warns
|
||||
|
||||
def test_multi_worker_uvicorn_starts_reaper(self):
|
||||
result, mock_uvicorn_run, mock_start_reaper = self._invoke_run_server(
|
||||
["--local", "--num_workers", "2"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue