mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
LITELLM_RUST already selects native paths elsewhere, so a missing gateway binary or a server Rust cannot front now warns and keeps the plain Python proxy instead of exiting Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
354 lines
15 KiB
Python
354 lines
15 KiB
Python
import gc
|
|
import importlib
|
|
import os
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import textwrap
|
|
import urllib.parse
|
|
from collections.abc import Iterator
|
|
from pathlib import Path
|
|
from typing import Final, cast
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
import uvicorn
|
|
from uvicorn.importer import import_from_string
|
|
from uvicorn.main import main as uvicorn_main
|
|
|
|
from gateway.launch import GATEWAY_APP, main, pool_database_url, python_server, uvicorn_argv
|
|
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
|
from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings
|
|
from litellm.proxy.proxy_server import app as proxy_app
|
|
from litellm.proxy.rust_forward.mode import AssignedPort, InheritedFd, Supervised, uvicorn_flags
|
|
from litellm.proxy.rust_forward.server import Listener
|
|
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
|
|
|
|
DB_ENV: Final = {
|
|
"DATABASE_HOST": "db.internal",
|
|
"DATABASE_PORT": "5432",
|
|
"DATABASE_USER": "litellm_pool",
|
|
"DATABASE_NAME": "litellm",
|
|
"DATABASE_PASSWORD": "p@ss",
|
|
}
|
|
|
|
|
|
def test_a_uvicorn_worker_reimports_the_launcher_without_loading_litellm():
|
|
reimported: Final = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-I",
|
|
"-c",
|
|
'import runpy, sys; runpy.run_module("gateway.launch", run_name="__mp_main__", alter_sys=True); '
|
|
'print("litellm" in sys.modules)',
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=120,
|
|
)
|
|
assert reimported.returncode == 0, reimported.stderr
|
|
assert reimported.stdout.strip() == "False"
|
|
|
|
|
|
def _free_port() -> int:
|
|
with socket.socket() as probe:
|
|
probe.bind(("127.0.0.1", 0))
|
|
return cast(tuple[str, int], probe.getsockname())[1]
|
|
|
|
|
|
def _fake_pooler(tmp_path: Path) -> Path:
|
|
script: Final = tmp_path / "fake-pgbouncer"
|
|
script.write_text(
|
|
textwrap.dedent(
|
|
f"""\
|
|
#!{sys.executable}
|
|
import configparser, select, socket, sys
|
|
if sys.argv[1:] == ["--version"]:
|
|
print("PgBouncer 1.25.2")
|
|
sys.exit(0)
|
|
ini = configparser.ConfigParser()
|
|
ini.read(sys.argv[1])
|
|
port = ini.getint("pgbouncer", "listen_port")
|
|
tcp = socket.socket()
|
|
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
tcp.bind(("127.0.0.1", port))
|
|
tcp.listen()
|
|
unix = socket.socket(socket.AF_UNIX)
|
|
unix.bind(ini.get("pgbouncer", "unix_socket_dir") + f"/.s.PGSQL.{{port}}")
|
|
unix.listen()
|
|
while True:
|
|
for ready in select.select([tcp, unix], [], [])[0]:
|
|
ready.accept()[0].close()
|
|
"""
|
|
)
|
|
)
|
|
script.chmod(0o700)
|
|
return script
|
|
|
|
|
|
def _query(url: str) -> dict[str, str]:
|
|
return dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query))
|
|
|
|
|
|
@pytest.fixture
|
|
def password_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[dict[str, str]]:
|
|
for var in (
|
|
"DATABASE_URL",
|
|
"IAM_TOKEN_DB_AUTH",
|
|
"AZURE_POSTGRESQL_AUTH",
|
|
"DATABASE_HOST_READ_REPLICA",
|
|
PGBOUNCER_POOLED_ENV_VAR,
|
|
):
|
|
monkeypatch.setenv(var, "")
|
|
monkeypatch.delenv(var)
|
|
for var, value in DB_ENV.items():
|
|
monkeypatch.setenv(var, value)
|
|
yield dict(DB_ENV)
|
|
os.environ.pop("DATABASE_URL", None)
|
|
|
|
|
|
def _minted_iam_token(token: str):
|
|
rds: Final = MagicMock()
|
|
rds.generate_db_auth_token.return_value = token
|
|
return patch("boto3.client", return_value=rds)
|
|
|
|
|
|
def _uvicorn_params(argv: tuple[str, ...]) -> dict[str, object]:
|
|
return uvicorn_main.make_context("uvicorn", list(argv)).params
|
|
|
|
|
|
class TestUvicornArgv:
|
|
@pytest.mark.parametrize(
|
|
("arguments", "listener"),
|
|
[
|
|
((), Listener.TCP),
|
|
(("--uds", "/tmp/gateway.sock"), Listener.UNIX_SOCKET),
|
|
(("--fd", "0"), Listener.INHERITED_FD),
|
|
(("--fd=17",), Listener.INHERITED_FD),
|
|
(("--ssl-keyfile", "unused"), Listener.TLS),
|
|
(("--ssl-ciphers", "TLSv1.2"), Listener.TLS),
|
|
],
|
|
)
|
|
def test_forwarding_sees_the_public_listener_uvicorn_would_open(
|
|
self, arguments: tuple[str, ...], listener: Listener
|
|
) -> None:
|
|
assert python_server(arguments).listener is listener
|
|
|
|
@pytest.mark.parametrize(
|
|
("argv", "trust"),
|
|
[
|
|
((), (True, None)),
|
|
(("--forwarded-allow-ips", "10.0.0.5"), (True, "10.0.0.5")),
|
|
(("--forwarded-allow-ips=*",), (True, "*")),
|
|
(("--no-proxy-headers",), (False, None)),
|
|
],
|
|
)
|
|
def test_forwarding_carries_operator_proxy_header_options_to_the_child(
|
|
self, argv: tuple[str, ...], trust: tuple[bool, str | None], monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
monkeypatch.delenv("FORWARDED_ALLOW_IPS", raising=False)
|
|
server: Final = python_server(("--host", "0.0.0.0", "--port", "4000", *argv))
|
|
assert (server.host, server.port) == ("0.0.0.0", 4000)
|
|
assert (server.proxy_headers, server.trusted_proxies) == trust
|
|
|
|
@pytest.mark.parametrize(
|
|
("argv", "environ"),
|
|
[(("--timeout-keep-alive", "2"), {}), ((), {"KEEPALIVE_TIMEOUT": "7"}), ((), {})],
|
|
)
|
|
def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with(
|
|
self, argv: tuple[str, ...], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False)
|
|
for name, value in environ.items():
|
|
monkeypatch.setenv(name, value)
|
|
served: Final = _uvicorn_params(uvicorn_argv(argv, os.environ))["timeout_keep_alive"]
|
|
assert python_server(argv).keepalive_seconds == served
|
|
|
|
def test_without_a_keepalive_rust_follows_uvicorns_own_default(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False)
|
|
assert python_server(()).keepalive_seconds == uvicorn.Config(app="", log_config=None).timeout_keep_alive
|
|
|
|
@pytest.mark.parametrize("argv", [(), ("--timeout-graceful-shutdown", "12")])
|
|
def test_rust_drains_for_pythons_graceful_timeout_plus_teardown(
|
|
self, argv: tuple[str, ...], monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "7.5")
|
|
flag: Final = _uvicorn_params(uvicorn_argv(argv, {}))["timeout_graceful_shutdown"]
|
|
graceful: Final = GracefulShutdownManager.get_timeout() if flag is None else float(str(flag))
|
|
assert python_server(argv).graceful_timeout_seconds == graceful
|
|
|
|
def test_supervised_listener_uses_the_inherited_loopback_socket(self):
|
|
supervised: Final = uvicorn_flags(Supervised(InheritedFd(17), True, "127.0.0.1"))
|
|
params: Final = _uvicorn_params((*uvicorn_argv(("--workers", "2"), {}), *supervised))
|
|
assert params["fd"] == 17
|
|
assert params["workers"] == 2
|
|
|
|
def test_assigned_port_listener_binds_only_loopback(self):
|
|
supervised: Final = uvicorn_flags(Supervised(AssignedPort(45556), True, "127.0.0.1"))
|
|
params: Final = _uvicorn_params((*uvicorn_argv(("--host", "0.0.0.0", "--port", "4000"), {}), *supervised))
|
|
assert (params["host"], params["port"], params["fd"]) == ("127.0.0.1", 45556, None)
|
|
|
|
@pytest.mark.parametrize("argv", [(), ("--proxy-headers",), ("--forwarded-allow-ips=*",)])
|
|
def test_private_hop_leaves_proxy_headers_to_the_app(self, argv: tuple[str, ...]):
|
|
params: Final = _uvicorn_params(
|
|
(*uvicorn_argv(argv, {}), *uvicorn_flags(Supervised(InheritedFd(17), True, "*")))
|
|
)
|
|
assert params["proxy_headers"] is False
|
|
|
|
def test_direct_listener_keeps_explicit_proxy_header_options(self):
|
|
params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {}))
|
|
assert params["proxy_headers"] is False
|
|
assert params["forwarded_allow_ips"] == "*"
|
|
|
|
def test_keepalive_env_reaches_uvicorn(self):
|
|
params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"}))
|
|
assert params["app"] == GATEWAY_APP
|
|
assert params["workers"] == 4
|
|
assert params["timeout_keep_alive"] == 75
|
|
|
|
def test_unset_env_keeps_the_uvicorn_default(self):
|
|
assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}))["timeout_keep_alive"] == 5
|
|
|
|
def test_an_explicit_flag_wins_over_the_env(self):
|
|
argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"})
|
|
assert _uvicorn_params(argv)["timeout_keep_alive"] == 30
|
|
|
|
def test_the_app_uvicorn_is_told_to_serve_is_the_trimmed_gateway(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(proxy_app.router, "lifespan_context", proxy_app.router.lifespan_context)
|
|
for key in (
|
|
"DATABASE_URL",
|
|
"DIRECT_URL",
|
|
"DATABASE_URL_READ_REPLICA",
|
|
"DATABASE_HOST",
|
|
"DATABASE_HOST_READ_REPLICA",
|
|
"DATABASE_PASSWORD",
|
|
"IAM_TOKEN_DB_AUTH",
|
|
"AZURE_POSTGRESQL_AUTH",
|
|
):
|
|
monkeypatch.delenv(key, raising=False)
|
|
|
|
params: Final = _uvicorn_params(uvicorn_argv((), {}))
|
|
assert params["factory"] is True
|
|
try:
|
|
served: Final = import_from_string(cast(str, params["app"]))()
|
|
finally:
|
|
gc.unfreeze()
|
|
assert served is importlib.import_module("gateway.main").app
|
|
|
|
|
|
class TestPoolDatabaseUrl:
|
|
def test_a_disabled_pooler_yields_no_url_to_install(self, password_env: dict[str, str]):
|
|
settings: Final = DatabaseURLSettings.from_env()
|
|
settings.apply_to_env()
|
|
environ: Final = {"DATABASE_URL": "postgresql://litellm_pool:p%40ss@db.internal:5432/litellm"}
|
|
assert pool_database_url(settings, PgBouncerSettings(enabled=False), environ) is None
|
|
|
|
def test_a_missing_upstream_url_is_reported(self, password_env: dict[str, str]):
|
|
environ: Final[dict[str, str]] = {}
|
|
outcome: Final = pool_database_url(DatabaseURLSettings.from_env(), PgBouncerSettings(enabled=True), environ)
|
|
assert isinstance(outcome, PgBouncerError)
|
|
assert "DATABASE_URL" in outcome.reason
|
|
|
|
def test_token_auth_hands_the_workers_the_pool_user_not_the_token(
|
|
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, socket_tempdir: Path
|
|
):
|
|
monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true")
|
|
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
|
|
port: Final = _free_port()
|
|
environ: Final = {"DATABASE_URL": "postgresql://litellm:MINTED_TOKEN@db.internal:5432/litellm"}
|
|
with _minted_iam_token("MINTED_TOKEN"):
|
|
outcome: Final = pool_database_url(
|
|
DatabaseURLSettings.from_env(),
|
|
PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(socket_tempdir))),
|
|
environ,
|
|
)
|
|
assert isinstance(outcome, str), outcome
|
|
pooled: Final = urllib.parse.urlsplit(outcome)
|
|
assert (pooled.username, pooled.hostname, pooled.port) == ("litellm_pgbouncer", "127.0.0.1", port)
|
|
assert "MINTED_TOKEN" not in outcome
|
|
|
|
|
|
class TestMain:
|
|
def test_workers_inherit_the_loopback_url_the_supervisor_installed(
|
|
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, socket_tempdir: Path
|
|
):
|
|
port: Final = _free_port()
|
|
monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true")
|
|
monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", str(port))
|
|
monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(socket_tempdir)))
|
|
monkeypatch.setenv("KEEPALIVE_TIMEOUT", "75")
|
|
served: Final[list[tuple[str, ...]]] = []
|
|
main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv)))
|
|
|
|
pooled: Final = os.environ["DATABASE_URL"]
|
|
assert urllib.parse.urlsplit(pooled).hostname == "127.0.0.1"
|
|
assert urllib.parse.urlsplit(pooled).port == port
|
|
assert urllib.parse.urlsplit(pooled).username == "litellm_pgbouncer"
|
|
assert "p%40ss" not in pooled
|
|
assert _query(pooled)["pgbouncer"] == "true"
|
|
assert _uvicorn_params(served[0])["timeout_keep_alive"] == 75
|
|
|
|
DatabaseURLSettings.from_env().apply_to_env()
|
|
assert urllib.parse.urlsplit(os.environ["DATABASE_URL"]).netloc == urllib.parse.urlsplit(pooled).netloc
|
|
assert _query(os.environ["DATABASE_URL"])["pgbouncer"] == "true"
|
|
|
|
def test_iam_workers_keep_the_loopback_url_instead_of_minting_their_own(
|
|
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, socket_tempdir: Path
|
|
):
|
|
port: Final = _free_port()
|
|
monkeypatch.delenv("DATABASE_PASSWORD")
|
|
monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true")
|
|
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
|
|
monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true")
|
|
monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", str(port))
|
|
monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(socket_tempdir)))
|
|
served: Final[list[tuple[str, ...]]] = []
|
|
with _minted_iam_token("SUPERVISOR_TOKEN"):
|
|
main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv)))
|
|
pooled: Final = os.environ["DATABASE_URL"]
|
|
assert urllib.parse.urlsplit(pooled).netloc.endswith(f"@127.0.0.1:{port}")
|
|
assert "SUPERVISOR_TOKEN" not in pooled
|
|
assert os.environ[PGBOUNCER_POOLED_ENV_VAR] == "true"
|
|
assert len(served) == 1
|
|
|
|
with _minted_iam_token("WORKER_TOKEN"):
|
|
DatabaseURLSettings.from_env().apply_to_env()
|
|
assert os.environ["DATABASE_URL"] == pooled
|
|
|
|
@pytest.mark.parametrize(
|
|
("rust", "warning"),
|
|
[
|
|
("false", ""),
|
|
(
|
|
"true",
|
|
"LITELLM_RUST does not support a Unix socket on the public port, "
|
|
"so Python serves the public port without Rust forwarding\n",
|
|
),
|
|
],
|
|
)
|
|
def test_python_alone_keeps_uvicorn_socket_options(
|
|
self,
|
|
rust: str,
|
|
warning: str,
|
|
password_env: dict[str, str],
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
capsys: pytest.CaptureFixture[str],
|
|
) -> None:
|
|
monkeypatch.setenv("LITELLM_RUST", rust)
|
|
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
|
|
monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False)
|
|
served: Final[list[tuple[str, ...]]] = []
|
|
main(("--uds", "/tmp/gateway.sock"), serve=lambda argv: served.append(tuple(argv)))
|
|
assert _uvicorn_params(served[0])["uds"] == "/tmp/gateway.sock"
|
|
assert capsys.readouterr().err == warning
|
|
|
|
def test_a_pooler_that_cannot_start_stops_the_gateway_before_uvicorn(
|
|
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
):
|
|
monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true")
|
|
monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(tmp_path / "missing-pgbouncer"))
|
|
served: Final[list[tuple[str, ...]]] = []
|
|
with pytest.raises(SystemExit) as stopped:
|
|
main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv)))
|
|
assert "missing-pgbouncer" in str(stopped.value)
|
|
assert served == []
|