From b6efbc03de65418dab42132ecd3a8c2415472fda Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:11:18 +0000 Subject: [PATCH 01/17] feat(proxy): supervise the opt-in Rust forwarding server --- litellm/proxy/proxy_cli.py | 35 +++++++- litellm/proxy/rust_gateway.py | 115 ++++++++++++++++++++++++++ tests/unit/proxy/test_rust_gateway.py | 62 ++++++++++++++ 3 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 litellm/proxy/rust_gateway.py create mode 100644 tests/unit/proxy/test_rust_gateway.py diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 0fbfa0b06a7..94878bd1af2 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -419,6 +419,7 @@ class ProxyInitializationHelpers: import litellm from litellm._logging import _get_uvicorn_json_log_config, resolve_log_level + from litellm.proxy.rust_gateway import inherited_fd from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager uvicorn_args: Final = { @@ -427,6 +428,7 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), + **({"fd": inherited_fd()} if inherited_fd() is not None else {}), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -752,7 +754,9 @@ class ProxyInitializationHelpers: ) graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout() gunicorn_options: Final = { - "bind": f"{host}:{port}", + "bind": ( + f"fd://{os.environ['_LITELLM_GATEWAY_FD']}" if "_LITELLM_GATEWAY_FD" in os.environ else f"{host}:{port}" + ), "workers": num_workers, # default is 1 "worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, @@ -1039,6 +1043,13 @@ class ProxyInitializationHelpers: is_flag=True, help="Starts proxy via gunicorn, instead of uvicorn (better for managing multiple workers)", ) +@click.option( + "--rust_gateway", + envvar="LITELLM_RUST_GATEWAY", + default=False, + is_flag=True, + help="Run the Rust forwarding server on the public port, with Python on loopback", +) @click.option( "--run_hypercorn", default=False, @@ -1241,6 +1252,7 @@ def run_server( setup, version, run_gunicorn, + rust_gateway, run_hypercorn, run_granian, ssl_keyfile_path, @@ -1280,6 +1292,27 @@ def run_server( run_setup_wizard() return + if rust_gateway and not any( + (version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data) + ): + from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise + + binary: Final = gateway_binary() + error: Final = startup_error( + binary, + { + "--ssl_keyfile_path": ssl_keyfile_path, + "--ssl_certfile_path": ssl_certfile_path, + "--ciphers": ciphers, + "--run_hypercorn": run_hypercorn, + "--run_granian": run_granian, + }, + ) + if error is not None: + raise click.ClickException(error) + assert binary is not None + raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, port)) + args: Final = locals() try: from litellm.proxy.proxy_server import ( diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py new file mode 100644 index 00000000000..830a4ab3d3d --- /dev/null +++ b/litellm/proxy/rust_gateway.py @@ -0,0 +1,115 @@ +import asyncio +import os +import shutil +import signal +import socket +import sys +import sysconfig +from collections.abc import Mapping, Sequence +from typing import Final + +from pydantic import TypeAdapter + +GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" + + +def gateway_binary() -> str | None: + name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway" + return shutil.which(name, path=sysconfig.get_path("scripts")) or shutil.which(name) + + +def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: + flag: Final = next((name for name, value in unsupported.items() if value), None) + if flag is not None: + return f"--rust_gateway does not support {flag}" + if os.name != "posix": + return "--rust_gateway requires POSIX inherited sockets and process signals" + if binary is None: + return "--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment" + return None + + +def child_argv(argv: Sequence[str], port: int) -> tuple[str, ...]: + return (*(arg for arg in argv if arg != "--rust_gateway"), "--host", "127.0.0.1", "--port", str(port)) + + +def inherited_fd() -> int | None: + value: Final = os.environ.get(GATEWAY_FD) + return int(value) if value is not None else None + + +def terminate_group(process: asyncio.subprocess.Process) -> None: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + + +async def _supervise(binary: str, command: Sequence[str], host: str, port: int) -> int: + loop: Final = asyncio.get_running_loop() + stopped: Final[asyncio.Future[signal.Signals]] = loop.create_future() + + def stop(signum: signal.Signals) -> None: + if not stopped.done(): + stopped.set_result(signum) + + for signum in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(signum, stop, signum) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen() + _, upstream_port = TypeAdapter(tuple[str, int]).validate_python(listener.getsockname()) + python: Final = await asyncio.create_subprocess_exec( + *child_argv(command, upstream_port), + env={ + **os.environ, + "LITELLM_RUST_GATEWAY": "false", + GATEWAY_FD: str(listener.fileno()), + "FORWARDED_ALLOW_IPS": "127.0.0.1", + }, + pass_fds=(listener.fileno(),), + start_new_session=True, + ) + try: + rust: Final = await asyncio.create_subprocess_exec( + binary, + env={ + **os.environ, + "HOST": host, + "PORT": str(port), + "LITELLM_FORWARD_UPSTREAM": f"http://127.0.0.1:{upstream_port}", + }, + start_new_session=True, + ) + except OSError: + terminate_group(python) + await python.wait() + raise + python_done: Final = asyncio.create_task(python.wait()) + rust_done: Final = asyncio.create_task(rust.wait()) + try: + done, _ = await asyncio.wait((stopped, python_done, rust_done), return_when=asyncio.FIRST_COMPLETED) + if python_done in done or rust_done in done: + return 1 + rust.send_signal(stopped.result()) + draining, _ = await asyncio.wait((python_done, rust_done), return_when=asyncio.FIRST_COMPLETED) + if python_done in draining or rust_done.result() != 0: + return 1 + python.send_signal(stopped.result()) + return 0 if await python_done == 0 else 1 + finally: + terminate_group(rust) + terminate_group(python) + await asyncio.gather(python_done, rust_done) + + +def supervise(binary: str, command: Sequence[str], host: str, port: int) -> int: + try: + return asyncio.run(_supervise(binary, command, host, port)) + except OSError as error: + sys.stderr.write(f"Rust gateway startup failed: {error}\n") + return 1 + + +def cli_command(argv: Sequence[str]) -> tuple[str, ...]: + return (sys.executable, "-m", "litellm.proxy.proxy_cli", *argv) diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py new file mode 100644 index 00000000000..c9992d07d14 --- /dev/null +++ b/tests/unit/proxy/test_rust_gateway.py @@ -0,0 +1,62 @@ +import sys +from pathlib import Path +from typing import Final + +import pytest +from click.testing import CliRunner + +from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server +from litellm.proxy.rust_gateway import child_argv, cli_command, gateway_binary, inherited_fd, startup_error + + +@pytest.mark.parametrize( + "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] +) +@pytest.mark.parametrize("enable", ["flag", "environment"]) +def test_unsupported_modes_fail_before_starting_children( + flag: str, enable: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("LITELLM_RUST_GATEWAY", "true" if enable == "environment" else "false") + arguments: Final = ([] if enable == "environment" else ["--rust_gateway"]) + [flag] + values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] + result: Final = CliRunner().invoke(run_server, arguments + values) + assert result.exit_code != 0 + assert f"--rust_gateway does not support {flag}" in result.output + + +def test_child_arguments_preserve_options_and_force_private_listener() -> None: + arguments: Final = ("--rust_gateway", "--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") + params: Final = run_server.make_context("litellm", list(child_argv(arguments, 45555))).params + assert params["host"] == "127.0.0.1" + assert params["port"] == 45555 + assert params["num_workers"] == 2 + assert params["reload"] is True + assert params["rust_gateway"] is False + assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli") + + +def test_missing_binary_gives_install_instruction() -> None: + assert startup_error(None, {}) == ( + "--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment" + ) + + +def test_current_environment_binary_wins_over_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + binary: Final = tmp_path / "litellm-gateway" + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o755) + monkeypatch.setattr("sysconfig.get_path", lambda _: str(tmp_path)) + monkeypatch.setenv("PATH", "") + assert gateway_binary() == str(binary) + + +@pytest.mark.parametrize("fd", [None, "17"]) +def test_inherited_socket_reaches_uvicorn(fd: str | None, monkeypatch: pytest.MonkeyPatch) -> None: + if fd is None: + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + else: + monkeypatch.setenv("_LITELLM_GATEWAY_FD", fd) + expected: Final = int(fd) if fd is not None else None + options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555) + assert inherited_fd() == expected + assert options.get("fd") == expected From b0ad7c4693a72246bde46811e46e1ee775be275b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:39:14 +0000 Subject: [PATCH 02/17] fix(proxy): preserve gateway metrics and default-port fallback --- litellm/proxy/proxy_cli.py | 25 +++--- litellm/proxy/rust_gateway.py | 5 ++ .../configuration/test_rust_gateway.py | 89 +++++++++++++++++++ tests/unit/proxy/test_rust_gateway.py | 22 ++++- 4 files changed, 130 insertions(+), 11 deletions(-) create mode 100644 tests/integration/configuration/test_rust_gateway.py diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 94878bd1af2..203358cba97 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -819,7 +819,7 @@ class ProxyInitializationHelpers: _run_ollama_serve = run_ollama_serve @staticmethod - def is_port_in_use(port): + def is_port_in_use(port: int): import socket with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: @@ -1221,8 +1221,8 @@ class ProxyInitializationHelpers: ) def run_server( cli_args, - host, - port, + host: str, + port: int, api_base, api_version, model, @@ -1252,7 +1252,7 @@ def run_server( setup, version, run_gunicorn, - rust_gateway, + rust_gateway: bool, run_hypercorn, run_granian, ssl_keyfile_path, @@ -1298,7 +1298,7 @@ def run_server( from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise binary: Final = gateway_binary() - error: Final = startup_error( + gateway_error: Final = startup_error( binary, { "--ssl_keyfile_path": ssl_keyfile_path, @@ -1308,10 +1308,13 @@ def run_server( "--run_granian": run_granian, }, ) - if error is not None: - raise click.ClickException(error) + if gateway_error is not None: + raise click.ClickException(gateway_error) assert binary is not None - raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, port)) + public_port: Final = ( + random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port + ) + raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port)) args: Final = locals() try: @@ -1761,15 +1764,17 @@ def run_server( if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None: from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process + from litellm.proxy.rust_gateway import metrics_host + metrics_bind_host: Final = metrics_host(host) try: metrics_process: Final = start_metrics_server_process( - host=host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir + host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir ) except MetricsServerStartupError as error: raise click.ClickException(str(error)) from error print_startup_message( - f"\033[1;32mLiteLLM: Serving Prometheus metrics on {host}:{prometheus_metrics_port}/metrics " + f"\033[1;32mLiteLLM: Serving Prometheus metrics on {metrics_bind_host}:{prometheus_metrics_port}/metrics " f"(pid {metrics_process.pid})\033[0m" ) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 830a4ab3d3d..9b7ba98c161 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -38,6 +38,10 @@ def inherited_fd() -> int | None: return int(value) if value is not None else None +def metrics_host(host: str) -> str: + return os.environ.get("_LITELLM_GATEWAY_METRICS_HOST", host) if inherited_fd() is not None else host + + def terminate_group(process: asyncio.subprocess.Process) -> None: try: os.killpg(process.pid, signal.SIGTERM) @@ -65,6 +69,7 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int) **os.environ, "LITELLM_RUST_GATEWAY": "false", GATEWAY_FD: str(listener.fileno()), + "_LITELLM_GATEWAY_METRICS_HOST": host, "FORWARDED_ALLOW_IPS": "127.0.0.1", }, pass_fds=(listener.fileno(),), diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py new file mode 100644 index 00000000000..bb0119f9398 --- /dev/null +++ b/tests/integration/configuration/test_rust_gateway.py @@ -0,0 +1,89 @@ +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Final + +import httpx +import pytest + +from tests.integration._support.client import Gateway, eventually +from tests.integration._support.ports import allocate_port +from tests.integration._support.process import stop_root_process + + +def ready(client: httpx.Client) -> bool: + try: + return client.get("/health/liveliness").status_code == 200 + except httpx.TransportError: + return False + + +@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) +def test_gateway_preserves_metrics_bind_and_mcp_client_ip( + gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...] +) -> None: + config: Final = tmp_path / "config.yaml" + config.write_text( + json.dumps( + { + "model_list": [], + "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY", "use_x_forwarded_for": False}, + } + ) + ) + metrics_port: Final = allocate_port() + public_port: Final = allocate_port() + environment: Final = { + **{key: value for key, value in os.environ.items() if key not in ("DATABASE_URL", "DATABASE_URL_READ_REPLICA")}, + "LITELLM_MASTER_KEY": gateway.key, + "STORE_MODEL_IN_DB": "False", + "PYTHON_DOTENV_DISABLED": "1", + } + with (tmp_path / "proxy.log").open("wb") as output: + with subprocess.Popen( + ( + sys.executable, + "-I", + "-m", + "litellm.proxy.proxy_cli", + "--rust_gateway", + "--config", + str(config), + "--host", + "0.0.0.0", + "--port", + str(public_port), + "--num_workers", + "2", + "--timeout_worker_healthcheck", + "120", + "--prometheus_metrics_port", + str(metrics_port), + *server_arguments, + ), + env=environment, + stdout=output, + stderr=output, + ) as supervisor: + try: + with httpx.Client( + base_url=f"http://127.0.0.1:{public_port}", + transport=httpx.HTTPTransport(local_address="127.0.0.2"), + trust_env=False, + follow_redirects=True, + timeout=5, + ) as external: + assert eventually(lambda: ready(external), bool, seconds=180), (tmp_path / "proxy.log").read_text() + response: Final = external.get(f"http://127.0.0.2:{metrics_port}/metrics") + assert response.status_code == 200, response.text + client_ip: Final = external.get( + "/v1/mcp/network/client-ip", + headers={"Authorization": f"Bearer {gateway.key}", "X-Forwarded-For": "1.2.3.4"}, + ) + assert client_ip.status_code == 200, client_ip.text + assert client_ip.json() == {"ip": "127.0.0.2"} + finally: + assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text() + assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text() diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index c9992d07d14..c1d03d696b9 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -6,7 +6,14 @@ import pytest from click.testing import CliRunner from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server -from litellm.proxy.rust_gateway import child_argv, cli_command, gateway_binary, inherited_fd, startup_error +from litellm.proxy.rust_gateway import ( + child_argv, + cli_command, + gateway_binary, + inherited_fd, + metrics_host, + startup_error, +) @pytest.mark.parametrize( @@ -41,6 +48,19 @@ def test_missing_binary_gives_install_instruction() -> None: ) +@pytest.mark.parametrize("supervised", [False, True]) +@pytest.mark.parametrize("host", ["0.0.0.0", "192.0.2.1", "::"]) +def test_metrics_keep_configured_host_separate_from_private_listener( + supervised: bool, host: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", host) + if supervised: + monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17") + else: + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + assert metrics_host("127.0.0.1") == (host if supervised else "127.0.0.1") + + def test_current_environment_binary_wins_over_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: binary: Final = tmp_path / "litellm-gateway" binary.write_text("#!/bin/sh\nexit 0\n") From a841047baaae56d996a70171678782a88069e6b9 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:49:20 +0000 Subject: [PATCH 03/17] test(proxy): cover gateway shutdown and child failures --- tests/unit/proxy/test_rust_gateway.py | 96 +++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index c1d03d696b9..969cfd12da4 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -1,6 +1,10 @@ +import asyncio +import signal +import socket import sys from pathlib import Path from typing import Final +from unittest.mock import MagicMock, Mock, call, patch import pytest from click.testing import CliRunner @@ -13,6 +17,7 @@ from litellm.proxy.rust_gateway import ( inherited_fd, metrics_host, startup_error, + supervise, ) @@ -80,3 +85,94 @@ def test_inherited_socket_reaches_uvicorn(fd: str | None, monkeypatch: pytest.Mo options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555) assert inherited_fd() == expected assert options.get("fd") == expected + + +@pytest.mark.parametrize( + ("event", "python_exit", "rust_exit", "expected"), + [ + ("signal", 0, 0, 0), + ("signal", 1, 0, 1), + ("signal", 0, 1, 1), + ("python-exit", 0, 0, 1), + ("rust-exit", 0, 0, 1), + ("python-during-drain", 1, 0, 1), + ("spawn-error", 0, 0, 1), + ], +) +def test_supervisor_drains_before_stopping_python_and_reaps_both_children( + event: str, python_exit: int, rust_exit: int, expected: int, capsys: pytest.CaptureFixture[str] +) -> None: + loop: Final = asyncio.new_event_loop() + handlers: Final = Mock() + listener: Final = MagicMock(spec=socket.socket) + listener.__enter__.return_value = listener + listener.getsockname.return_value = ("127.0.0.1", 45555) + listener.fileno.return_value = 17 + children: Final = {} + sent: Final = Mock() + + def current_loop(): + return Mock(wraps=loop, add_signal_handler=handlers) + + async def spawn(*argv, **kwargs): + done: Final = asyncio.events.get_running_loop().create_future() + pid: Final = 101 if argv[0] == "python" else 102 + + async def wait() -> int: + return await done + + def send_signal(signum: signal.Signals) -> None: + sent(pid, signum) + if pid == 102 and event == "python-during-drain": + children[101][1].set_result(python_exit) + return + if pid == 101: + assert children[102][1].done(), "Python must remain alive until Rust has drained" + done.set_result(python_exit if pid == 101 else rust_exit) + + process: Final = Mock(spec=asyncio.subprocess.Process, pid=pid, wait=wait, send_signal=send_signal) + if pid == 102 and event == "spawn-error": + raise OSError("gateway cannot start") + children[pid] = (process, done) + if pid == 102: + if event == "python-exit": + children[101][1].set_result(python_exit) + elif event == "rust-exit": + done.set_result(rust_exit) + else: + _, stop, signum = handlers.call_args_list[0].args + stop(signum) + stop(signum) + return process + + def kill_group(pid: int, signum: signal.Signals) -> None: + assert signum == signal.SIGTERM + _, done = children[pid] + if done.done(): + raise ProcessLookupError + done.set_result(1) + + with ( + patch("asyncio.events.new_event_loop", return_value=loop), + patch("asyncio.get_running_loop", side_effect=current_loop), + patch("asyncio.create_subprocess_exec", side_effect=spawn) as launch, + patch("socket.socket", return_value=listener), + patch("os.killpg", side_effect=kill_group) as kill, + ): + assert supervise("gateway", ("python", "--rust_gateway"), "0.0.0.0", 4000) == expected + + assert all(done.done() for _, done in children.values()) + assert call(101, signal.SIGTERM) in kill.call_args_list + python_call, rust_call = launch.call_args_list + assert python_call.args == ("python", "--host", "127.0.0.1", "--port", "45555") + assert python_call.kwargs["pass_fds"] == (17,) + assert python_call.kwargs["env"]["LITELLM_RUST_GATEWAY"] == "false" + assert python_call.kwargs["env"]["FORWARDED_ALLOW_IPS"] == "127.0.0.1" + assert rust_call.kwargs["env"]["LITELLM_FORWARD_UPSTREAM"] == "http://127.0.0.1:45555" + assert rust_call.kwargs["env"]["PORT"] == "4000" + if event == "spawn-error": + assert "Rust gateway startup failed: gateway cannot start" in capsys.readouterr().err + elif event == "signal" and rust_exit == 0: + assert sent.call_args_list == [call(102, signal.SIGTERM), call(101, signal.SIGTERM)] + else: + assert call(101, signal.SIGTERM) not in sent.call_args_list From e1c2be72705127aab2ea6dc0d8145470b9cc37c9 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 02:43:32 +0000 Subject: [PATCH 04/17] fix(proxy): retain configured trusted proxies behind Rust --- litellm/proxy/rust_gateway.py | 9 ++++++++- .../configuration/test_rust_gateway.py | 8 +++++--- tests/unit/proxy/test_rust_gateway.py | 19 ++++++++++++++++++- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 9b7ba98c161..85a850978e8 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -18,6 +18,13 @@ def gateway_binary() -> str | None: return shutil.which(name, path=sysconfig.get_path("scripts")) or shutil.which(name) +def forwarded_allow_ips(trusted: str | None) -> str: + if trusted == "*": + return trusted + hosts: Final = tuple(host.strip() for host in (trusted or "").split(",") if host.strip()) + return ",".join(dict.fromkeys(("127.0.0.1", *hosts))) + + def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: flag: Final = next((name for name, value in unsupported.items() if value), None) if flag is not None: @@ -70,7 +77,7 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int) "LITELLM_RUST_GATEWAY": "false", GATEWAY_FD: str(listener.fileno()), "_LITELLM_GATEWAY_METRICS_HOST": host, - "FORWARDED_ALLOW_IPS": "127.0.0.1", + "FORWARDED_ALLOW_IPS": forwarded_allow_ips(os.environ.get("FORWARDED_ALLOW_IPS")), }, pass_fds=(listener.fileno(),), start_new_session=True, diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index bb0119f9398..aa5ce9825ae 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -21,8 +21,9 @@ def ready(client: httpx.Client) -> bool: @pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) +@pytest.mark.parametrize(("trusted_proxies", "expected_ip"), [("", "127.0.0.2"), ("127.0.0.2", "198.51.100.10")]) def test_gateway_preserves_metrics_bind_and_mcp_client_ip( - gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...] + gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...], trusted_proxies: str, expected_ip: str ) -> None: config: Final = tmp_path / "config.yaml" config.write_text( @@ -40,6 +41,7 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( "LITELLM_MASTER_KEY": gateway.key, "STORE_MODEL_IN_DB": "False", "PYTHON_DOTENV_DISABLED": "1", + "FORWARDED_ALLOW_IPS": trusted_proxies, } with (tmp_path / "proxy.log").open("wb") as output: with subprocess.Popen( @@ -80,10 +82,10 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( assert response.status_code == 200, response.text client_ip: Final = external.get( "/v1/mcp/network/client-ip", - headers={"Authorization": f"Bearer {gateway.key}", "X-Forwarded-For": "1.2.3.4"}, + headers={"Authorization": f"Bearer {gateway.key}", "X-Forwarded-For": "198.51.100.10"}, ) assert client_ip.status_code == 200, client_ip.text - assert client_ip.json() == {"ip": "127.0.0.2"} + assert client_ip.json() == {"ip": expected_ip} finally: assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text() assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text() diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index 969cfd12da4..44771ff9616 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -13,6 +13,7 @@ from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server from litellm.proxy.rust_gateway import ( child_argv, cli_command, + forwarded_allow_ips, gateway_binary, inherited_fd, metrics_host, @@ -53,6 +54,21 @@ def test_missing_binary_gives_install_instruction() -> None: ) +@pytest.mark.parametrize( + ("trusted", "expected"), + [ + (None, "127.0.0.1"), + ("", "127.0.0.1"), + ("192.0.2.1", "127.0.0.1,192.0.2.1"), + ("127.0.0.1, 192.0.2.0/24", "127.0.0.1,192.0.2.0/24"), + ("*", "*"), + ("127.0.0.1,*", "127.0.0.1,*"), + ], +) +def test_private_hop_preserves_configured_trusted_proxies(trusted: str | None, expected: str) -> None: + assert forwarded_allow_ips(trusted) == expected + + @pytest.mark.parametrize("supervised", [False, True]) @pytest.mark.parametrize("host", ["0.0.0.0", "192.0.2.1", "::"]) def test_metrics_keep_configured_host_separate_from_private_listener( @@ -158,6 +174,7 @@ def test_supervisor_drains_before_stopping_python_and_reaps_both_children( patch("asyncio.create_subprocess_exec", side_effect=spawn) as launch, patch("socket.socket", return_value=listener), patch("os.killpg", side_effect=kill_group) as kill, + patch.dict("os.environ", {"FORWARDED_ALLOW_IPS": "192.0.2.1"}), ): assert supervise("gateway", ("python", "--rust_gateway"), "0.0.0.0", 4000) == expected @@ -167,7 +184,7 @@ def test_supervisor_drains_before_stopping_python_and_reaps_both_children( assert python_call.args == ("python", "--host", "127.0.0.1", "--port", "45555") assert python_call.kwargs["pass_fds"] == (17,) assert python_call.kwargs["env"]["LITELLM_RUST_GATEWAY"] == "false" - assert python_call.kwargs["env"]["FORWARDED_ALLOW_IPS"] == "127.0.0.1" + assert python_call.kwargs["env"]["FORWARDED_ALLOW_IPS"] == "127.0.0.1,192.0.2.1" assert rust_call.kwargs["env"]["LITELLM_FORWARD_UPSTREAM"] == "http://127.0.0.1:45555" assert rust_call.kwargs["env"]["PORT"] == "4000" if event == "spawn-error": From bb89ffea75b259ccfc186d422e4ad6b441b7629a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 02:49:29 +0000 Subject: [PATCH 05/17] fix(proxy): enforce specified loopback-only forwarding trust --- litellm/proxy/rust_gateway.py | 9 +-------- .../configuration/test_rust_gateway.py | 8 +++----- tests/unit/proxy/test_rust_gateway.py | 19 +------------------ 3 files changed, 5 insertions(+), 31 deletions(-) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 85a850978e8..9b7ba98c161 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -18,13 +18,6 @@ def gateway_binary() -> str | None: return shutil.which(name, path=sysconfig.get_path("scripts")) or shutil.which(name) -def forwarded_allow_ips(trusted: str | None) -> str: - if trusted == "*": - return trusted - hosts: Final = tuple(host.strip() for host in (trusted or "").split(",") if host.strip()) - return ",".join(dict.fromkeys(("127.0.0.1", *hosts))) - - def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: flag: Final = next((name for name, value in unsupported.items() if value), None) if flag is not None: @@ -77,7 +70,7 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int) "LITELLM_RUST_GATEWAY": "false", GATEWAY_FD: str(listener.fileno()), "_LITELLM_GATEWAY_METRICS_HOST": host, - "FORWARDED_ALLOW_IPS": forwarded_allow_ips(os.environ.get("FORWARDED_ALLOW_IPS")), + "FORWARDED_ALLOW_IPS": "127.0.0.1", }, pass_fds=(listener.fileno(),), start_new_session=True, diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index aa5ce9825ae..bb0119f9398 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -21,9 +21,8 @@ def ready(client: httpx.Client) -> bool: @pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) -@pytest.mark.parametrize(("trusted_proxies", "expected_ip"), [("", "127.0.0.2"), ("127.0.0.2", "198.51.100.10")]) def test_gateway_preserves_metrics_bind_and_mcp_client_ip( - gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...], trusted_proxies: str, expected_ip: str + gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...] ) -> None: config: Final = tmp_path / "config.yaml" config.write_text( @@ -41,7 +40,6 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( "LITELLM_MASTER_KEY": gateway.key, "STORE_MODEL_IN_DB": "False", "PYTHON_DOTENV_DISABLED": "1", - "FORWARDED_ALLOW_IPS": trusted_proxies, } with (tmp_path / "proxy.log").open("wb") as output: with subprocess.Popen( @@ -82,10 +80,10 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( assert response.status_code == 200, response.text client_ip: Final = external.get( "/v1/mcp/network/client-ip", - headers={"Authorization": f"Bearer {gateway.key}", "X-Forwarded-For": "198.51.100.10"}, + headers={"Authorization": f"Bearer {gateway.key}", "X-Forwarded-For": "1.2.3.4"}, ) assert client_ip.status_code == 200, client_ip.text - assert client_ip.json() == {"ip": expected_ip} + assert client_ip.json() == {"ip": "127.0.0.2"} finally: assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text() assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text() diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index 44771ff9616..969cfd12da4 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -13,7 +13,6 @@ from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server from litellm.proxy.rust_gateway import ( child_argv, cli_command, - forwarded_allow_ips, gateway_binary, inherited_fd, metrics_host, @@ -54,21 +53,6 @@ def test_missing_binary_gives_install_instruction() -> None: ) -@pytest.mark.parametrize( - ("trusted", "expected"), - [ - (None, "127.0.0.1"), - ("", "127.0.0.1"), - ("192.0.2.1", "127.0.0.1,192.0.2.1"), - ("127.0.0.1, 192.0.2.0/24", "127.0.0.1,192.0.2.0/24"), - ("*", "*"), - ("127.0.0.1,*", "127.0.0.1,*"), - ], -) -def test_private_hop_preserves_configured_trusted_proxies(trusted: str | None, expected: str) -> None: - assert forwarded_allow_ips(trusted) == expected - - @pytest.mark.parametrize("supervised", [False, True]) @pytest.mark.parametrize("host", ["0.0.0.0", "192.0.2.1", "::"]) def test_metrics_keep_configured_host_separate_from_private_listener( @@ -174,7 +158,6 @@ def test_supervisor_drains_before_stopping_python_and_reaps_both_children( patch("asyncio.create_subprocess_exec", side_effect=spawn) as launch, patch("socket.socket", return_value=listener), patch("os.killpg", side_effect=kill_group) as kill, - patch.dict("os.environ", {"FORWARDED_ALLOW_IPS": "192.0.2.1"}), ): assert supervise("gateway", ("python", "--rust_gateway"), "0.0.0.0", 4000) == expected @@ -184,7 +167,7 @@ def test_supervisor_drains_before_stopping_python_and_reaps_both_children( assert python_call.args == ("python", "--host", "127.0.0.1", "--port", "45555") assert python_call.kwargs["pass_fds"] == (17,) assert python_call.kwargs["env"]["LITELLM_RUST_GATEWAY"] == "false" - assert python_call.kwargs["env"]["FORWARDED_ALLOW_IPS"] == "127.0.0.1,192.0.2.1" + assert python_call.kwargs["env"]["FORWARDED_ALLOW_IPS"] == "127.0.0.1" assert rust_call.kwargs["env"]["LITELLM_FORWARD_UPSTREAM"] == "http://127.0.0.1:45555" assert rust_call.kwargs["env"]["PORT"] == "4000" if event == "spawn-error": From 92e4111cb11f37425a06a2d3e84d23a9e4938dd1 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 02:51:32 +0000 Subject: [PATCH 06/17] fix(proxy): reject incompatible forwarding trust configuration --- litellm/proxy/rust_gateway.py | 6 ++++++ tests/unit/proxy/test_rust_gateway.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 9b7ba98c161..cd02be26c77 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -18,6 +18,10 @@ def gateway_binary() -> str | None: return shutil.which(name, path=sysconfig.get_path("scripts")) or shutil.which(name) +def unsupported_proxy_trust(value: str | None) -> bool: + return any(host.strip() not in ("", "127.0.0.1") for host in (value or "").split(",")) + + def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: flag: Final = next((name for name, value in unsupported.items() if value), None) if flag is not None: @@ -26,6 +30,8 @@ def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str return "--rust_gateway requires POSIX inherited sockets and process signals" if binary is None: return "--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment" + if unsupported_proxy_trust(os.environ.get("FORWARDED_ALLOW_IPS")): + return "--rust_gateway does not support non-loopback FORWARDED_ALLOW_IPS; disable Rust forwarding for this deployment" return None diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index 969cfd12da4..27b2e9b54e9 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -53,6 +53,20 @@ def test_missing_binary_gives_install_instruction() -> None: ) +@pytest.mark.parametrize("trusted", [None, "", "127.0.0.1", "192.0.2.1", "*", "127.0.0.1,192.0.2.1"]) +def test_configured_proxy_trust_is_not_silently_discarded(trusted: str | None, monkeypatch: pytest.MonkeyPatch) -> None: + if trusted is None: + monkeypatch.delenv("FORWARDED_ALLOW_IPS", raising=False) + else: + monkeypatch.setenv("FORWARDED_ALLOW_IPS", trusted) + expected: Final = ( + None + if trusted in (None, "", "127.0.0.1") + else "--rust_gateway does not support non-loopback FORWARDED_ALLOW_IPS; disable Rust forwarding for this deployment" + ) + assert startup_error("gateway", {}) == expected + + @pytest.mark.parametrize("supervised", [False, True]) @pytest.mark.parametrize("host", ["0.0.0.0", "192.0.2.1", "::"]) def test_metrics_keep_configured_host_separate_from_private_listener( @@ -158,6 +172,7 @@ def test_supervisor_drains_before_stopping_python_and_reaps_both_children( patch("asyncio.create_subprocess_exec", side_effect=spawn) as launch, patch("socket.socket", return_value=listener), patch("os.killpg", side_effect=kill_group) as kill, + patch.dict("os.environ", {"FORWARDED_ALLOW_IPS": "*"}), ): assert supervise("gateway", ("python", "--rust_gateway"), "0.0.0.0", 4000) == expected From faae83d97de712629cff3d377b1cbc41b678494b Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 06:31:41 +0900 Subject: [PATCH 07/17] refactor(proxy): resolve the rust gateway mode once and make the supervisor policy a pure state machine LITELLM_RUST_GATEWAY is now the only switch. rust_gateway_mode reads the environment once and the entry points pass the result down. Shutdown order and exit codes live in step(), child commands in python_spec and rust_spec. The binary is looked up in the environment's scripts directory only, and an operator's FORWARDED_ALLOW_IPS is kept with the Rust hop added instead of being rejected. Co-Authored-By: Claude Opus 5.5 --- litellm/proxy/proxy_cli.py | 31 +- litellm/proxy/rust_gateway.py | 280 ++++++++++++---- .../configuration/test_rust_gateway.py | 34 +- tests/unit/proxy/test_rust_gateway.py | 303 +++++++++--------- 4 files changed, 406 insertions(+), 242 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 203358cba97..8a3d85c81b3 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -8,6 +8,7 @@ import random import re import subprocess import sys +import sysconfig import urllib.parse as urlparse from collections.abc import Callable, Iterable, Mapping, Sequence from pathlib import Path @@ -409,6 +410,7 @@ class ProxyInitializationHelpers: log_config: str | None = None, keepalive_timeout: int | None = None, timeout_worker_healthcheck: int | None = None, + listener_fd: int | None = None, ) -> dict: """ Get the arguments for `uvicorn` worker @@ -419,7 +421,6 @@ class ProxyInitializationHelpers: import litellm from litellm._logging import _get_uvicorn_json_log_config, resolve_log_level - from litellm.proxy.rust_gateway import inherited_fd from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager uvicorn_args: Final = { @@ -428,7 +429,7 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), - **({"fd": inherited_fd()} if inherited_fd() is not None else {}), + **({"fd": listener_fd} if listener_fd is not None else {}), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -682,6 +683,7 @@ class ProxyInitializationHelpers: ssl_keyfile_path: str, max_requests_before_restart: int | None = None, max_requests_before_restart_jitter: int | None = None, + listener_fd: int | None = None, ): """ Run litellm with `gunicorn` @@ -754,9 +756,7 @@ class ProxyInitializationHelpers: ) graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout() gunicorn_options: Final = { - "bind": ( - f"fd://{os.environ['_LITELLM_GATEWAY_FD']}" if "_LITELLM_GATEWAY_FD" in os.environ else f"{host}:{port}" - ), + "bind": f"fd://{listener_fd}" if listener_fd is not None else f"{host}:{port}", "workers": num_workers, # default is 1 "worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, @@ -1043,13 +1043,6 @@ class ProxyInitializationHelpers: is_flag=True, help="Starts proxy via gunicorn, instead of uvicorn (better for managing multiple workers)", ) -@click.option( - "--rust_gateway", - envvar="LITELLM_RUST_GATEWAY", - default=False, - is_flag=True, - help="Run the Rust forwarding server on the public port, with Python on loopback", -) @click.option( "--run_hypercorn", default=False, @@ -1252,7 +1245,6 @@ def run_server( setup, version, run_gunicorn, - rust_gateway: bool, run_hypercorn, run_granian, ssl_keyfile_path, @@ -1292,12 +1284,15 @@ def run_server( run_setup_wizard() return - if rust_gateway and not any( + from litellm.proxy.rust_gateway import Supervised, Supervisor, rust_gateway_mode + + gateway_mode: Final = rust_gateway_mode(os.environ) + if isinstance(gateway_mode, Supervisor) and not any( (version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data) ): from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise - binary: Final = gateway_binary() + binary: Final = gateway_binary(sysconfig.get_path("scripts")) gateway_error: Final = startup_error( binary, { @@ -1315,6 +1310,7 @@ def run_server( random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port ) raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port)) + supervised: Final = gateway_mode if isinstance(gateway_mode, Supervised) else None args: Final = locals() try: @@ -1764,9 +1760,8 @@ def run_server( if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None: from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process - from litellm.proxy.rust_gateway import metrics_host - metrics_bind_host: Final = metrics_host(host) + metrics_bind_host: Final = supervised.metrics_host if supervised is not None else host try: metrics_process: Final = start_metrics_server_process( host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir @@ -1785,6 +1780,7 @@ def run_server( log_config=log_config, keepalive_timeout=keepalive_timeout, timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None), + listener_fd=supervised.listener_fd if supervised is not None else None, ) # Optional: recycle uvicorn workers after N requests if max_requests_before_restart is not None: @@ -1859,6 +1855,7 @@ def run_server( ssl_keyfile_path=ssl_keyfile_path, max_requests_before_restart=max_requests_before_restart, max_requests_before_restart_jitter=max_requests_before_restart_jitter, + listener_fd=supervised.listener_fd if supervised is not None else None, ) elif run_hypercorn is True: ProxyInitializationHelpers.init_hypercorn_server( diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index cd02be26c77..9a2d61f09f7 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -1,117 +1,255 @@ import asyncio +import contextlib import os import shutil import signal import socket import sys -import sysconfig from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from enum import Enum, IntEnum +from types import MappingProxyType from typing import Final from pydantic import TypeAdapter +from typing_extensions import assert_never +ENABLE: Final = "LITELLM_RUST_GATEWAY" GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" +METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" +TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" +RUST_HOP: Final = "127.0.0.1" + +_FD: Final = TypeAdapter(int) +_ENABLED: Final = TypeAdapter(bool) +_ADDRESS: Final = TypeAdapter(tuple[str, int]) -def gateway_binary() -> str | None: +@dataclass(frozen=True, slots=True) +class Disabled: + pass + + +@dataclass(frozen=True, slots=True) +class Supervisor: + pass + + +@dataclass(frozen=True, slots=True) +class Supervised: + listener_fd: int + metrics_host: str + trusted_proxies: tuple[str, ...] + + +RustGatewayMode = Disabled | Supervisor | Supervised + + +def trusted_proxies(operator_value: str | None) -> tuple[str, ...]: + configured: Final = tuple(host.strip() for host in (operator_value or "").split(",") if host.strip()) + return ("*",) if "*" in configured else (RUST_HOP, *(host for host in configured if host != RUST_HOP)) + + +def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode: + fd: Final = environ.get(GATEWAY_FD) + if fd is not None: + return Supervised(_FD.validate_python(fd), environ[METRICS_HOST], trusted_proxies(environ.get(TRUSTED_PROXIES))) + if _ENABLED.validate_python(environ.get(ENABLE, "false")): + return Supervisor() + return Disabled() + + +def gateway_binary(scripts_dir: str) -> str | None: name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway" - return shutil.which(name, path=sysconfig.get_path("scripts")) or shutil.which(name) - - -def unsupported_proxy_trust(value: str | None) -> bool: - return any(host.strip() not in ("", "127.0.0.1") for host in (value or "").split(",")) + return shutil.which(name, path=scripts_dir) def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: flag: Final = next((name for name, value in unsupported.items() if value), None) if flag is not None: - return f"--rust_gateway does not support {flag}" + return f"{ENABLE} does not support {flag}" if os.name != "posix": - return "--rust_gateway requires POSIX inherited sockets and process signals" + return f"{ENABLE} requires POSIX inherited sockets and process signals" if binary is None: - return "--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment" - if unsupported_proxy_trust(os.environ.get("FORWARDED_ALLOW_IPS")): - return "--rust_gateway does not support non-loopback FORWARDED_ALLOW_IPS; disable Rust forwarding for this deployment" + return f"{ENABLE} requires the litellm-gateway binary. Install litellm[proxy] in this environment" return None -def child_argv(argv: Sequence[str], port: int) -> tuple[str, ...]: - return (*(arg for arg in argv if arg != "--rust_gateway"), "--host", "127.0.0.1", "--port", str(port)) +class Child(Enum): + RUST = "rust" + PYTHON = "python" -def inherited_fd() -> int | None: - value: Final = os.environ.get(GATEWAY_FD) - return int(value) if value is not None else None +DRAIN_ORDER: Final = (Child.RUST, Child.PYTHON) -def metrics_host(host: str) -> str: - return os.environ.get("_LITELLM_GATEWAY_METRICS_HOST", host) if inherited_fd() is not None else host +class ExitCode(IntEnum): + CLEAN = 0 + CHILD_FAILED = 1 -def terminate_group(process: asyncio.subprocess.Process) -> None: - try: +@dataclass(frozen=True, slots=True) +class Stop: + signum: signal.Signals + + +@dataclass(frozen=True, slots=True) +class Exited: + child: Child + code: int + + +@dataclass(frozen=True, slots=True) +class Running: + pass + + +@dataclass(frozen=True, slots=True) +class Draining: + signum: signal.Signals + signalled: Child + remaining: tuple[Child, ...] + + +@dataclass(frozen=True, slots=True) +class Finished: + code: ExitCode + + +@dataclass(frozen=True, slots=True) +class Signal: + child: Child + signum: signal.Signals + + +Event = Stop | Exited +State = Running | Draining | Finished + + +def _signal_next(signum: signal.Signals, order: tuple[Child, ...]) -> tuple[State, Signal | None]: + if not order: + return Finished(ExitCode.CLEAN), None + return Draining(signum, order[0], order[1:]), Signal(order[0], signum) + + +def step(state: State, event: Event) -> tuple[State, Signal | None]: + match state: + case Running(): + match event: + case Stop(signum): + return _signal_next(signum, DRAIN_ORDER) + case Exited(): + return Finished(ExitCode.CHILD_FAILED), None + case _: + assert_never(event) + case Draining(signum, signalled, remaining): + match event: + case Stop(): + return state, None + case Exited(child, code) if child is signalled and code == 0: + return _signal_next(signum, remaining) + case Exited(): + return Finished(ExitCode.CHILD_FAILED), None + case _: + assert_never(event) + case Finished(): + return state, None + case _: + assert_never(state) + + +@dataclass(frozen=True, slots=True) +class ChildSpec: + argv: tuple[str, ...] + env: Mapping[str, str] + pass_fds: tuple[int, ...] + + +def python_spec( + command: Sequence[str], listener_fd: int, upstream_port: int, public_host: str, environ: Mapping[str, str] +) -> ChildSpec: + return ChildSpec( + argv=(*command, "--host", RUST_HOP, "--port", str(upstream_port)), + env=MappingProxyType( + { + **environ, + GATEWAY_FD: str(listener_fd), + METRICS_HOST: public_host, + TRUSTED_PROXIES: ",".join(trusted_proxies(environ.get(TRUSTED_PROXIES))), + } + ), + pass_fds=(listener_fd,), + ) + + +def rust_spec(binary: str, upstream_port: int, host: str, port: int, environ: Mapping[str, str]) -> ChildSpec: + return ChildSpec( + argv=(binary,), + env=MappingProxyType( + { + **environ, + "HOST": host, + "PORT": str(port), + "LITELLM_FORWARD_UPSTREAM": f"http://{RUST_HOP}:{upstream_port}", + } + ), + pass_fds=(), + ) + + +async def _spawn(spec: ChildSpec) -> asyncio.subprocess.Process: + return await asyncio.create_subprocess_exec( + *spec.argv, env=dict(spec.env), pass_fds=spec.pass_fds, start_new_session=True + ) + + +def _terminate_group(process: asyncio.subprocess.Process) -> None: + with contextlib.suppress(ProcessLookupError): os.killpg(process.pid, signal.SIGTERM) - except ProcessLookupError: - pass -async def _supervise(binary: str, command: Sequence[str], host: str, port: int) -> int: +async def _report_exit(child: Child, process: asyncio.subprocess.Process, events: asyncio.Queue[Event]) -> None: + events.put_nowait(Exited(child, await process.wait())) + + +async def _drive( + state: State, events: asyncio.Queue[Event], children: Mapping[Child, asyncio.subprocess.Process] +) -> ExitCode: + if isinstance(state, Finished): + return state.code + following, action = step(state, await events.get()) + if action is not None: + with contextlib.suppress(ProcessLookupError): + children[action.child].send_signal(action.signum) + return await _drive(following, events, children) + + +async def _supervise(binary: str, command: Sequence[str], host: str, port: int) -> ExitCode: loop: Final = asyncio.get_running_loop() - stopped: Final[asyncio.Future[signal.Signals]] = loop.create_future() - - def stop(signum: signal.Signals) -> None: - if not stopped.done(): - stopped.set_result(signum) - + events: Final[asyncio.Queue[Event]] = asyncio.Queue() for signum in (signal.SIGTERM, signal.SIGINT): - loop.add_signal_handler(signum, stop, signum) + loop.add_signal_handler(signum, events.put_nowait, Stop(signum)) with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: - listener.bind(("127.0.0.1", 0)) + listener.bind((RUST_HOP, 0)) listener.listen() - _, upstream_port = TypeAdapter(tuple[str, int]).validate_python(listener.getsockname()) - python: Final = await asyncio.create_subprocess_exec( - *child_argv(command, upstream_port), - env={ - **os.environ, - "LITELLM_RUST_GATEWAY": "false", - GATEWAY_FD: str(listener.fileno()), - "_LITELLM_GATEWAY_METRICS_HOST": host, - "FORWARDED_ALLOW_IPS": "127.0.0.1", - }, - pass_fds=(listener.fileno(),), - start_new_session=True, - ) + _, upstream_port = _ADDRESS.validate_python(listener.getsockname()) + python: Final = await _spawn(python_spec(command, listener.fileno(), upstream_port, host, os.environ)) try: - rust: Final = await asyncio.create_subprocess_exec( - binary, - env={ - **os.environ, - "HOST": host, - "PORT": str(port), - "LITELLM_FORWARD_UPSTREAM": f"http://127.0.0.1:{upstream_port}", - }, - start_new_session=True, - ) + rust: Final = await _spawn(rust_spec(binary, upstream_port, host, port, os.environ)) except OSError: - terminate_group(python) + _terminate_group(python) await python.wait() raise - python_done: Final = asyncio.create_task(python.wait()) - rust_done: Final = asyncio.create_task(rust.wait()) + children: Final = MappingProxyType({Child.RUST: rust, Child.PYTHON: python}) + waiters: Final = tuple( + asyncio.create_task(_report_exit(child, process, events)) for child, process in children.items() + ) try: - done, _ = await asyncio.wait((stopped, python_done, rust_done), return_when=asyncio.FIRST_COMPLETED) - if python_done in done or rust_done in done: - return 1 - rust.send_signal(stopped.result()) - draining, _ = await asyncio.wait((python_done, rust_done), return_when=asyncio.FIRST_COMPLETED) - if python_done in draining or rust_done.result() != 0: - return 1 - python.send_signal(stopped.result()) - return 0 if await python_done == 0 else 1 + return await _drive(Running(), events, children) finally: - terminate_group(rust) - terminate_group(python) - await asyncio.gather(python_done, rust_done) + for process in children.values(): + _terminate_group(process) + await asyncio.gather(*waiters) def supervise(binary: str, command: Sequence[str], host: str, port: int) -> int: @@ -119,7 +257,7 @@ def supervise(binary: str, command: Sequence[str], host: str, port: int) -> int: return asyncio.run(_supervise(binary, command, host, port)) except OSError as error: sys.stderr.write(f"Rust gateway startup failed: {error}\n") - return 1 + return ExitCode.CHILD_FAILED def cli_command(argv: Sequence[str]) -> tuple[str, ...]: diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index bb0119f9398..bbcb20cfa8c 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -12,6 +12,8 @@ from tests.integration._support.client import Gateway, eventually from tests.integration._support.ports import allocate_port from tests.integration._support.process import stop_root_process +TRUSTED_BALANCER: Final = "127.0.0.3" + def ready(client: httpx.Client) -> bool: try: @@ -21,6 +23,19 @@ def ready(client: httpx.Client) -> bool: @pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) +def client_ip(base_url: str, local_address: str, key: str, forwarded_for: str) -> httpx.Response: + with httpx.Client( + base_url=base_url, + transport=httpx.HTTPTransport(local_address=local_address), + trust_env=False, + timeout=5, + ) as client: + return client.get( + "/v1/mcp/network/client-ip", + headers={"Authorization": f"Bearer {key}", "X-Forwarded-For": forwarded_for}, + ) + + def test_gateway_preserves_metrics_bind_and_mcp_client_ip( gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...] ) -> None: @@ -40,6 +55,8 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( "LITELLM_MASTER_KEY": gateway.key, "STORE_MODEL_IN_DB": "False", "PYTHON_DOTENV_DISABLED": "1", + "LITELLM_RUST_GATEWAY": "true", + "FORWARDED_ALLOW_IPS": TRUSTED_BALANCER, } with (tmp_path / "proxy.log").open("wb") as output: with subprocess.Popen( @@ -48,7 +65,6 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( "-I", "-m", "litellm.proxy.proxy_cli", - "--rust_gateway", "--config", str(config), "--host", @@ -78,12 +94,16 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( assert eventually(lambda: ready(external), bool, seconds=180), (tmp_path / "proxy.log").read_text() response: Final = external.get(f"http://127.0.0.2:{metrics_port}/metrics") assert response.status_code == 200, response.text - client_ip: Final = external.get( - "/v1/mcp/network/client-ip", - headers={"Authorization": f"Bearer {gateway.key}", "X-Forwarded-For": "1.2.3.4"}, - ) - assert client_ip.status_code == 200, client_ip.text - assert client_ip.json() == {"ip": "127.0.0.2"} + public: Final = f"http://127.0.0.1:{public_port}" + spoofed: Final = client_ip(public, "127.0.0.2", gateway.key, "1.2.3.4") + assert spoofed.status_code == 200, spoofed.text + assert spoofed.json() == {"ip": "127.0.0.2"} + behind_balancer: Final = client_ip(public, TRUSTED_BALANCER, gateway.key, "203.0.113.9") + assert behind_balancer.status_code == 200, behind_balancer.text + assert behind_balancer.json() == {"ip": "203.0.113.9"} + borrowed: Final = client_ip(public, "127.0.0.2", gateway.key, f"203.0.113.9, {TRUSTED_BALANCER}") + assert borrowed.status_code == 200, borrowed.text + assert borrowed.json() == {"ip": "127.0.0.2"} finally: assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text() assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text() diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index 27b2e9b54e9..278dc4621d3 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -1,193 +1,202 @@ -import asyncio +import functools import signal -import socket import sys from pathlib import Path from typing import Final -from unittest.mock import MagicMock, Mock, call, patch import pytest from click.testing import CliRunner +from pydantic import ValidationError from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server from litellm.proxy.rust_gateway import ( - child_argv, + Child, + Disabled, + Event, + ExitCode, + Exited, + Finished, + Running, + Signal, + State, + Stop, + Supervised, + Supervisor, cli_command, gateway_binary, - inherited_fd, - metrics_host, + python_spec, + rust_gateway_mode, + rust_spec, startup_error, - supervise, + step, + trusted_proxies, ) +TERM: Final = signal.SIGTERM +INT: Final = signal.SIGINT + @pytest.mark.parametrize( "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] ) -@pytest.mark.parametrize("enable", ["flag", "environment"]) -def test_unsupported_modes_fail_before_starting_children( - flag: str, enable: str, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("LITELLM_RUST_GATEWAY", "true" if enable == "environment" else "false") - arguments: Final = ([] if enable == "environment" else ["--rust_gateway"]) + [flag] +def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST_GATEWAY", "true") + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] - result: Final = CliRunner().invoke(run_server, arguments + values) + result: Final = CliRunner().invoke(run_server, [flag, *values]) assert result.exit_code != 0 - assert f"--rust_gateway does not support {flag}" in result.output + assert f"LITELLM_RUST_GATEWAY does not support {flag}" in result.output -def test_child_arguments_preserve_options_and_force_private_listener() -> None: - arguments: Final = ("--rust_gateway", "--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") - params: Final = run_server.make_context("litellm", list(child_argv(arguments, 45555))).params +@pytest.mark.parametrize( + ("environ", "expected"), + [ + ({}, Disabled()), + ({"LITELLM_RUST_GATEWAY": "false"}, Disabled()), + ({"LITELLM_RUST_GATEWAY": "off"}, Disabled()), + ({"LITELLM_RUST_GATEWAY": "0"}, Disabled()), + ({"LITELLM_RUST_GATEWAY": "true"}, Supervisor()), + ({"LITELLM_RUST_GATEWAY": "1"}, Supervisor()), + ({"LITELLM_RUST_GATEWAY": "yes"}, Supervisor()), + ({"LITELLM_RUST_GATEWAY": "on"}, Supervisor()), + ( + { + "LITELLM_RUST_GATEWAY": "true", + "_LITELLM_GATEWAY_FD": "17", + "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", + "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", + }, + Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5")), + ), + ], +) +def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None: + assert rust_gateway_mode(environ) == expected + + +def test_unrecognized_switch_value_is_rejected() -> None: + with pytest.raises(ValidationError): + rust_gateway_mode({"LITELLM_RUST_GATEWAY": "enabled"}) + + +@pytest.mark.parametrize( + ("operator_value", "expected"), + [ + (None, ("127.0.0.1",)), + ("", ("127.0.0.1",)), + ("10.0.0.5", ("127.0.0.1", "10.0.0.5")), + ("127.0.0.1, 10.0.0.5", ("127.0.0.1", "10.0.0.5")), + ("10.0.0.5,127.0.0.1", ("127.0.0.1", "10.0.0.5")), + ("*", ("*",)), + ("10.0.0.5,*", ("*",)), + ], +) +def test_operator_trust_is_kept_and_rust_hop_added(operator_value: str | None, expected: tuple[str, ...]) -> None: + assert trusted_proxies(operator_value) == expected + + +@pytest.mark.parametrize("operator_value", [None, "", "10.0.0.5", "10.0.0.5, 192.0.2.0/24", "127.0.0.1"]) +def test_python_child_always_trusts_the_rust_hop(operator_value: str | None) -> None: + environ: Final = {} if operator_value is None else {"FORWARDED_ALLOW_IPS": operator_value} + spec: Final = python_spec(("python",), 17, 45555, "0.0.0.0", environ) + assert spec.env["FORWARDED_ALLOW_IPS"].split(",")[0] == "127.0.0.1" + assert rust_gateway_mode(spec.env) == Supervised(17, "0.0.0.0", trusted_proxies(operator_value)) + + +def test_python_child_listens_privately_and_keeps_options() -> None: + arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") + spec: Final = python_spec(arguments, 17, 45555, "0.0.0.0", {}) + params: Final = run_server.make_context("litellm", list(spec.argv)).params assert params["host"] == "127.0.0.1" assert params["port"] == 45555 assert params["num_workers"] == 2 assert params["reload"] is True - assert params["rust_gateway"] is False + assert spec.pass_fds == (17,) assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli") +def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None: + spec: Final = rust_spec("gateway", 45555, "0.0.0.0", 4000, {"RUST_LOG": "info"}) + assert spec.argv == ("gateway",) + assert spec.pass_fds == () + assert dict(spec.env) == { + "RUST_LOG": "info", + "HOST": "0.0.0.0", + "PORT": "4000", + "LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555", + } + + def test_missing_binary_gives_install_instruction() -> None: assert startup_error(None, {}) == ( - "--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment" + "LITELLM_RUST_GATEWAY requires the litellm-gateway binary. Install litellm[proxy] in this environment" ) -@pytest.mark.parametrize("trusted", [None, "", "127.0.0.1", "192.0.2.1", "*", "127.0.0.1,192.0.2.1"]) -def test_configured_proxy_trust_is_not_silently_discarded(trusted: str | None, monkeypatch: pytest.MonkeyPatch) -> None: - if trusted is None: - monkeypatch.delenv("FORWARDED_ALLOW_IPS", raising=False) - else: - monkeypatch.setenv("FORWARDED_ALLOW_IPS", trusted) - expected: Final = ( - None - if trusted in (None, "", "127.0.0.1") - else "--rust_gateway does not support non-loopback FORWARDED_ALLOW_IPS; disable Rust forwarding for this deployment" - ) - assert startup_error("gateway", {}) == expected - - -@pytest.mark.parametrize("supervised", [False, True]) -@pytest.mark.parametrize("host", ["0.0.0.0", "192.0.2.1", "::"]) -def test_metrics_keep_configured_host_separate_from_private_listener( - supervised: bool, host: str, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", host) - if supervised: - monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17") - else: - monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) - assert metrics_host("127.0.0.1") == (host if supervised else "127.0.0.1") - - -def test_current_environment_binary_wins_over_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - binary: Final = tmp_path / "litellm-gateway" +def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + scripts: Final = tmp_path / "scripts" + elsewhere: Final = tmp_path / "elsewhere" + scripts.mkdir() + elsewhere.mkdir() + binary: Final = elsewhere / "litellm-gateway" binary.write_text("#!/bin/sh\nexit 0\n") binary.chmod(0o755) - monkeypatch.setattr("sysconfig.get_path", lambda _: str(tmp_path)) - monkeypatch.setenv("PATH", "") - assert gateway_binary() == str(binary) + monkeypatch.setenv("PATH", str(elsewhere)) + assert gateway_binary(str(scripts)) is None + assert gateway_binary(str(elsewhere)) == str(binary) -@pytest.mark.parametrize("fd", [None, "17"]) -def test_inherited_socket_reaches_uvicorn(fd: str | None, monkeypatch: pytest.MonkeyPatch) -> None: - if fd is None: - monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) - else: - monkeypatch.setenv("_LITELLM_GATEWAY_FD", fd) - expected: Final = int(fd) if fd is not None else None - options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555) - assert inherited_fd() == expected - assert options.get("fd") == expected +@pytest.mark.parametrize("listener_fd", [None, 17]) +def test_inherited_socket_reaches_uvicorn(listener_fd: int | None) -> None: + options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 45555, listener_fd=listener_fd + ) + assert options.get("fd") == listener_fd + + +def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]: + def advance(acc: tuple[State, tuple[Signal, ...]], event: Event) -> tuple[State, tuple[Signal, ...]]: + state, sent = acc + following, action = step(state, event) + return following, sent if action is None else (*sent, action) + + return functools.reduce(advance, events, (Running(), ())) + + +RUST: Final = Child.RUST +PYTHON: Final = Child.PYTHON @pytest.mark.parametrize( - ("event", "python_exit", "rust_exit", "expected"), + ("events", "signals", "code"), [ - ("signal", 0, 0, 0), - ("signal", 1, 0, 1), - ("signal", 0, 1, 1), - ("python-exit", 0, 0, 1), - ("rust-exit", 0, 0, 1), - ("python-during-drain", 1, 0, 1), - ("spawn-error", 0, 0, 1), + ( + (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 0)), + (Signal(RUST, TERM), Signal(PYTHON, TERM)), + ExitCode.CLEAN, + ), + ((Exited(PYTHON, 0),), (), ExitCode.CHILD_FAILED), + ((Exited(RUST, 1),), (), ExitCode.CHILD_FAILED), + ((Stop(TERM), Exited(PYTHON, 0)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), + ((Stop(TERM), Exited(RUST, 1)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), + ( + (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 1)), + (Signal(RUST, TERM), Signal(PYTHON, TERM)), + ExitCode.CHILD_FAILED, + ), + ( + (Stop(INT), Stop(INT), Exited(RUST, 0), Exited(PYTHON, 0)), + (Signal(RUST, INT), Signal(PYTHON, INT)), + ExitCode.CLEAN, + ), + ( + (Exited(RUST, 1), Stop(TERM), Exited(PYTHON, 0)), + (), + ExitCode.CHILD_FAILED, + ), ], ) -def test_supervisor_drains_before_stopping_python_and_reaps_both_children( - event: str, python_exit: int, rust_exit: int, expected: int, capsys: pytest.CaptureFixture[str] +def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit( + events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode ) -> None: - loop: Final = asyncio.new_event_loop() - handlers: Final = Mock() - listener: Final = MagicMock(spec=socket.socket) - listener.__enter__.return_value = listener - listener.getsockname.return_value = ("127.0.0.1", 45555) - listener.fileno.return_value = 17 - children: Final = {} - sent: Final = Mock() - - def current_loop(): - return Mock(wraps=loop, add_signal_handler=handlers) - - async def spawn(*argv, **kwargs): - done: Final = asyncio.events.get_running_loop().create_future() - pid: Final = 101 if argv[0] == "python" else 102 - - async def wait() -> int: - return await done - - def send_signal(signum: signal.Signals) -> None: - sent(pid, signum) - if pid == 102 and event == "python-during-drain": - children[101][1].set_result(python_exit) - return - if pid == 101: - assert children[102][1].done(), "Python must remain alive until Rust has drained" - done.set_result(python_exit if pid == 101 else rust_exit) - - process: Final = Mock(spec=asyncio.subprocess.Process, pid=pid, wait=wait, send_signal=send_signal) - if pid == 102 and event == "spawn-error": - raise OSError("gateway cannot start") - children[pid] = (process, done) - if pid == 102: - if event == "python-exit": - children[101][1].set_result(python_exit) - elif event == "rust-exit": - done.set_result(rust_exit) - else: - _, stop, signum = handlers.call_args_list[0].args - stop(signum) - stop(signum) - return process - - def kill_group(pid: int, signum: signal.Signals) -> None: - assert signum == signal.SIGTERM - _, done = children[pid] - if done.done(): - raise ProcessLookupError - done.set_result(1) - - with ( - patch("asyncio.events.new_event_loop", return_value=loop), - patch("asyncio.get_running_loop", side_effect=current_loop), - patch("asyncio.create_subprocess_exec", side_effect=spawn) as launch, - patch("socket.socket", return_value=listener), - patch("os.killpg", side_effect=kill_group) as kill, - patch.dict("os.environ", {"FORWARDED_ALLOW_IPS": "*"}), - ): - assert supervise("gateway", ("python", "--rust_gateway"), "0.0.0.0", 4000) == expected - - assert all(done.done() for _, done in children.values()) - assert call(101, signal.SIGTERM) in kill.call_args_list - python_call, rust_call = launch.call_args_list - assert python_call.args == ("python", "--host", "127.0.0.1", "--port", "45555") - assert python_call.kwargs["pass_fds"] == (17,) - assert python_call.kwargs["env"]["LITELLM_RUST_GATEWAY"] == "false" - assert python_call.kwargs["env"]["FORWARDED_ALLOW_IPS"] == "127.0.0.1" - assert rust_call.kwargs["env"]["LITELLM_FORWARD_UPSTREAM"] == "http://127.0.0.1:45555" - assert rust_call.kwargs["env"]["PORT"] == "4000" - if event == "spawn-error": - assert "Rust gateway startup failed: gateway cannot start" in capsys.readouterr().err - elif event == "signal" and rust_exit == 0: - assert sent.call_args_list == [call(102, signal.SIGTERM), call(101, signal.SIGTERM)] - else: - assert call(101, signal.SIGTERM) not in sent.call_args_list + assert _fold(events) == (Finished(code), signals) From f4403ad699f926e0e1d5e1eaaf39a45e16ffe708 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 06:32:23 +0900 Subject: [PATCH 08/17] refactor(proxy): pass the supervised children's environment into supervise Co-Authored-By: Claude Opus 5.5 --- litellm/proxy/proxy_cli.py | 2 +- litellm/proxy/rust_gateway.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 8a3d85c81b3..68db6d3f316 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -1309,7 +1309,7 @@ def run_server( public_port: Final = ( random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port ) - raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port)) + raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port, os.environ)) supervised: Final = gateway_mode if isinstance(gateway_mode, Supervised) else None args: Final = locals() diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 9a2d61f09f7..f2ebd870853 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -224,7 +224,7 @@ async def _drive( return await _drive(following, events, children) -async def _supervise(binary: str, command: Sequence[str], host: str, port: int) -> ExitCode: +async def _supervise(binary: str, command: Sequence[str], host: str, port: int, environ: Mapping[str, str]) -> ExitCode: loop: Final = asyncio.get_running_loop() events: Final[asyncio.Queue[Event]] = asyncio.Queue() for signum in (signal.SIGTERM, signal.SIGINT): @@ -233,9 +233,9 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int) listener.bind((RUST_HOP, 0)) listener.listen() _, upstream_port = _ADDRESS.validate_python(listener.getsockname()) - python: Final = await _spawn(python_spec(command, listener.fileno(), upstream_port, host, os.environ)) + python: Final = await _spawn(python_spec(command, listener.fileno(), upstream_port, host, environ)) try: - rust: Final = await _spawn(rust_spec(binary, upstream_port, host, port, os.environ)) + rust: Final = await _spawn(rust_spec(binary, upstream_port, host, port, environ)) except OSError: _terminate_group(python) await python.wait() @@ -252,9 +252,9 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int) await asyncio.gather(*waiters) -def supervise(binary: str, command: Sequence[str], host: str, port: int) -> int: +def supervise(binary: str, command: Sequence[str], host: str, port: int, environ: Mapping[str, str]) -> int: try: - return asyncio.run(_supervise(binary, command, host, port)) + return asyncio.run(_supervise(binary, command, host, port, environ)) except OSError as error: sys.stderr.write(f"Rust gateway startup failed: {error}\n") return ExitCode.CHILD_FAILED From 3442588fa0fe9a0f28cf2b8864b9b500d824b630 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 06:35:19 +0900 Subject: [PATCH 09/17] refactor(proxy): declare rust gateway unions as explicit type aliases Co-Authored-By: Claude Opus 5.5 --- litellm/proxy/rust_gateway.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index f2ebd870853..65237eb00e0 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -9,7 +9,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from enum import Enum, IntEnum from types import MappingProxyType -from typing import Final +from typing import Final, TypeAlias from pydantic import TypeAdapter from typing_extensions import assert_never @@ -42,7 +42,7 @@ class Supervised: trusted_proxies: tuple[str, ...] -RustGatewayMode = Disabled | Supervisor | Supervised +RustGatewayMode: TypeAlias = Disabled | Supervisor | Supervised def trusted_proxies(operator_value: str | None) -> tuple[str, ...]: @@ -122,8 +122,8 @@ class Signal: signum: signal.Signals -Event = Stop | Exited -State = Running | Draining | Finished +Event: TypeAlias = Stop | Exited +State: TypeAlias = Running | Draining | Finished def _signal_next(signum: signal.Signals, order: tuple[Child, ...]) -> tuple[State, Signal | None]: From 3399254dd1c6ba1bf50be01df4974e605263b597 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 08:50:07 +0900 Subject: [PATCH 10/17] feat(proxy): supervise the Rust forwarding server from gateway.launch Co-Authored-By: Claude Opus 5.5 --- gateway/launch.py | 76 +++++++++++++++++-- .../configuration/test_rust_gateway.py | 71 +++++++++++++++++ tests/unit/gateway/test_launch.py | 69 +++++++++++++++-- 3 files changed, 206 insertions(+), 10 deletions(-) diff --git a/gateway/launch.py b/gateway/launch.py index 59f7121e513..d5d798f262b 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -19,9 +19,11 @@ Run with: import os import sys +import sysconfig from collections.abc import Callable, Mapping, Sequence from typing import TYPE_CHECKING, Final +from typing_extensions import assert_never from uvicorn.main import main as uvicorn_main from litellm_cli.main import without_gc_pauses @@ -31,17 +33,29 @@ if TYPE_CHECKING: from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings + from litellm.proxy.rust_gateway import Supervised GATEWAY_APP: Final = "gateway.launch:worker_app" KEEPALIVE_FLAG: Final = "--timeout-keep-alive" -def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str, ...]: +def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str], supervised: "Supervised | None") -> tuple[str, ...]: """Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly.""" keepalive: Final = environ.get("KEEPALIVE_TIMEOUT") + inherited_socket: Final = ( + ( + "--fd", + str(supervised.listener_fd), + "--proxy-headers", + "--forwarded-allow-ips", + ",".join(supervised.trusted_proxies), + ) + if supervised is not None + else () + ) if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv): - return (GATEWAY_APP, "--factory", *argv) - return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive) + return (GATEWAY_APP, "--factory", *argv, *inherited_socket) + return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive, *inherited_socket) def pool_database_url( @@ -75,7 +89,34 @@ def _serve(argv: Sequence[str]) -> None: uvicorn_main(tuple(argv), prog_name="uvicorn") -def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: +def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, int, Mapping[str, str]]: + """Validate the uvicorn flags for Rust forwarding and return the binary, public address and child environment.""" + from pydantic import TypeAdapter + + from litellm.proxy.rust_gateway import TRUSTED_PROXIES, startup_error + + params: Final[Mapping[str, object]] = uvicorn_main.make_context( + "uvicorn", list(uvicorn_argv(argv, os.environ, None)) + ).params + error: Final = startup_error( + binary, + { + "--ssl-keyfile": params["ssl_keyfile"], + "--ssl-certfile": params["ssl_certfile"], + "--ssl-ciphers": any(arg.split("=")[0] == "--ssl-ciphers" for arg in argv), + "--uds": params["uds"] is not None, + "--fd": params["fd"] is not None, + }, + ) + if error is not None: + sys.exit(error) + assert binary is not None + host, port = TypeAdapter(tuple[str, int]).validate_python((params["host"], params["port"])) + operator_trust: Final = TypeAdapter(str | None).validate_python(params["forwarded_allow_ips"]) + return binary, host, port, {} if operator_trust is None else {TRUSTED_PROXIES: operator_trust} + + +def configure_database() -> None: from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url @@ -86,7 +127,32 @@ def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) - sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}") if pooled_url is not None: export_pooled_database_url(pooled_url) - serve(uvicorn_argv(argv, os.environ)) + + +def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: + from litellm.proxy.rust_gateway import ( + Disabled, + Supervised, + Supervisor, + gateway_binary, + rust_gateway_mode, + supervise, + ) + + mode: Final = rust_gateway_mode(os.environ) + match mode: + case Disabled(): + configure_database() + serve(uvicorn_argv(argv, os.environ, None)) + case Supervisor(): + binary, host, port, trust = public_listener(argv, gateway_binary(sysconfig.get_path("scripts"))) + configure_database() + command: Final = (sys.executable, "-m", "gateway.launch", *argv) + sys.exit(supervise(binary, command, host, port, {**os.environ, **trust})) + case Supervised(): + serve(uvicorn_argv(argv, os.environ, mode)) + case _: + assert_never(mode) if __name__ == "__main__": diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index bbcb20cfa8c..729c0b847e3 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -107,3 +107,74 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( finally: assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text() assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text() + + +@pytest.mark.parametrize( + "server_arguments", + [("--no-proxy-headers",), ("--forwarded-allow-ips", "127.0.0.1")], +) +def test_gateway_launcher_private_hop_preserves_client_address( + tmp_path: Path, server_arguments: tuple[str, ...] +) -> None: + config: Final = tmp_path / "config.yaml" + config.write_text( + json.dumps({"model_list": [], "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY"}}) + ) + public_port: Final = allocate_port() + environment: Final = { + **{key: value for key, value in os.environ.items() if not key.startswith("DATABASE_") and key != "DIRECT_URL"}, + "LITELLM_MASTER_KEY": "sk-rust-forwarding-integration-key-not-a-secret", + "LITELLM_RUST_GATEWAY": "true", + "LITELLM_PGBOUNCER_ENABLED": "false", + "CONFIG_FILE_PATH": str(config), + "STORE_MODEL_IN_DB": "False", + "PYTHON_DOTENV_DISABLED": "1", + } + log: Final = tmp_path / "gateway.log" + with log.open("wb") as output: + with subprocess.Popen( + ( + sys.executable, + "-I", + "-m", + "gateway.launch", + "--host", + "0.0.0.0", + "--port", + str(public_port), + "--workers", + "2", + "--timeout-worker-healthcheck", + "120", + *server_arguments, + ), + env=environment, + stdout=output, + stderr=output, + ) as supervisor: + try: + with httpx.Client( + base_url=f"http://127.0.0.1:{public_port}", + transport=httpx.HTTPTransport(local_address="127.0.0.2"), + trust_env=False, + timeout=5, + ) as external: + assert eventually(lambda: ready(external), bool, seconds=180), log.read_text() + response: Final = external.get( + "/health/liveliness?client-ip-proof", headers={"X-Forwarded-For": "198.51.100.10"} + ) + assert response.status_code == 200, response.text + events: Final = eventually( + lambda: tuple( + line + for line in log.read_text().splitlines() + if "GET /health/liveliness?client-ip-proof " in line + ), + bool, + seconds=10, + ) + assert len(events) == 1, log.read_text() + assert "127.0.0.2:" in events[0], events + finally: + assert stop_root_process(supervisor), log.read_text() + assert supervisor.returncode == 0, log.read_text() diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index d682fcbad2d..606e760ab3f 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -15,10 +15,11 @@ import pytest from uvicorn.importer import import_from_string from uvicorn.main import main as uvicorn_main -from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv +from gateway.launch import GATEWAY_APP, main, pool_database_url, public_listener, uvicorn_argv from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings from litellm.proxy.proxy_server import app as proxy_app +from litellm.proxy.rust_gateway import Supervised DB_ENV: Final = { "DATABASE_HOST": "db.internal", @@ -114,17 +115,65 @@ def _uvicorn_params(argv: tuple[str, ...]) -> dict[str, object]: class TestUvicornArgv: + @pytest.mark.parametrize( + "arguments", [("--uds", "/tmp/gateway.sock"), ("--fd", "0"), ("--fd=17",), ("--ssl-keyfile", "unused")] + ) + def test_forwarding_rejects_unsupported_listeners_before_starting_processes( + self, arguments: tuple[str, ...] + ) -> None: + with pytest.raises(SystemExit, match=f"does not support {arguments[0].split('=')[0]}"): + public_listener(arguments, "gateway") + + @pytest.mark.parametrize( + ("argv", "trust"), + [ + ((), {}), + (("--forwarded-allow-ips", "10.0.0.5"), {"FORWARDED_ALLOW_IPS": "10.0.0.5"}), + (("--forwarded-allow-ips=*",), {"FORWARDED_ALLOW_IPS": "*"}), + ], + ) + def test_forwarding_carries_operator_proxy_trust_to_the_child( + self, argv: tuple[str, ...], trust: dict[str, str] + ) -> None: + assert public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") == ( + "gateway", + "0.0.0.0", + 4000, + trust, + ) + + def test_supervised_listener_uses_the_inherited_loopback_socket(self): + supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1",)) + params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised)) + assert params["fd"] == 17 + assert params["workers"] == 2 + + @pytest.mark.parametrize( + "argv", + [(), ("--no-proxy-headers",), ("--forwarded-allow-ips", "192.0.2.1"), ("--forwarded-allow-ips=*",)], + ) + def test_private_hop_always_processes_headers_with_the_resolved_trust(self, argv: tuple[str, ...]): + supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5")) + params: Final = _uvicorn_params(uvicorn_argv(argv, {}, supervised)) + assert params["proxy_headers"] is True + assert params["forwarded_allow_ips"] == "127.0.0.1,10.0.0.5" + + def test_direct_listener_keeps_explicit_proxy_header_options(self): + params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {}, None)) + assert params["proxy_headers"] is False + assert params["forwarded_allow_ips"] == "*" + def test_keepalive_env_reaches_uvicorn(self): - params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"})) + params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"}, None)) assert params["app"] == GATEWAY_APP assert params["workers"] == 4 assert params["timeout_keep_alive"] == 75 def test_unset_env_keeps_the_uvicorn_default(self): - assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}))["timeout_keep_alive"] == 5 + assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}, None))["timeout_keep_alive"] == 5 def test_an_explicit_flag_wins_over_the_env(self): - argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}) + argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}, None) assert _uvicorn_params(argv)["timeout_keep_alive"] == 30 def test_the_app_uvicorn_is_told_to_serve_is_the_trimmed_gateway(self, monkeypatch: pytest.MonkeyPatch) -> None: @@ -141,7 +190,7 @@ class TestUvicornArgv: ): monkeypatch.delenv(key, raising=False) - params: Final = _uvicorn_params(uvicorn_argv((), {})) + params: Final = _uvicorn_params(uvicorn_argv((), {}, None)) assert params["factory"] is True try: served: Final = import_from_string(cast(str, params["app"]))() @@ -229,6 +278,16 @@ class TestMain: DatabaseURLSettings.from_env().apply_to_env() assert os.environ["DATABASE_URL"] == pooled + def test_forwarding_disabled_keeps_uvicorn_socket_options( + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LITELLM_RUST_GATEWAY", "false") + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False) + served: Final[list[tuple[str, ...]]] = [] + main(("--uds", "/tmp/gateway.sock"), serve=lambda argv: served.append(tuple(argv))) + assert _uvicorn_params(served[0])["uds"] == "/tmp/gateway.sock" + def test_a_pooler_that_cannot_start_stops_the_gateway_before_uvicorn( self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path ): From 5be0eb7666164af2acf4bb2dc8018c84a9d7662f Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 09:07:08 +0900 Subject: [PATCH 11/17] test(proxy): expect the listener fd argument in uvicorn init calls Co-Authored-By: Claude Opus 5.5 --- tests/unit/proxy/test_proxy_cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index 643c01e3e7f..db36f0d681f 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -1550,6 +1550,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=30, timeout_worker_healthcheck=None, + listener_fd=None, ) mock_uvicorn_run.assert_called_once() @@ -1626,6 +1627,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=None, timeout_worker_healthcheck=15, + listener_fd=None, ) @patch("uvicorn.run") From 543c01530530f30eb0397f4eca1f1318f43df871 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 09:07:08 +0900 Subject: [PATCH 12/17] fix(proxy): restore the client's peer behind Rust instead of trusting the Rust hop Python now sees each request as if the client had connected to it directly. The Rust hop sends the real TCP peer in x-litellm-rust-peer, an app middleware puts it back into the ASGI scope, and uvicorn's ProxyHeadersMiddleware then runs with the same trust settings Python alone would use. Server-level proxy headers are off in supervised mode for uvicorn and gunicorn, since they would judge the loopback hop. The trust-list merge that always trusted 127.0.0.1 is gone, so operators keep the exact FORWARDED_ALLOW_IPS behavior they had. Co-Authored-By: Claude Opus 5.5 --- gateway/launch.py | 15 +- litellm/proxy/proxy_cli.py | 6 +- litellm/proxy/proxy_server.py | 5 + litellm/proxy/rust_gateway.py | 59 ++++++-- .../proxy/shutdown/graceful_uvicorn_worker.py | 10 +- .../configuration/test_rust_gateway.py | 2 +- tests/unit/gateway/test_launch.py | 27 ++-- tests/unit/proxy/test_rust_gateway.py | 135 +++++++++++++++--- 8 files changed, 206 insertions(+), 53 deletions(-) diff --git a/gateway/launch.py b/gateway/launch.py index d5d798f262b..5dec5de46fa 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -46,9 +46,7 @@ def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str], supervised: "S ( "--fd", str(supervised.listener_fd), - "--proxy-headers", - "--forwarded-allow-ips", - ",".join(supervised.trusted_proxies), + "--no-proxy-headers", ) if supervised is not None else () @@ -93,7 +91,7 @@ def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, """Validate the uvicorn flags for Rust forwarding and return the binary, public address and child environment.""" from pydantic import TypeAdapter - from litellm.proxy.rust_gateway import TRUSTED_PROXIES, startup_error + from litellm.proxy.rust_gateway import PROXY_HEADERS, TRUSTED_PROXIES, startup_error params: Final[Mapping[str, object]] = uvicorn_main.make_context( "uvicorn", list(uvicorn_argv(argv, os.environ, None)) @@ -113,7 +111,14 @@ def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, assert binary is not None host, port = TypeAdapter(tuple[str, int]).validate_python((params["host"], params["port"])) operator_trust: Final = TypeAdapter(str | None).validate_python(params["forwarded_allow_ips"]) - return binary, host, port, {} if operator_trust is None else {TRUSTED_PROXIES: operator_trust} + proxy_headers: Final = TypeAdapter(bool).validate_python(params["proxy_headers"]) + return ( + binary, + host, + port, + {PROXY_HEADERS: str(proxy_headers).lower()} + | ({} if operator_trust is None else {TRUSTED_PROXIES: operator_trust}), + ) def configure_database() -> None: diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 68db6d3f316..d1520fd56d9 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -429,7 +429,7 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), - **({"fd": listener_fd} if listener_fd is not None else {}), + **({"fd": listener_fd, "proxy_headers": False} if listener_fd is not None else {}), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -758,7 +758,9 @@ class ProxyInitializationHelpers: gunicorn_options: Final = { "bind": f"fd://{listener_fd}" if listener_fd is not None else f"{host}:{port}", "workers": num_workers, # default is 1 - "worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout), + "worker_class": build_graceful_shutdown_uvicorn_worker( + graceful_shutdown_timeout, proxy_headers=listener_fd is None + ), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, "preload": True, # Add the preload flag, "accesslog": "-", # Log to stdout diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c5fcbfb2564..8a50c116125 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -835,6 +835,7 @@ from litellm.proxy.plugin_routes import ( from litellm.proxy.plugin_routes import ( router as plugin_router, ) +from litellm.proxy.rust_gateway import RustPeerMiddleware, Supervised, rust_gateway_mode from litellm.proxy.spend_tracking.spend_event_producer import ( CollectorSettings, SpendEventProducer, @@ -21924,6 +21925,10 @@ if _server_root_paths: ) app.add_middleware(PerRequestRootPathMiddleware, root_paths=_server_root_paths) +_rust_gateway_mode: Final = rust_gateway_mode(os.environ) +if isinstance(_rust_gateway_mode, Supervised): + app.add_middleware(RustPeerMiddleware, mode=_rust_gateway_mode) + async def _stream_mcp_asgi_response(handle_fn, scope: dict, receive) -> "StreamingResponse": """ diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 65237eb00e0..0dbf3a4de3d 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -9,16 +9,22 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from enum import Enum, IntEnum from types import MappingProxyType -from typing import Final, TypeAlias +from typing import TYPE_CHECKING, Final, TypeAlias from pydantic import TypeAdapter from typing_extensions import assert_never +if TYPE_CHECKING: + from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope + ENABLE: Final = "LITELLM_RUST_GATEWAY" GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" +PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" +DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1" RUST_HOP: Final = "127.0.0.1" +PEER_HEADER: Final = b"x-litellm-rust-peer" _FD: Final = TypeAdapter(int) _ENABLED: Final = TypeAdapter(bool) @@ -39,26 +45,62 @@ class Supervisor: class Supervised: listener_fd: int metrics_host: str - trusted_proxies: tuple[str, ...] + proxy_headers: bool + trusted_proxies: str RustGatewayMode: TypeAlias = Disabled | Supervisor | Supervised -def trusted_proxies(operator_value: str | None) -> tuple[str, ...]: - configured: Final = tuple(host.strip() for host in (operator_value or "").split(",") if host.strip()) - return ("*",) if "*" in configured else (RUST_HOP, *(host for host in configured if host != RUST_HOP)) - - def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode: fd: Final = environ.get(GATEWAY_FD) if fd is not None: - return Supervised(_FD.validate_python(fd), environ[METRICS_HOST], trusted_proxies(environ.get(TRUSTED_PROXIES))) + return Supervised( + _FD.validate_python(fd), + environ[METRICS_HOST], + _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), + environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), + ) if _ENABLED.validate_python(environ.get(ENABLE, "false")): return Supervisor() return Disabled() +def _restore_peer(scope: "WWWScope") -> None: + headers: Final = tuple(scope["headers"]) + peer: Final = next((value for name, value in headers if name == PEER_HEADER), None) + if peer is None: + return + remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER) + scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop + client: Final = scope["client"] + if client is None or client[0] != RUST_HOP: + return + host, _, port = peer.decode("latin-1").rpartition(":") + scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope + + +class RustPeerMiddleware: + """Make Python see each request as if the client had connected to it directly. + + Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real + TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the + real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust + settings Python would have used on the public port. The server's proxy-header handling is off in + this mode, since it would judge Rust's loopback address instead of the real peer. + """ + + def __init__(self, app: "ASGI3Application", mode: Supervised) -> None: + from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + + self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app + + async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + if scope["type"] != "lifespan": + _restore_peer(scope) + await self.app(scope, receive, send) + + def gateway_binary(scripts_dir: str) -> str | None: name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway" return shutil.which(name, path=scripts_dir) @@ -175,7 +217,6 @@ def python_spec( **environ, GATEWAY_FD: str(listener_fd), METRICS_HOST: public_host, - TRUSTED_PROXIES: ",".join(trusted_proxies(environ.get(TRUSTED_PROXIES))), } ), pass_fds=(listener_fd,), diff --git a/litellm/proxy/shutdown/graceful_uvicorn_worker.py b/litellm/proxy/shutdown/graceful_uvicorn_worker.py index 6ae5ae60c3f..8bbd64190f8 100644 --- a/litellm/proxy/shutdown/graceful_uvicorn_worker.py +++ b/litellm/proxy/shutdown/graceful_uvicorn_worker.py @@ -1,10 +1,16 @@ from uvicorn.workers import UvicornWorker -def build_graceful_shutdown_uvicorn_worker(timeout_graceful_shutdown: float) -> type[UvicornWorker]: +def build_graceful_shutdown_uvicorn_worker( + timeout_graceful_shutdown: float, *, proxy_headers: bool = True +) -> type[UvicornWorker]: class GracefulShutdownUvicornWorker(UvicornWorker): def __init__(self, *args: object) -> None: - self.CONFIG_KWARGS = {**UvicornWorker.CONFIG_KWARGS, "timeout_graceful_shutdown": timeout_graceful_shutdown} + self.CONFIG_KWARGS = { + **UvicornWorker.CONFIG_KWARGS, + "timeout_graceful_shutdown": timeout_graceful_shutdown, + "proxy_headers": proxy_headers, + } super().__init__(*args) return GracefulShutdownUvicornWorker diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index 729c0b847e3..06d77d6f3d0 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -22,7 +22,6 @@ def ready(client: httpx.Client) -> bool: return False -@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) def client_ip(base_url: str, local_address: str, key: str, forwarded_for: str) -> httpx.Response: with httpx.Client( base_url=base_url, @@ -36,6 +35,7 @@ def client_ip(base_url: str, local_address: str, key: str, forwarded_for: str) - ) +@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) def test_gateway_preserves_metrics_bind_and_mcp_client_ip( gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...] ) -> None: diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index 606e760ab3f..789067653b7 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -127,12 +127,16 @@ class TestUvicornArgv: @pytest.mark.parametrize( ("argv", "trust"), [ - ((), {}), - (("--forwarded-allow-ips", "10.0.0.5"), {"FORWARDED_ALLOW_IPS": "10.0.0.5"}), - (("--forwarded-allow-ips=*",), {"FORWARDED_ALLOW_IPS": "*"}), + ((), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true"}), + ( + ("--forwarded-allow-ips", "10.0.0.5"), + {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "10.0.0.5"}, + ), + (("--forwarded-allow-ips=*",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "*"}), + (("--no-proxy-headers",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}), ], ) - def test_forwarding_carries_operator_proxy_trust_to_the_child( + def test_forwarding_carries_operator_proxy_header_options_to_the_child( self, argv: tuple[str, ...], trust: dict[str, str] ) -> None: assert public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") == ( @@ -143,20 +147,15 @@ class TestUvicornArgv: ) def test_supervised_listener_uses_the_inherited_loopback_socket(self): - supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1",)) + supervised: Final = Supervised(17, "0.0.0.0", True, "127.0.0.1") params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised)) assert params["fd"] == 17 assert params["workers"] == 2 - @pytest.mark.parametrize( - "argv", - [(), ("--no-proxy-headers",), ("--forwarded-allow-ips", "192.0.2.1"), ("--forwarded-allow-ips=*",)], - ) - def test_private_hop_always_processes_headers_with_the_resolved_trust(self, argv: tuple[str, ...]): - supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5")) - params: Final = _uvicorn_params(uvicorn_argv(argv, {}, supervised)) - assert params["proxy_headers"] is True - assert params["forwarded_allow_ips"] == "127.0.0.1,10.0.0.5" + @pytest.mark.parametrize("argv", [(), ("--proxy-headers",), ("--forwarded-allow-ips=*",)]) + def test_private_hop_leaves_proxy_headers_to_the_app(self, argv: tuple[str, ...]): + params: Final = _uvicorn_params(uvicorn_argv(argv, {}, Supervised(17, "0.0.0.0", True, "*"))) + assert params["proxy_headers"] is False def test_direct_listener_keeps_explicit_proxy_header_options(self): params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {}, None)) diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index 278dc4621d3..b71a098ddb3 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -1,12 +1,24 @@ +import asyncio import functools import signal import sys from pathlib import Path -from typing import Final +from typing import TYPE_CHECKING, Final import pytest from click.testing import CliRunner from pydantic import ValidationError +from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + +if TYPE_CHECKING: + from uvicorn._types import ( + ASGIReceiveCallable, + ASGISendCallable, + ASGISendEvent, + HTTPDisconnectEvent, + HTTPScope, + Scope, + ) from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server from litellm.proxy.rust_gateway import ( @@ -17,6 +29,7 @@ from litellm.proxy.rust_gateway import ( Exited, Finished, Running, + RustPeerMiddleware, Signal, State, Stop, @@ -29,7 +42,6 @@ from litellm.proxy.rust_gateway import ( rust_spec, startup_error, step, - trusted_proxies, ) TERM: Final = signal.SIGTERM @@ -66,7 +78,7 @@ def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", }, - Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5")), + Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"), ), ], ) @@ -80,27 +92,17 @@ def test_unrecognized_switch_value_is_rejected() -> None: @pytest.mark.parametrize( - ("operator_value", "expected"), + ("environ", "expected"), [ - (None, ("127.0.0.1",)), - ("", ("127.0.0.1",)), - ("10.0.0.5", ("127.0.0.1", "10.0.0.5")), - ("127.0.0.1, 10.0.0.5", ("127.0.0.1", "10.0.0.5")), - ("10.0.0.5,127.0.0.1", ("127.0.0.1", "10.0.0.5")), - ("*", ("*",)), - ("10.0.0.5,*", ("*",)), + ({}, Supervised(17, "0.0.0.0", True, "127.0.0.1")), + ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, Supervised(17, "0.0.0.0", True, "10.0.0.5")), + ({"FORWARDED_ALLOW_IPS": "*"}, Supervised(17, "0.0.0.0", True, "*")), + ({"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}, Supervised(17, "0.0.0.0", False, "127.0.0.1")), ], ) -def test_operator_trust_is_kept_and_rust_hop_added(operator_value: str | None, expected: tuple[str, ...]) -> None: - assert trusted_proxies(operator_value) == expected - - -@pytest.mark.parametrize("operator_value", [None, "", "10.0.0.5", "10.0.0.5, 192.0.2.0/24", "127.0.0.1"]) -def test_python_child_always_trusts_the_rust_hop(operator_value: str | None) -> None: - environ: Final = {} if operator_value is None else {"FORWARDED_ALLOW_IPS": operator_value} +def test_python_child_keeps_the_trust_python_alone_would_use(environ: dict[str, str], expected: Supervised) -> None: spec: Final = python_spec(("python",), 17, 45555, "0.0.0.0", environ) - assert spec.env["FORWARDED_ALLOW_IPS"].split(",")[0] == "127.0.0.1" - assert rust_gateway_mode(spec.env) == Supervised(17, "0.0.0.0", trusted_proxies(operator_value)) + assert rust_gateway_mode(spec.env) == expected def test_python_child_listens_privately_and_keeps_options() -> None: @@ -152,6 +154,7 @@ def test_inherited_socket_reaches_uvicorn(listener_fd: int | None) -> None: "127.0.0.1", 45555, listener_fd=listener_fd ) assert options.get("fd") == listener_fd + assert options.get("proxy_headers", True) is (listener_fd is None) def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]: @@ -200,3 +203,95 @@ def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit( events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode ) -> None: assert _fold(events) == (Finished(code), signals) + + +Headers = tuple[tuple[bytes, bytes], ...] + + +async def _receive() -> "HTTPDisconnectEvent": + return {"type": "http.disconnect"} + + +async def _send(message: "ASGISendEvent") -> None: + return None + + +async def _seen_by_app( + behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised +) -> tuple[tuple[str, int] | None, str, Headers]: + seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = ( + asyncio.get_running_loop().create_future() + ) + + async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + assert scope["type"] == "http" + seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"]))) + + direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe + app: Final = RustPeerMiddleware(probe, mode) if behind_rust else direct + scope: Final[HTTPScope] = { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": "/", + "raw_path": b"/", + "query_string": b"", + "root_path": "", + "headers": headers, + "client": client, + "server": ("127.0.0.1", 4000), + } + await app(scope, _receive, _send) + return seen.result() + + +PEER: Final = ("203.0.113.7", 51234) +TRUSTED_PEER: Final = ("10.0.12.34", 40000) +FORWARDED: Final[Headers] = ( + (b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"), + (b"x-forwarded-proto", b"https"), +) + + +@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)]) +@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))]) +@pytest.mark.parametrize( + "mode", + [ + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + Supervised(17, "0.0.0.0", True, "10.0.0.0/8"), + Supervised(17, "0.0.0.0", True, "*"), + Supervised(17, "0.0.0.0", False, "10.0.0.0/8"), + ], +) +async def test_python_behind_rust_sees_what_a_direct_connection_would( + peer: tuple[str, int], headers: Headers, mode: Supervised +) -> None: + rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}" + behind_rust: Final = await _seen_by_app( + True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode + ) + direct: Final = await _seen_by_app(False, peer, headers, mode) + assert behind_rust == direct + + +async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None: + client, scheme, _ = await _seen_by_app( + True, + ("127.0.0.1", 60000), + ((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")), + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + ) + assert (client, scheme) == (("203.0.113.7", 51234), "http") + + +async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None: + client, _, headers = await _seen_by_app( + True, + ("192.0.2.9", 1000), + ((b"x-litellm-rust-peer", b"10.0.12.34:1"),), + Supervised(17, "0.0.0.0", False, "127.0.0.1"), + ) + assert (client, headers) == (("192.0.2.9", 1000), ()) From 4a8563b01b2a9f086836f839883cb07ed13521b4 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 10:24:24 +0900 Subject: [PATCH 13/17] feat(proxy): opt into Rust forwarding with LITELLM_RUST One switch turns on every optional Rust path, so forwarding reads the same variable through the same parser as the rust bridge instead of its own LITELLM_RUST_GATEWAY. An unrecognized value now leaves forwarding off, like every other Rust path, instead of failing startup. Co-Authored-By: Claude Opus 5.5 --- litellm/proxy/rust_gateway.py | 11 +++---- litellm/rust_bridge/configuration.py | 9 ++++-- .../configuration/test_rust_gateway.py | 4 +-- tests/unit/gateway/test_launch.py | 2 +- tests/unit/proxy/test_rust_gateway.py | 30 ++++++++----------- 5 files changed, 29 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 0dbf3a4de3d..8be060e362c 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -14,10 +14,11 @@ from typing import TYPE_CHECKING, Final, TypeAlias from pydantic import TypeAdapter from typing_extensions import assert_never +from litellm.rust_bridge.configuration import ENV_NAME, environment_switch + if TYPE_CHECKING: from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope -ENABLE: Final = "LITELLM_RUST_GATEWAY" GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" @@ -61,7 +62,7 @@ def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode: _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), ) - if _ENABLED.validate_python(environ.get(ENABLE, "false")): + if environment_switch(environ): return Supervisor() return Disabled() @@ -109,11 +110,11 @@ def gateway_binary(scripts_dir: str) -> str | None: def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: flag: Final = next((name for name, value in unsupported.items() if value), None) if flag is not None: - return f"{ENABLE} does not support {flag}" + return f"{ENV_NAME} does not support {flag}" if os.name != "posix": - return f"{ENABLE} requires POSIX inherited sockets and process signals" + return f"{ENV_NAME} requires POSIX inherited sockets and process signals" if binary is None: - return f"{ENABLE} requires the litellm-gateway binary. Install litellm[proxy] in this environment" + return f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment" return None diff --git a/litellm/rust_bridge/configuration.py b/litellm/rust_bridge/configuration.py index 152ba632996..e36527e9f4e 100644 --- a/litellm/rust_bridge/configuration.py +++ b/litellm/rust_bridge/configuration.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +from collections.abc import Mapping from enum import Enum, auto from functools import lru_cache from typing import Final @@ -8,7 +9,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never -_GLOBAL_ENV_NAME: Final = "LITELLM_RUST" +ENV_NAME: Final = "LITELLM_RUST" _ENV_BOOL: Final = TypeAdapter(bool) @@ -44,6 +45,10 @@ def _parse_env_bool(value: str | None) -> bool | None: return None +def environment_switch(environ: Mapping[str, str]) -> bool | None: + return _parse_env_bool(environ.get(ENV_NAME)) + + def decide( rollout: Rollout, *, @@ -72,7 +77,7 @@ def decision(rollout: Rollout) -> Decision: return decide( rollout, process_override=_CONFIGURATION.override, - environment_override=_parse_env_bool(os.getenv(_GLOBAL_ENV_NAME)), + environment_override=environment_switch(os.environ), ) diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index 06d77d6f3d0..b75da286bf7 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -55,7 +55,7 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( "LITELLM_MASTER_KEY": gateway.key, "STORE_MODEL_IN_DB": "False", "PYTHON_DOTENV_DISABLED": "1", - "LITELLM_RUST_GATEWAY": "true", + "LITELLM_RUST": "true", "FORWARDED_ALLOW_IPS": TRUSTED_BALANCER, } with (tmp_path / "proxy.log").open("wb") as output: @@ -124,7 +124,7 @@ def test_gateway_launcher_private_hop_preserves_client_address( environment: Final = { **{key: value for key, value in os.environ.items() if not key.startswith("DATABASE_") and key != "DIRECT_URL"}, "LITELLM_MASTER_KEY": "sk-rust-forwarding-integration-key-not-a-secret", - "LITELLM_RUST_GATEWAY": "true", + "LITELLM_RUST": "true", "LITELLM_PGBOUNCER_ENABLED": "false", "CONFIG_FILE_PATH": str(config), "STORE_MODEL_IN_DB": "False", diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index 789067653b7..418e0358b64 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -280,7 +280,7 @@ class TestMain: def test_forwarding_disabled_keeps_uvicorn_socket_options( self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch ) -> None: - monkeypatch.setenv("LITELLM_RUST_GATEWAY", "false") + monkeypatch.setenv("LITELLM_RUST", "false") monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False) served: Final[list[tuple[str, ...]]] = [] diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index b71a098ddb3..1072148be9f 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Final import pytest from click.testing import CliRunner -from pydantic import ValidationError from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware if TYPE_CHECKING: @@ -52,28 +51,30 @@ INT: Final = signal.SIGINT "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] ) def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("LITELLM_RUST_GATEWAY", "true") + monkeypatch.setenv("LITELLM_RUST", "true") monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] result: Final = CliRunner().invoke(run_server, [flag, *values]) assert result.exit_code != 0 - assert f"LITELLM_RUST_GATEWAY does not support {flag}" in result.output + assert f"LITELLM_RUST does not support {flag}" in result.output @pytest.mark.parametrize( ("environ", "expected"), [ ({}, Disabled()), - ({"LITELLM_RUST_GATEWAY": "false"}, Disabled()), - ({"LITELLM_RUST_GATEWAY": "off"}, Disabled()), - ({"LITELLM_RUST_GATEWAY": "0"}, Disabled()), - ({"LITELLM_RUST_GATEWAY": "true"}, Supervisor()), - ({"LITELLM_RUST_GATEWAY": "1"}, Supervisor()), - ({"LITELLM_RUST_GATEWAY": "yes"}, Supervisor()), - ({"LITELLM_RUST_GATEWAY": "on"}, Supervisor()), + ({"LITELLM_RUST": "false"}, Disabled()), + ({"LITELLM_RUST": "off"}, Disabled()), + ({"LITELLM_RUST": "0"}, Disabled()), + ({"LITELLM_RUST": "true"}, Supervisor()), + ({"LITELLM_RUST": "1"}, Supervisor()), + ({"LITELLM_RUST": "yes"}, Supervisor()), + ({"LITELLM_RUST": "on"}, Supervisor()), + ({"LITELLM_RUST": " 1 "}, Supervisor()), + ({"LITELLM_RUST": "enabled"}, Disabled()), ( { - "LITELLM_RUST_GATEWAY": "true", + "LITELLM_RUST": "true", "_LITELLM_GATEWAY_FD": "17", "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", @@ -86,11 +87,6 @@ def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: ob assert rust_gateway_mode(environ) == expected -def test_unrecognized_switch_value_is_rejected() -> None: - with pytest.raises(ValidationError): - rust_gateway_mode({"LITELLM_RUST_GATEWAY": "enabled"}) - - @pytest.mark.parametrize( ("environ", "expected"), [ @@ -131,7 +127,7 @@ def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> Non def test_missing_binary_gives_install_instruction() -> None: assert startup_error(None, {}) == ( - "LITELLM_RUST_GATEWAY requires the litellm-gateway binary. Install litellm[proxy] in this environment" + "LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment" ) From b2e18b93a39d47843841dfa7d9bb7860f2f46183 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 13:21:05 +0900 Subject: [PATCH 14/17] fix(proxy): gather Rust forwarding into rust_forward and hand Rust Python's keepalive and drain Every Python line that exists only because Rust forwards to Python now lives in litellm/proxy/rust_forward (mode, peer, listener, supervisor, asgi), paired with the gateway-forward crate. proxy_server no longer knows about Rust. proxy_cli and gateway.launch wrap the app where they load it, apply server_overrides in the supervised child, and turn their own flags into one PublicListener The supervisor now resolves the keepalive and shutdown drain Python actually runs with and passes them to Rust as LITELLM_FORWARD_UPSTREAM_KEEPALIVE and LITELLM_FORWARD_SHUTDOWN_DRAIN, along with LITELLM_FORWARD_HOST and LITELLM_FORWARD_PORT. Before, `--keepalive_timeout 2` gave uvicorn 2s while Rust kept its default 5s pool and reused connections uvicorn was closing Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test-unit.yml | 1 + gateway/launch.py | 77 +++-- litellm/proxy/proxy_cli.py | 99 ++++-- litellm/proxy/proxy_server.py | 5 - litellm/proxy/rust_forward/AGENTS.md | 8 + litellm/proxy/rust_forward/__init__.py | 0 litellm/proxy/rust_forward/asgi.py | 10 + litellm/proxy/rust_forward/listener.py | 46 +++ litellm/proxy/rust_forward/mode.py | 68 ++++ litellm/proxy/rust_forward/peer.py | 57 ++++ .../supervisor.py} | 133 ++------ ...t_rust_gateway.py => test_rust_forward.py} | 0 tests/unit/gateway/test_launch.py | 63 +++- tests/unit/proxy/rust_forward/__init__.py | 0 .../unit/proxy/rust_forward/test_listener.py | 74 +++++ tests/unit/proxy/rust_forward/test_mode.py | 43 +++ tests/unit/proxy/rust_forward/test_peer.py | 144 +++++++++ .../proxy/rust_forward/test_supervisor.py | 139 +++++++++ tests/unit/proxy/test_proxy_cli.py | 4 +- tests/unit/proxy/test_rust_gateway.py | 293 ------------------ 20 files changed, 769 insertions(+), 495 deletions(-) create mode 100644 litellm/proxy/rust_forward/AGENTS.md create mode 100644 litellm/proxy/rust_forward/__init__.py create mode 100644 litellm/proxy/rust_forward/asgi.py create mode 100644 litellm/proxy/rust_forward/listener.py create mode 100644 litellm/proxy/rust_forward/mode.py create mode 100644 litellm/proxy/rust_forward/peer.py rename litellm/proxy/{rust_gateway.py => rust_forward/supervisor.py} (53%) rename tests/integration/configuration/{test_rust_gateway.py => test_rust_forward.py} (100%) create mode 100644 tests/unit/proxy/rust_forward/__init__.py create mode 100644 tests/unit/proxy/rust_forward/test_listener.py create mode 100644 tests/unit/proxy/rust_forward/test_mode.py create mode 100644 tests/unit/proxy/rust_forward/test_peer.py create mode 100644 tests/unit/proxy/rust_forward/test_supervisor.py delete mode 100644 tests/unit/proxy/test_rust_gateway.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 17f68cd7d32..5e61d6ebd4c 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -583,6 +583,7 @@ jobs: tests/unit/proxy/credential_endpoints tests/unit/proxy/discovery_endpoints tests/unit/proxy/shutdown + tests/unit/proxy/rust_forward tests/unit/proxy/public_endpoints tests/unit/proxy/prompts tests/unit/proxy/rag_endpoints diff --git a/gateway/launch.py b/gateway/launch.py index 5dec5de46fa..89341a4cb89 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -29,27 +29,29 @@ from uvicorn.main import main as uvicorn_main from litellm_cli.main import without_gc_pauses if TYPE_CHECKING: - from fastapi import FastAPI - from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings - from litellm.proxy.rust_gateway import Supervised + from litellm.proxy.rust_forward.listener import PublicListener + from litellm.proxy.rust_forward.mode import ServerOverrides + from litellm.proxy.rust_forward.peer import ASGIApp GATEWAY_APP: Final = "gateway.launch:worker_app" KEEPALIVE_FLAG: Final = "--timeout-keep-alive" -def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str], supervised: "Supervised | None") -> tuple[str, ...]: +def uvicorn_argv( + argv: Sequence[str], environ: Mapping[str, str], forwarded: "ServerOverrides | None" +) -> tuple[str, ...]: """Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly.""" keepalive: Final = environ.get("KEEPALIVE_TIMEOUT") inherited_socket: Final = ( - ( + () + if forwarded is None + else ( "--fd", - str(supervised.listener_fd), - "--no-proxy-headers", + str(forwarded.fd), + "--proxy-headers" if forwarded.proxy_headers else "--no-proxy-headers", ) - if supervised is not None - else () ) if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv): return (GATEWAY_APP, "--factory", *argv, *inherited_socket) @@ -77,26 +79,29 @@ def pool_database_url( return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth()) -def worker_app() -> "FastAPI": +def worker_app() -> "ASGIApp": with without_gc_pauses(): from gateway.main import app - return app + from litellm.proxy.rust_forward.mode import rust_forward_mode + from litellm.proxy.rust_forward.peer import wrap + return wrap(app, rust_forward_mode(os.environ)) def _serve(argv: Sequence[str]) -> None: uvicorn_main(tuple(argv), prog_name="uvicorn") -def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, int, Mapping[str, str]]: - """Validate the uvicorn flags for Rust forwarding and return the binary, public address and child environment.""" +def public_listener(argv: Sequence[str], binary: str | None) -> "tuple[str, PublicListener]": + """Validate the uvicorn flags for Rust forwarding and return the binary and the public listener.""" from pydantic import TypeAdapter - from litellm.proxy.rust_gateway import PROXY_HEADERS, TRUSTED_PROXIES, startup_error + from litellm.proxy.rust_forward.listener import PublicListener, StartupError, shutdown_drain_seconds, startup_check + from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager params: Final[Mapping[str, object]] = uvicorn_main.make_context( "uvicorn", list(uvicorn_argv(argv, os.environ, None)) ).params - error: Final = startup_error( + checked: Final = startup_check( binary, { "--ssl-keyfile": params["ssl_keyfile"], @@ -106,18 +111,18 @@ def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, "--fd": params["fd"] is not None, }, ) - if error is not None: - sys.exit(error) - assert binary is not None - host, port = TypeAdapter(tuple[str, int]).validate_python((params["host"], params["port"])) - operator_trust: Final = TypeAdapter(str | None).validate_python(params["forwarded_allow_ips"]) - proxy_headers: Final = TypeAdapter(bool).validate_python(params["proxy_headers"]) - return ( - binary, - host, - port, - {PROXY_HEADERS: str(proxy_headers).lower()} - | ({} if operator_trust is None else {TRUSTED_PROXIES: operator_trust}), + if isinstance(checked, StartupError): + sys.exit(checked.reason) + graceful_timeout: Final = TypeAdapter[int | None](int | None).validate_python(params["timeout_graceful_shutdown"]) + return checked, PublicListener( + host=TypeAdapter(str).validate_python(params["host"]), + port=TypeAdapter(int).validate_python(params["port"]), + keepalive_seconds=TypeAdapter(int).validate_python(params["timeout_keep_alive"]), + shutdown_drain_seconds=shutdown_drain_seconds( + GracefulShutdownManager.get_timeout() if graceful_timeout is None else graceful_timeout + ), + proxy_headers=TypeAdapter(bool).validate_python(params["proxy_headers"]), + trusted_proxies=TypeAdapter[str | None](str | None).validate_python(params["forwarded_allow_ips"]), ) @@ -135,27 +140,21 @@ def configure_database() -> None: def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: - from litellm.proxy.rust_gateway import ( - Disabled, - Supervised, - Supervisor, - gateway_binary, - rust_gateway_mode, - supervise, - ) + from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides + from litellm.proxy.rust_forward.supervisor import gateway_binary, supervise - mode: Final = rust_gateway_mode(os.environ) + mode: Final = rust_forward_mode(os.environ) match mode: case Disabled(): configure_database() serve(uvicorn_argv(argv, os.environ, None)) case Supervisor(): - binary, host, port, trust = public_listener(argv, gateway_binary(sysconfig.get_path("scripts"))) + binary, listener = public_listener(argv, gateway_binary(sysconfig.get_path("scripts"))) configure_database() command: Final = (sys.executable, "-m", "gateway.launch", *argv) - sys.exit(supervise(binary, command, host, port, {**os.environ, **trust})) + sys.exit(supervise(binary, command, listener, os.environ)) case Supervised(): - serve(uvicorn_argv(argv, os.environ, mode)) + serve(uvicorn_argv(argv, os.environ, server_overrides(mode))) case _: assert_never(mode) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index d1520fd56d9..cb8e936f3c5 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -36,6 +36,10 @@ from litellm.types.llms.base import LiteLLMBaseModel if TYPE_CHECKING: from fastapi import FastAPI + + from litellm.proxy.rust_forward.listener import PublicListener + from litellm.proxy.rust_forward.mode import ServerOverrides + from litellm.proxy.rust_forward.peer import ASGIApp else: FastAPI = Any @@ -262,6 +266,7 @@ def _apply_config_logging_settings(litellm_settings: StartupOutputSettings | Non WORKER_APP: Final = "litellm_cli.main:proxy_worker_app" +FORWARD_APP: Final = "litellm.proxy.rust_forward.asgi:app" WORKER_LOG_CONFIG_ENV: Final = "LITELLM_WORKER_LOG_CONFIG" _LOG_CONFIG: Final = TypeAdapter[str | Mapping[str, object]](str | Mapping[str, object]) _LOG_CONFIG_FILE: Final = TypeAdapter[Mapping[str, object]](Mapping[str, object]) @@ -390,12 +395,12 @@ class ProxyInitializationHelpers: logging.config.fileConfig(log_config, disable_existing_loggers=False) @staticmethod - def child_process_uvicorn_overrides(log_config: object) -> Mapping[str, object]: + def child_process_uvicorn_overrides(log_config: object, app: str = WORKER_APP) -> Mapping[str, object]: if log_config is None: - return MappingProxyType({"app": WORKER_APP, "factory": True}) + return MappingProxyType({"app": app, "factory": True}) ProxyInitializationHelpers.apply_log_config(_LOG_CONFIG.validate_python(log_config)) os.environ[WORKER_LOG_CONFIG_ENV] = json.dumps(log_config) - return MappingProxyType({"app": WORKER_APP, "factory": True, "log_config": None}) + return MappingProxyType({"app": app, "factory": True, "log_config": None}) @staticmethod def apply_worker_log_config(environ: Mapping[str, str]) -> None: @@ -410,7 +415,7 @@ class ProxyInitializationHelpers: log_config: str | None = None, keepalive_timeout: int | None = None, timeout_worker_healthcheck: int | None = None, - listener_fd: int | None = None, + forwarded: "ServerOverrides | None" = None, ) -> dict: """ Get the arguments for `uvicorn` worker @@ -429,7 +434,16 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), - **({"fd": listener_fd, "proxy_headers": False} if listener_fd is not None else {}), + **( + {} + if forwarded is None + else { + "app": FORWARD_APP, + "factory": True, + "fd": forwarded.fd, + "proxy_headers": forwarded.proxy_headers, + } + ), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -677,13 +691,13 @@ class ProxyInitializationHelpers: def run_gunicorn_server( host: str, port: int, - app: FastAPI, + app: "ASGIApp", num_workers: int, ssl_certfile_path: str, ssl_keyfile_path: str, max_requests_before_restart: int | None = None, max_requests_before_restart_jitter: int | None = None, - listener_fd: int | None = None, + forwarded: "ServerOverrides | None" = None, ): """ Run litellm with `gunicorn` @@ -756,10 +770,10 @@ class ProxyInitializationHelpers: ) graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout() gunicorn_options: Final = { - "bind": f"fd://{listener_fd}" if listener_fd is not None else f"{host}:{port}", + "bind": f"{host}:{port}" if forwarded is None else f"fd://{forwarded.fd}", "workers": num_workers, # default is 1 "worker_class": build_graceful_shutdown_uvicorn_worker( - graceful_shutdown_timeout, proxy_headers=listener_fd is None + graceful_shutdown_timeout, proxy_headers=True if forwarded is None else forwarded.proxy_headers ), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, "preload": True, # Add the preload flag, @@ -829,6 +843,34 @@ class ProxyInitializationHelpers: _is_port_in_use = is_port_in_use + @staticmethod + def public_listener(host: str, port: int, keepalive_timeout: int | None, run_gunicorn: bool) -> "PublicListener": + """``run_gunicorn_server`` never hands ``--keepalive_timeout`` to gunicorn, so its workers keep gunicorn's.""" + from gunicorn.config import Config as GunicornConfig + + from litellm.proxy.rust_forward.listener import ( + PublicListener, + shutdown_drain_seconds, + uvicorn_default_keepalive, + ) + from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager + + keepalive: Final = ( + TypeAdapter(int).validate_python(GunicornConfig().keepalive) + if run_gunicorn + else (uvicorn_default_keepalive() if keepalive_timeout is None else keepalive_timeout) + ) + return PublicListener( + host=host, + port=random.randint(1024, 49152) + if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) + else port, + keepalive_seconds=keepalive, + shutdown_drain_seconds=shutdown_drain_seconds(GracefulShutdownManager.get_timeout()), + proxy_headers=True, + trusted_proxies=None, + ) + @staticmethod def get_loop_type(): """Helper function to determine the event loop type based on platform""" @@ -1286,17 +1328,17 @@ def run_server( run_setup_wizard() return - from litellm.proxy.rust_gateway import Supervised, Supervisor, rust_gateway_mode + from litellm.proxy.rust_forward.mode import Supervised, Supervisor, rust_forward_mode, server_overrides - gateway_mode: Final = rust_gateway_mode(os.environ) - if isinstance(gateway_mode, Supervisor) and not any( + forward_mode: Final = rust_forward_mode(os.environ) + if isinstance(forward_mode, Supervisor) and not any( (version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data) ): - from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise + from litellm.proxy.rust_forward.listener import StartupError, startup_check + from litellm.proxy.rust_forward.supervisor import cli_command, gateway_binary, supervise - binary: Final = gateway_binary(sysconfig.get_path("scripts")) - gateway_error: Final = startup_error( - binary, + binary: Final = startup_check( + gateway_binary(sysconfig.get_path("scripts")), { "--ssl_keyfile_path": ssl_keyfile_path, "--ssl_certfile_path": ssl_certfile_path, @@ -1305,14 +1347,11 @@ def run_server( "--run_granian": run_granian, }, ) - if gateway_error is not None: - raise click.ClickException(gateway_error) - assert binary is not None - public_port: Final = ( - random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port - ) - raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port, os.environ)) - supervised: Final = gateway_mode if isinstance(gateway_mode, Supervised) else None + if isinstance(binary, StartupError): + raise click.ClickException(binary.reason) + listener: Final = ProxyInitializationHelpers.public_listener(host, port, keepalive_timeout, run_gunicorn) + raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), listener, os.environ)) + forwarded: Final = server_overrides(forward_mode) if isinstance(forward_mode, Supervised) else None args: Final = locals() try: @@ -1763,7 +1802,7 @@ def run_server( if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None: from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process - metrics_bind_host: Final = supervised.metrics_host if supervised is not None else host + metrics_bind_host: Final = host if forwarded is None else forwarded.metrics_host try: metrics_process: Final = start_metrics_server_process( host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir @@ -1782,7 +1821,7 @@ def run_server( log_config=log_config, keepalive_timeout=keepalive_timeout, timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None), - listener_fd=supervised.listener_fd if supervised is not None else None, + forwarded=forwarded, ) # Optional: recycle uvicorn workers after N requests if max_requests_before_restart is not None: @@ -1822,7 +1861,7 @@ def run_server( start_query_engine_reaper() if num_workers > 1 or reload: worker_overrides: Final = ProxyInitializationHelpers.child_process_uvicorn_overrides( - uvicorn_args.get("log_config") + uvicorn_args.get("log_config"), WORKER_APP if forwarded is None else FORWARD_APP ) if reload and num_workers > 1: from litellm.proxy.dev_reload import ( @@ -1848,16 +1887,18 @@ def run_server( workers=num_workers, ) elif run_gunicorn is True: + from litellm.proxy.rust_forward.peer import wrap + ProxyInitializationHelpers.run_gunicorn_server( host=host, port=port, - app=app, + app=wrap(app, forward_mode), num_workers=num_workers, ssl_certfile_path=ssl_certfile_path, ssl_keyfile_path=ssl_keyfile_path, max_requests_before_restart=max_requests_before_restart, max_requests_before_restart_jitter=max_requests_before_restart_jitter, - listener_fd=supervised.listener_fd if supervised is not None else None, + forwarded=forwarded, ) elif run_hypercorn is True: ProxyInitializationHelpers.init_hypercorn_server( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8a50c116125..c5fcbfb2564 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -835,7 +835,6 @@ from litellm.proxy.plugin_routes import ( from litellm.proxy.plugin_routes import ( router as plugin_router, ) -from litellm.proxy.rust_gateway import RustPeerMiddleware, Supervised, rust_gateway_mode from litellm.proxy.spend_tracking.spend_event_producer import ( CollectorSettings, SpendEventProducer, @@ -21925,10 +21924,6 @@ if _server_root_paths: ) app.add_middleware(PerRequestRootPathMiddleware, root_paths=_server_root_paths) -_rust_gateway_mode: Final = rust_gateway_mode(os.environ) -if isinstance(_rust_gateway_mode, Supervised): - app.add_middleware(RustPeerMiddleware, mode=_rust_gateway_mode) - async def _stream_mcp_asgi_response(handle_fn, scope: dict, receive) -> "StreamingResponse": """ diff --git a/litellm/proxy/rust_forward/AGENTS.md b/litellm/proxy/rust_forward/AGENTS.md new file mode 100644 index 00000000000..a2ab93cde5c --- /dev/null +++ b/litellm/proxy/rust_forward/AGENTS.md @@ -0,0 +1,8 @@ +# Rust forwarding, Python half + +With `LITELLM_RUST` on, the supervisor here runs the Rust gateway on the public port and Python on a private loopback listener. This package is the Python half of forwarding and pairs with the crate `litellm-rust/crates/gateway-forward`. Once Rust serves every route, both get deleted together, along with one dispatch branch in each entry point + +- Nothing outside this package imports Rust forwarding concepts, except the dispatch, the flag adapter and the app wrap in `litellm/proxy/proxy_cli.py` and `gateway/launch.py`. The proxy app itself never knows Rust is in front of it +- Python must see each request as if the client had connected directly. `x-litellm-rust-peer` is the only Rust-specific input, and `peer.py` honors it only from the loopback hop +- Python owns every setting Rust needs. The supervisor resolves the host, port, keepalive and shutdown drain Python actually runs with and hands them to Rust as the `LITELLM_FORWARD_*` variables in `rust_spec`. Rust never reads Python's own settings, so a new value Rust needs gets a new variable there +- Entry points turn their own flags into a `PublicListener`, apply `server_overrides` in the supervised child, and wrap the app with `peer.wrap` where they load it diff --git a/litellm/proxy/rust_forward/__init__.py b/litellm/proxy/rust_forward/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/proxy/rust_forward/asgi.py b/litellm/proxy/rust_forward/asgi.py new file mode 100644 index 00000000000..ff9bdc57cf8 --- /dev/null +++ b/litellm/proxy/rust_forward/asgi.py @@ -0,0 +1,10 @@ +import os + +from litellm.proxy.rust_forward.mode import rust_forward_mode +from litellm.proxy.rust_forward.peer import ASGIApp, wrap + + +def app() -> ASGIApp: + from litellm_cli.main import proxy_worker_app + + return wrap(proxy_worker_app(), rust_forward_mode(os.environ)) diff --git a/litellm/proxy/rust_forward/listener.py b/litellm/proxy/rust_forward/listener.py new file mode 100644 index 00000000000..28595a4099e --- /dev/null +++ b/litellm/proxy/rust_forward/listener.py @@ -0,0 +1,46 @@ +import os +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +from litellm.rust_bridge.configuration import ENV_NAME + + +@dataclass(frozen=True, slots=True) +class PublicListener: + host: str + port: int + keepalive_seconds: int + shutdown_drain_seconds: float + proxy_headers: bool + trusted_proxies: str | None + + +@dataclass(frozen=True, slots=True) +class StartupError: + reason: str + + +def startup_check(binary: str | None, unsupported: Mapping[str, object]) -> str | StartupError: + flag: Final = next((name for name, value in unsupported.items() if value), None) + if flag is not None: + return StartupError(f"{ENV_NAME} does not support {flag}") + if os.name != "posix": + return StartupError(f"{ENV_NAME} requires POSIX inherited sockets and process signals") + if binary is None: + return StartupError( + f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment" + ) + return binary + + +def uvicorn_default_keepalive() -> int: + import uvicorn + + return uvicorn.Config(app="", log_config=None).timeout_keep_alive + + +def shutdown_drain_seconds(graceful_timeout: float) -> float: + from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS + + return graceful_timeout + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS diff --git a/litellm/proxy/rust_forward/mode.py b/litellm/proxy/rust_forward/mode.py new file mode 100644 index 00000000000..a75287688ea --- /dev/null +++ b/litellm/proxy/rust_forward/mode.py @@ -0,0 +1,68 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final, TypeAlias + +from pydantic import TypeAdapter + +from litellm.rust_bridge.configuration import environment_switch + +GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" +METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" +PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" +TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" +DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1" +RUST_HOP: Final = "127.0.0.1" + +_FD: Final = TypeAdapter(int) +_ENABLED: Final = TypeAdapter(bool) + + +@dataclass(frozen=True, slots=True) +class Disabled: + pass + + +@dataclass(frozen=True, slots=True) +class Supervisor: + pass + + +@dataclass(frozen=True, slots=True) +class Supervised: + listener_fd: int + metrics_host: str + proxy_headers: bool + trusted_proxies: str + + +RustForwardMode: TypeAlias = Disabled | Supervisor | Supervised + + +def rust_forward_mode(environ: Mapping[str, str]) -> RustForwardMode: + fd: Final = environ.get(GATEWAY_FD) + if fd is not None: + return Supervised( + _FD.validate_python(fd), + environ[METRICS_HOST], + _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), + environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), + ) + if environment_switch(environ): + return Supervisor() + return Disabled() + + +@dataclass(frozen=True, slots=True) +class ServerOverrides: + fd: int + proxy_headers: bool + metrics_host: str + + +def server_overrides(mode: Supervised) -> ServerOverrides: + """The private listener the supervised server binds instead of its own flags. + + Proxy headers stay off at the server, since it would judge Rust's loopback address instead of the real peer. + ``peer.wrap`` applies the operator's trust settings to the restored peer. + """ + return ServerOverrides(fd=mode.listener_fd, proxy_headers=False, metrics_host=mode.metrics_host) diff --git a/litellm/proxy/rust_forward/peer.py b/litellm/proxy/rust_forward/peer.py new file mode 100644 index 00000000000..2d03d6ca35c --- /dev/null +++ b/litellm/proxy/rust_forward/peer.py @@ -0,0 +1,57 @@ +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Final, TypeAlias + +from typing_extensions import assert_never + +from litellm.proxy.rust_forward.mode import RUST_HOP, Disabled, RustForwardMode, Supervised, Supervisor + +if TYPE_CHECKING: + from uvicorn._types import ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope + +PEER_HEADER: Final = b"x-litellm-rust-peer" + +ASGIApp: TypeAlias = Callable[..., Awaitable[None]] + + +def _restore_peer(scope: "WWWScope") -> None: + headers: Final = tuple(scope["headers"]) + peer: Final = next((value for name, value in headers if name == PEER_HEADER), None) + if peer is None: + return + remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER) + scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop + client: Final = scope["client"] + if client is None or client[0] != RUST_HOP: + return + host, _, port = peer.decode("latin-1").rpartition(":") + scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope + + +class RustPeerMiddleware: + """Make Python see each request as if the client had connected to it directly. + + Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real + TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the + real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust + settings Python would have used on the public port. + """ + + def __init__(self, app: ASGIApp, mode: Supervised) -> None: + from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + + self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app + + async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + if scope["type"] != "lifespan": + _restore_peer(scope) + await self.app(scope, receive, send) + + +def wrap(app: ASGIApp, mode: RustForwardMode) -> ASGIApp: + match mode: + case Supervised(): + return RustPeerMiddleware(app, mode) + case Disabled() | Supervisor(): + return app + case _: + assert_never(mode) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_forward/supervisor.py similarity index 53% rename from litellm/proxy/rust_gateway.py rename to litellm/proxy/rust_forward/supervisor.py index 8be060e362c..6ab4b86dca3 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_forward/supervisor.py @@ -9,115 +9,22 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from enum import Enum, IntEnum from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeAlias +from typing import Final, TypeAlias from pydantic import TypeAdapter from typing_extensions import assert_never -from litellm.rust_bridge.configuration import ENV_NAME, environment_switch +from litellm.proxy.rust_forward.listener import PublicListener +from litellm.proxy.rust_forward.mode import GATEWAY_FD, METRICS_HOST, PROXY_HEADERS, RUST_HOP, TRUSTED_PROXIES -if TYPE_CHECKING: - from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope - -GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" -METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" -PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" -TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" -DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1" -RUST_HOP: Final = "127.0.0.1" -PEER_HEADER: Final = b"x-litellm-rust-peer" - -_FD: Final = TypeAdapter(int) -_ENABLED: Final = TypeAdapter(bool) _ADDRESS: Final = TypeAdapter(tuple[str, int]) -@dataclass(frozen=True, slots=True) -class Disabled: - pass - - -@dataclass(frozen=True, slots=True) -class Supervisor: - pass - - -@dataclass(frozen=True, slots=True) -class Supervised: - listener_fd: int - metrics_host: str - proxy_headers: bool - trusted_proxies: str - - -RustGatewayMode: TypeAlias = Disabled | Supervisor | Supervised - - -def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode: - fd: Final = environ.get(GATEWAY_FD) - if fd is not None: - return Supervised( - _FD.validate_python(fd), - environ[METRICS_HOST], - _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), - environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), - ) - if environment_switch(environ): - return Supervisor() - return Disabled() - - -def _restore_peer(scope: "WWWScope") -> None: - headers: Final = tuple(scope["headers"]) - peer: Final = next((value for name, value in headers if name == PEER_HEADER), None) - if peer is None: - return - remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER) - scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop - client: Final = scope["client"] - if client is None or client[0] != RUST_HOP: - return - host, _, port = peer.decode("latin-1").rpartition(":") - scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope - - -class RustPeerMiddleware: - """Make Python see each request as if the client had connected to it directly. - - Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real - TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the - real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust - settings Python would have used on the public port. The server's proxy-header handling is off in - this mode, since it would judge Rust's loopback address instead of the real peer. - """ - - def __init__(self, app: "ASGI3Application", mode: Supervised) -> None: - from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware - - self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app - - async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: - if scope["type"] != "lifespan": - _restore_peer(scope) - await self.app(scope, receive, send) - - def gateway_binary(scripts_dir: str) -> str | None: name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway" return shutil.which(name, path=scripts_dir) -def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: - flag: Final = next((name for name, value in unsupported.items() if value), None) - if flag is not None: - return f"{ENV_NAME} does not support {flag}" - if os.name != "posix": - return f"{ENV_NAME} requires POSIX inherited sockets and process signals" - if binary is None: - return f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment" - return None - - class Child(Enum): RUST = "rust" PYTHON = "python" @@ -209,7 +116,7 @@ class ChildSpec: def python_spec( - command: Sequence[str], listener_fd: int, upstream_port: int, public_host: str, environ: Mapping[str, str] + command: Sequence[str], listener_fd: int, upstream_port: int, listener: PublicListener, environ: Mapping[str, str] ) -> ChildSpec: return ChildSpec( argv=(*command, "--host", RUST_HOP, "--port", str(upstream_port)), @@ -217,22 +124,26 @@ def python_spec( { **environ, GATEWAY_FD: str(listener_fd), - METRICS_HOST: public_host, + METRICS_HOST: listener.host, + PROXY_HEADERS: str(listener.proxy_headers).lower(), + **({} if listener.trusted_proxies is None else {TRUSTED_PROXIES: listener.trusted_proxies}), } ), pass_fds=(listener_fd,), ) -def rust_spec(binary: str, upstream_port: int, host: str, port: int, environ: Mapping[str, str]) -> ChildSpec: +def rust_spec(binary: str, upstream_port: int, listener: PublicListener, environ: Mapping[str, str]) -> ChildSpec: return ChildSpec( argv=(binary,), env=MappingProxyType( { **environ, - "HOST": host, - "PORT": str(port), "LITELLM_FORWARD_UPSTREAM": f"http://{RUST_HOP}:{upstream_port}", + "LITELLM_FORWARD_HOST": listener.host, + "LITELLM_FORWARD_PORT": str(listener.port), + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE": str(listener.keepalive_seconds), + "LITELLM_FORWARD_SHUTDOWN_DRAIN": str(listener.shutdown_drain_seconds), } ), pass_fds=(), @@ -266,18 +177,20 @@ async def _drive( return await _drive(following, events, children) -async def _supervise(binary: str, command: Sequence[str], host: str, port: int, environ: Mapping[str, str]) -> ExitCode: +async def _supervise( + binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str] +) -> ExitCode: loop: Final = asyncio.get_running_loop() events: Final[asyncio.Queue[Event]] = asyncio.Queue() for signum in (signal.SIGTERM, signal.SIGINT): loop.add_signal_handler(signum, events.put_nowait, Stop(signum)) - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: - listener.bind((RUST_HOP, 0)) - listener.listen() - _, upstream_port = _ADDRESS.validate_python(listener.getsockname()) - python: Final = await _spawn(python_spec(command, listener.fileno(), upstream_port, host, environ)) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as private: + private.bind((RUST_HOP, 0)) + private.listen() + _, upstream_port = _ADDRESS.validate_python(private.getsockname()) + python: Final = await _spawn(python_spec(command, private.fileno(), upstream_port, listener, environ)) try: - rust: Final = await _spawn(rust_spec(binary, upstream_port, host, port, environ)) + rust: Final = await _spawn(rust_spec(binary, upstream_port, listener, environ)) except OSError: _terminate_group(python) await python.wait() @@ -294,9 +207,9 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int, await asyncio.gather(*waiters) -def supervise(binary: str, command: Sequence[str], host: str, port: int, environ: Mapping[str, str]) -> int: +def supervise(binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str]) -> int: try: - return asyncio.run(_supervise(binary, command, host, port, environ)) + return asyncio.run(_supervise(binary, command, listener, environ)) except OSError as error: sys.stderr.write(f"Rust gateway startup failed: {error}\n") return ExitCode.CHILD_FAILED diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_forward.py similarity index 100% rename from tests/integration/configuration/test_rust_gateway.py rename to tests/integration/configuration/test_rust_forward.py diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index 418e0358b64..36eaabfb029 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -12,14 +12,18 @@ from typing import Final, cast from unittest.mock import MagicMock, patch import pytest +import uvicorn from uvicorn.importer import import_from_string from uvicorn.main import main as uvicorn_main from gateway.launch import GATEWAY_APP, main, pool_database_url, public_listener, uvicorn_argv +from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings from litellm.proxy.proxy_server import app as proxy_app -from litellm.proxy.rust_gateway import Supervised +from litellm.proxy.rust_forward.mode import Supervised, server_overrides +from litellm.proxy.rust_forward.supervisor import rust_spec +from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager DB_ENV: Final = { "DATABASE_HOST": "db.internal", @@ -127,34 +131,59 @@ class TestUvicornArgv: @pytest.mark.parametrize( ("argv", "trust"), [ - ((), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true"}), - ( - ("--forwarded-allow-ips", "10.0.0.5"), - {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "10.0.0.5"}, - ), - (("--forwarded-allow-ips=*",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "*"}), - (("--no-proxy-headers",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}), + ((), (True, None)), + (("--forwarded-allow-ips", "10.0.0.5"), (True, "10.0.0.5")), + (("--forwarded-allow-ips=*",), (True, "*")), + (("--no-proxy-headers",), (False, None)), ], ) def test_forwarding_carries_operator_proxy_header_options_to_the_child( - self, argv: tuple[str, ...], trust: dict[str, str] + self, argv: tuple[str, ...], trust: tuple[bool, str | None], monkeypatch: pytest.MonkeyPatch ) -> None: - assert public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") == ( - "gateway", - "0.0.0.0", - 4000, - trust, - ) + monkeypatch.delenv("FORWARDED_ALLOW_IPS", raising=False) + binary, listener = public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") + assert (binary, listener.host, listener.port) == ("gateway", "0.0.0.0", 4000) + assert (listener.proxy_headers, listener.trusted_proxies) == trust + + @pytest.mark.parametrize( + ("argv", "environ"), + [(("--timeout-keep-alive", "2"), {}), ((), {"KEEPALIVE_TIMEOUT": "7"}), ((), {})], + ) + def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with( + self, argv: tuple[str, ...], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + for name, value in environ.items(): + monkeypatch.setenv(name, value) + _, listener = public_listener(argv, "gateway") + sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"] + assert sent == str(_uvicorn_params(uvicorn_argv(argv, os.environ, None))["timeout_keep_alive"]) + + def test_without_a_keepalive_rust_follows_uvicorns_own_default(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + _, listener = public_listener((), "gateway") + assert listener.keepalive_seconds == uvicorn.Config(app="", log_config=None).timeout_keep_alive + + @pytest.mark.parametrize("argv", [(), ("--timeout-graceful-shutdown", "12")]) + def test_rust_drains_for_pythons_graceful_timeout_plus_teardown( + self, argv: tuple[str, ...], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "7.5") + _, listener = public_listener(argv, "gateway") + flag: Final = _uvicorn_params(uvicorn_argv(argv, {}, None))["timeout_graceful_shutdown"] + graceful: Final = GracefulShutdownManager.get_timeout() if flag is None else float(str(flag)) + sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"] + assert float(sent) == graceful + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS def test_supervised_listener_uses_the_inherited_loopback_socket(self): - supervised: Final = Supervised(17, "0.0.0.0", True, "127.0.0.1") + supervised: Final = server_overrides(Supervised(17, "0.0.0.0", True, "127.0.0.1")) params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised)) assert params["fd"] == 17 assert params["workers"] == 2 @pytest.mark.parametrize("argv", [(), ("--proxy-headers",), ("--forwarded-allow-ips=*",)]) def test_private_hop_leaves_proxy_headers_to_the_app(self, argv: tuple[str, ...]): - params: Final = _uvicorn_params(uvicorn_argv(argv, {}, Supervised(17, "0.0.0.0", True, "*"))) + params: Final = _uvicorn_params(uvicorn_argv(argv, {}, server_overrides(Supervised(17, "0.0.0.0", True, "*")))) assert params["proxy_headers"] is False def test_direct_listener_keeps_explicit_proxy_header_options(self): diff --git a/tests/unit/proxy/rust_forward/__init__.py b/tests/unit/proxy/rust_forward/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/rust_forward/test_listener.py b/tests/unit/proxy/rust_forward/test_listener.py new file mode 100644 index 00000000000..d17b0e74e14 --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_listener.py @@ -0,0 +1,74 @@ +from collections.abc import Mapping +from typing import Final + +import pytest +import uvicorn +from click.testing import CliRunner +from gunicorn.config import Config as GunicornConfig + +from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS +from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server +from litellm.proxy.rust_forward.listener import PublicListener, StartupError, startup_check +from litellm.proxy.rust_forward.supervisor import rust_spec + + +@pytest.mark.parametrize( + "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] +) +def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "true") + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] + result: Final = CliRunner().invoke(run_server, [flag, *values]) + assert result.exit_code != 0 + assert f"LITELLM_RUST does not support {flag}" in result.output + + +def test_missing_binary_gives_install_instruction() -> None: + assert startup_check(None, {}) == StartupError( + "LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment" + ) + + +def _cli_listener(argv: list[str]) -> tuple[Mapping[str, object], PublicListener]: + params: Final = run_server.make_context("litellm", ["--port", "45999", *argv]).params + return params, ProxyInitializationHelpers.public_listener( + str(params["host"]), int(str(params["port"])), params["keepalive_timeout"], params["run_gunicorn"] is True + ) + + +def _uvicorn_keepalive(options: Mapping[str, object]) -> int: + configured: Final = {"timeout_keep_alive": options["timeout_keep_alive"]} if "timeout_keep_alive" in options else {} + return uvicorn.Config(app="", log_config=None, **configured).timeout_keep_alive + + +@pytest.mark.parametrize( + ("argv", "environ"), + [(["--keepalive_timeout", "2"], {}), ([], {"KEEPALIVE_TIMEOUT": "7"}), ([], {})], +) +def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with( + argv: list[str], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + for name, value in environ.items(): + monkeypatch.setenv(name, value) + params, listener = _cli_listener(argv) + uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 45555, keepalive_timeout=params["keepalive_timeout"] + ) + sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"] + assert sent == str(_uvicorn_keepalive(uvicorn_options)) + + +def test_gunicorn_workers_keep_gunicorns_keepalive_and_rust_follows(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + _, listener = _cli_listener(["--run_gunicorn", "--keepalive_timeout", "9"]) + assert listener.keepalive_seconds == GunicornConfig().keepalive + + +def test_rust_drains_for_pythons_graceful_timeout_plus_teardown(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "12.5") + _, listener = _cli_listener([]) + uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555) + sent: Final = float(rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"]) + assert sent == uvicorn_options["timeout_graceful_shutdown"] + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS diff --git a/tests/unit/proxy/rust_forward/test_mode.py b/tests/unit/proxy/rust_forward/test_mode.py new file mode 100644 index 00000000000..20005a21255 --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_mode.py @@ -0,0 +1,43 @@ +from typing import Final + +import pytest + +from litellm.proxy.proxy_cli import ProxyInitializationHelpers +from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides + + +@pytest.mark.parametrize( + ("environ", "expected"), + [ + ({}, Disabled()), + ({"LITELLM_RUST": "false"}, Disabled()), + ({"LITELLM_RUST": "off"}, Disabled()), + ({"LITELLM_RUST": "0"}, Disabled()), + ({"LITELLM_RUST": "true"}, Supervisor()), + ({"LITELLM_RUST": "1"}, Supervisor()), + ({"LITELLM_RUST": "yes"}, Supervisor()), + ({"LITELLM_RUST": "on"}, Supervisor()), + ({"LITELLM_RUST": " 1 "}, Supervisor()), + ({"LITELLM_RUST": "enabled"}, Disabled()), + ( + { + "LITELLM_RUST": "true", + "_LITELLM_GATEWAY_FD": "17", + "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", + "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", + }, + Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"), + ), + ], +) +def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None: + assert rust_forward_mode(environ) == expected + + +@pytest.mark.parametrize("mode", [None, Supervised(17, "0.0.0.0", True, "127.0.0.1")]) +def test_inherited_socket_reaches_uvicorn(mode: Supervised | None) -> None: + options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 45555, forwarded=None if mode is None else server_overrides(mode) + ) + assert options.get("fd") == (None if mode is None else mode.listener_fd) + assert options.get("proxy_headers", True) is (mode is None) diff --git a/tests/unit/proxy/rust_forward/test_peer.py b/tests/unit/proxy/rust_forward/test_peer.py new file mode 100644 index 00000000000..e96d9cf2b6a --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_peer.py @@ -0,0 +1,144 @@ +import asyncio +import gc +from typing import TYPE_CHECKING, Final + +import httpx +import pytest +from fastapi import Request +from fastapi.routing import APIRoute +from uvicorn.importer import import_from_string +from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + +if TYPE_CHECKING: + from uvicorn._types import ( + ASGIReceiveCallable, + ASGISendCallable, + ASGISendEvent, + HTTPDisconnectEvent, + HTTPScope, + Scope, + ) + +from litellm.proxy.proxy_server import app as proxy_app +from litellm.proxy.rust_forward.mode import Supervised +from litellm.proxy.rust_forward.peer import wrap + + +Headers = tuple[tuple[bytes, bytes], ...] + + +async def _receive() -> "HTTPDisconnectEvent": + return {"type": "http.disconnect"} + + +async def _send(message: "ASGISendEvent") -> None: + return None + + +async def _seen_by_app( + behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised +) -> tuple[tuple[str, int] | None, str, Headers]: + seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = ( + asyncio.get_running_loop().create_future() + ) + + async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + assert scope["type"] == "http" + seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"]))) + + direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe + app: Final = wrap(probe, mode) if behind_rust else direct + scope: Final[HTTPScope] = { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": "/", + "raw_path": b"/", + "query_string": b"", + "root_path": "", + "headers": headers, + "client": client, + "server": ("127.0.0.1", 4000), + } + await app(scope, _receive, _send) + return seen.result() + + +PEER: Final = ("203.0.113.7", 51234) +TRUSTED_PEER: Final = ("10.0.12.34", 40000) +FORWARDED: Final[Headers] = ( + (b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"), + (b"x-forwarded-proto", b"https"), +) + + +@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)]) +@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))]) +@pytest.mark.parametrize( + "mode", + [ + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + Supervised(17, "0.0.0.0", True, "10.0.0.0/8"), + Supervised(17, "0.0.0.0", True, "*"), + Supervised(17, "0.0.0.0", False, "10.0.0.0/8"), + ], +) +async def test_python_behind_rust_sees_what_a_direct_connection_would( + peer: tuple[str, int], headers: Headers, mode: Supervised +) -> None: + rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}" + behind_rust: Final = await _seen_by_app( + True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode + ) + direct: Final = await _seen_by_app(False, peer, headers, mode) + assert behind_rust == direct + + +async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None: + client, scheme, _ = await _seen_by_app( + True, + ("127.0.0.1", 60000), + ((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")), + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + ) + assert (client, scheme) == (("203.0.113.7", 51234), "http") + + +async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None: + client, _, headers = await _seen_by_app( + True, + ("192.0.2.9", 1000), + ((b"x-litellm-rust-peer", b"10.0.12.34:1"),), + Supervised(17, "0.0.0.0", False, "127.0.0.1"), + ) + assert (client, headers) == (("192.0.2.9", 1000), ()) + + +async def _peer_probe(request: Request) -> dict[str, object]: + assert request.client is not None + return {"client": [request.client.host, request.client.port], "peer": request.headers.get("x-litellm-rust-peer")} + + +@pytest.mark.parametrize("entry_point", ["litellm.proxy.rust_forward.asgi:app", "gateway.launch:worker_app"]) +async def test_every_entry_point_hands_routes_the_peer_rust_saw( + entry_point: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17") + monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", "0.0.0.0") + monkeypatch.setattr(proxy_app.router, "lifespan_context", proxy_app.router.lifespan_context) + monkeypatch.setattr( + proxy_app.router, "routes", [APIRoute("/__rust_forward_peer_probe", _peer_probe), *proxy_app.router.routes] + ) + try: + served: Final = import_from_string(entry_point)() + finally: + gc.unfreeze() + transport: Final = httpx.ASGITransport(app=served, client=("127.0.0.1", 60000)) + async with httpx.AsyncClient(transport=transport, base_url="http://proxy") as client: + response: Final = await client.get( + "/__rust_forward_peer_probe", headers={"x-litellm-rust-peer": "203.0.113.7:51234"} + ) + assert response.status_code == 200, response.text + assert response.json() == {"client": ["203.0.113.7", 51234], "peer": None} diff --git a/tests/unit/proxy/rust_forward/test_supervisor.py b/tests/unit/proxy/rust_forward/test_supervisor.py new file mode 100644 index 00000000000..224d97f349d --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_supervisor.py @@ -0,0 +1,139 @@ +import functools +import signal +import sys +from pathlib import Path +from typing import Final + +import pytest + +from litellm.proxy.proxy_cli import run_server +from litellm.proxy.rust_forward.listener import PublicListener +from litellm.proxy.rust_forward.mode import Supervised, rust_forward_mode +from litellm.proxy.rust_forward.supervisor import ( + Child, + Event, + ExitCode, + Exited, + Finished, + Running, + Signal, + State, + Stop, + cli_command, + gateway_binary, + python_spec, + rust_spec, + step, +) + +TERM: Final = signal.SIGTERM +INT: Final = signal.SIGINT + + +def _listener(proxy_headers: bool = True, trusted_proxies: str | None = None) -> PublicListener: + return PublicListener("0.0.0.0", 4000, 5, 40.0, proxy_headers, trusted_proxies) + + +@pytest.mark.parametrize( + ("environ", "listener", "expected"), + [ + ({}, _listener(), Supervised(17, "0.0.0.0", True, "127.0.0.1")), + ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(), Supervised(17, "0.0.0.0", True, "10.0.0.5")), + ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(True, "*"), Supervised(17, "0.0.0.0", True, "*")), + ({}, _listener(False), Supervised(17, "0.0.0.0", False, "127.0.0.1")), + ], +) +def test_python_child_keeps_the_trust_python_alone_would_use( + environ: dict[str, str], listener: PublicListener, expected: Supervised +) -> None: + spec: Final = python_spec(("python",), 17, 45555, listener, environ) + assert rust_forward_mode(spec.env) == expected + + +def test_python_child_listens_privately_and_keeps_options() -> None: + arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") + spec: Final = python_spec(arguments, 17, 45555, _listener(), {}) + params: Final = run_server.make_context("litellm", list(spec.argv)).params + assert params["host"] == "127.0.0.1" + assert params["port"] == 45555 + assert params["num_workers"] == 2 + assert params["reload"] is True + assert spec.pass_fds == (17,) + assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli") + + +def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None: + spec: Final = rust_spec( + "gateway", 45555, PublicListener("0.0.0.0", 4000, 2, 12.5, True, None), {"RUST_LOG": "info"} + ) + assert spec.argv == ("gateway",) + assert spec.pass_fds == () + assert dict(spec.env) == { + "RUST_LOG": "info", + "LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555", + "LITELLM_FORWARD_HOST": "0.0.0.0", + "LITELLM_FORWARD_PORT": "4000", + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE": "2", + "LITELLM_FORWARD_SHUTDOWN_DRAIN": "12.5", + } + + +def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + scripts: Final = tmp_path / "scripts" + elsewhere: Final = tmp_path / "elsewhere" + scripts.mkdir() + elsewhere.mkdir() + binary: Final = elsewhere / "litellm-gateway" + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o755) + monkeypatch.setenv("PATH", str(elsewhere)) + assert gateway_binary(str(scripts)) is None + assert gateway_binary(str(elsewhere)) == str(binary) + + +def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]: + def advance(acc: tuple[State, tuple[Signal, ...]], event: Event) -> tuple[State, tuple[Signal, ...]]: + state, sent = acc + following, action = step(state, event) + return following, sent if action is None else (*sent, action) + + return functools.reduce(advance, events, (Running(), ())) + + +RUST: Final = Child.RUST +PYTHON: Final = Child.PYTHON + + +@pytest.mark.parametrize( + ("events", "signals", "code"), + [ + ( + (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 0)), + (Signal(RUST, TERM), Signal(PYTHON, TERM)), + ExitCode.CLEAN, + ), + ((Exited(PYTHON, 0),), (), ExitCode.CHILD_FAILED), + ((Exited(RUST, 1),), (), ExitCode.CHILD_FAILED), + ((Stop(TERM), Exited(PYTHON, 0)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), + ((Stop(TERM), Exited(RUST, 1)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), + ( + (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 1)), + (Signal(RUST, TERM), Signal(PYTHON, TERM)), + ExitCode.CHILD_FAILED, + ), + ( + (Stop(INT), Stop(INT), Exited(RUST, 0), Exited(PYTHON, 0)), + (Signal(RUST, INT), Signal(PYTHON, INT)), + ExitCode.CLEAN, + ), + ( + (Exited(RUST, 1), Stop(TERM), Exited(PYTHON, 0)), + (), + ExitCode.CHILD_FAILED, + ), + ], +) +def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit( + events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode +) -> None: + assert _fold(events) == (Finished(code), signals) diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index db36f0d681f..0ce5cc58191 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -1550,7 +1550,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=30, timeout_worker_healthcheck=None, - listener_fd=None, + forwarded=None, ) mock_uvicorn_run.assert_called_once() @@ -1627,7 +1627,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=None, timeout_worker_healthcheck=15, - listener_fd=None, + forwarded=None, ) @patch("uvicorn.run") diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py deleted file mode 100644 index 1072148be9f..00000000000 --- a/tests/unit/proxy/test_rust_gateway.py +++ /dev/null @@ -1,293 +0,0 @@ -import asyncio -import functools -import signal -import sys -from pathlib import Path -from typing import TYPE_CHECKING, Final - -import pytest -from click.testing import CliRunner -from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware - -if TYPE_CHECKING: - from uvicorn._types import ( - ASGIReceiveCallable, - ASGISendCallable, - ASGISendEvent, - HTTPDisconnectEvent, - HTTPScope, - Scope, - ) - -from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server -from litellm.proxy.rust_gateway import ( - Child, - Disabled, - Event, - ExitCode, - Exited, - Finished, - Running, - RustPeerMiddleware, - Signal, - State, - Stop, - Supervised, - Supervisor, - cli_command, - gateway_binary, - python_spec, - rust_gateway_mode, - rust_spec, - startup_error, - step, -) - -TERM: Final = signal.SIGTERM -INT: Final = signal.SIGINT - - -@pytest.mark.parametrize( - "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] -) -def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("LITELLM_RUST", "true") - monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) - values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] - result: Final = CliRunner().invoke(run_server, [flag, *values]) - assert result.exit_code != 0 - assert f"LITELLM_RUST does not support {flag}" in result.output - - -@pytest.mark.parametrize( - ("environ", "expected"), - [ - ({}, Disabled()), - ({"LITELLM_RUST": "false"}, Disabled()), - ({"LITELLM_RUST": "off"}, Disabled()), - ({"LITELLM_RUST": "0"}, Disabled()), - ({"LITELLM_RUST": "true"}, Supervisor()), - ({"LITELLM_RUST": "1"}, Supervisor()), - ({"LITELLM_RUST": "yes"}, Supervisor()), - ({"LITELLM_RUST": "on"}, Supervisor()), - ({"LITELLM_RUST": " 1 "}, Supervisor()), - ({"LITELLM_RUST": "enabled"}, Disabled()), - ( - { - "LITELLM_RUST": "true", - "_LITELLM_GATEWAY_FD": "17", - "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", - "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", - }, - Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"), - ), - ], -) -def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None: - assert rust_gateway_mode(environ) == expected - - -@pytest.mark.parametrize( - ("environ", "expected"), - [ - ({}, Supervised(17, "0.0.0.0", True, "127.0.0.1")), - ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, Supervised(17, "0.0.0.0", True, "10.0.0.5")), - ({"FORWARDED_ALLOW_IPS": "*"}, Supervised(17, "0.0.0.0", True, "*")), - ({"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}, Supervised(17, "0.0.0.0", False, "127.0.0.1")), - ], -) -def test_python_child_keeps_the_trust_python_alone_would_use(environ: dict[str, str], expected: Supervised) -> None: - spec: Final = python_spec(("python",), 17, 45555, "0.0.0.0", environ) - assert rust_gateway_mode(spec.env) == expected - - -def test_python_child_listens_privately_and_keeps_options() -> None: - arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") - spec: Final = python_spec(arguments, 17, 45555, "0.0.0.0", {}) - params: Final = run_server.make_context("litellm", list(spec.argv)).params - assert params["host"] == "127.0.0.1" - assert params["port"] == 45555 - assert params["num_workers"] == 2 - assert params["reload"] is True - assert spec.pass_fds == (17,) - assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli") - - -def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None: - spec: Final = rust_spec("gateway", 45555, "0.0.0.0", 4000, {"RUST_LOG": "info"}) - assert spec.argv == ("gateway",) - assert spec.pass_fds == () - assert dict(spec.env) == { - "RUST_LOG": "info", - "HOST": "0.0.0.0", - "PORT": "4000", - "LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555", - } - - -def test_missing_binary_gives_install_instruction() -> None: - assert startup_error(None, {}) == ( - "LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment" - ) - - -def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - scripts: Final = tmp_path / "scripts" - elsewhere: Final = tmp_path / "elsewhere" - scripts.mkdir() - elsewhere.mkdir() - binary: Final = elsewhere / "litellm-gateway" - binary.write_text("#!/bin/sh\nexit 0\n") - binary.chmod(0o755) - monkeypatch.setenv("PATH", str(elsewhere)) - assert gateway_binary(str(scripts)) is None - assert gateway_binary(str(elsewhere)) == str(binary) - - -@pytest.mark.parametrize("listener_fd", [None, 17]) -def test_inherited_socket_reaches_uvicorn(listener_fd: int | None) -> None: - options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( - "127.0.0.1", 45555, listener_fd=listener_fd - ) - assert options.get("fd") == listener_fd - assert options.get("proxy_headers", True) is (listener_fd is None) - - -def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]: - def advance(acc: tuple[State, tuple[Signal, ...]], event: Event) -> tuple[State, tuple[Signal, ...]]: - state, sent = acc - following, action = step(state, event) - return following, sent if action is None else (*sent, action) - - return functools.reduce(advance, events, (Running(), ())) - - -RUST: Final = Child.RUST -PYTHON: Final = Child.PYTHON - - -@pytest.mark.parametrize( - ("events", "signals", "code"), - [ - ( - (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 0)), - (Signal(RUST, TERM), Signal(PYTHON, TERM)), - ExitCode.CLEAN, - ), - ((Exited(PYTHON, 0),), (), ExitCode.CHILD_FAILED), - ((Exited(RUST, 1),), (), ExitCode.CHILD_FAILED), - ((Stop(TERM), Exited(PYTHON, 0)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), - ((Stop(TERM), Exited(RUST, 1)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), - ( - (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 1)), - (Signal(RUST, TERM), Signal(PYTHON, TERM)), - ExitCode.CHILD_FAILED, - ), - ( - (Stop(INT), Stop(INT), Exited(RUST, 0), Exited(PYTHON, 0)), - (Signal(RUST, INT), Signal(PYTHON, INT)), - ExitCode.CLEAN, - ), - ( - (Exited(RUST, 1), Stop(TERM), Exited(PYTHON, 0)), - (), - ExitCode.CHILD_FAILED, - ), - ], -) -def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit( - events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode -) -> None: - assert _fold(events) == (Finished(code), signals) - - -Headers = tuple[tuple[bytes, bytes], ...] - - -async def _receive() -> "HTTPDisconnectEvent": - return {"type": "http.disconnect"} - - -async def _send(message: "ASGISendEvent") -> None: - return None - - -async def _seen_by_app( - behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised -) -> tuple[tuple[str, int] | None, str, Headers]: - seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = ( - asyncio.get_running_loop().create_future() - ) - - async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: - assert scope["type"] == "http" - seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"]))) - - direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe - app: Final = RustPeerMiddleware(probe, mode) if behind_rust else direct - scope: Final[HTTPScope] = { - "type": "http", - "asgi": {"version": "3.0", "spec_version": "2.3"}, - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": "/", - "raw_path": b"/", - "query_string": b"", - "root_path": "", - "headers": headers, - "client": client, - "server": ("127.0.0.1", 4000), - } - await app(scope, _receive, _send) - return seen.result() - - -PEER: Final = ("203.0.113.7", 51234) -TRUSTED_PEER: Final = ("10.0.12.34", 40000) -FORWARDED: Final[Headers] = ( - (b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"), - (b"x-forwarded-proto", b"https"), -) - - -@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)]) -@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))]) -@pytest.mark.parametrize( - "mode", - [ - Supervised(17, "0.0.0.0", True, "127.0.0.1"), - Supervised(17, "0.0.0.0", True, "10.0.0.0/8"), - Supervised(17, "0.0.0.0", True, "*"), - Supervised(17, "0.0.0.0", False, "10.0.0.0/8"), - ], -) -async def test_python_behind_rust_sees_what_a_direct_connection_would( - peer: tuple[str, int], headers: Headers, mode: Supervised -) -> None: - rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}" - behind_rust: Final = await _seen_by_app( - True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode - ) - direct: Final = await _seen_by_app(False, peer, headers, mode) - assert behind_rust == direct - - -async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None: - client, scheme, _ = await _seen_by_app( - True, - ("127.0.0.1", 60000), - ((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")), - Supervised(17, "0.0.0.0", True, "127.0.0.1"), - ) - assert (client, scheme) == (("203.0.113.7", 51234), "http") - - -async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None: - client, _, headers = await _seen_by_app( - True, - ("192.0.2.9", 1000), - ((b"x-litellm-rust-peer", b"10.0.12.34:1"),), - Supervised(17, "0.0.0.0", False, "127.0.0.1"), - ) - assert (client, headers) == (("192.0.2.9", 1000), ()) From 4f0519274d04797dfa925498b62d037e9f926f13 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 13:39:56 +0900 Subject: [PATCH 15/17] fix(proxy): honor a factory app in the multi-worker reload options The supervised Python child serves rust_forward.asgi:app as a factory, so uvicorn's options carry factory=True. pool_config passed it to uvicorn.Config twice and `--reload --num_workers 2` crashed at startup under LITELLM_RUST Co-Authored-By: Claude Opus 5.5 --- litellm/proxy/dev_reload.py | 7 +++-- tests/unit/proxy/test_dev_reload.py | 41 +++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/dev_reload.py b/litellm/proxy/dev_reload.py index 500f581b34d..7b057271cb6 100644 --- a/litellm/proxy/dev_reload.py +++ b/litellm/proxy/dev_reload.py @@ -13,7 +13,7 @@ from socket import socket from typing import Final, Literal from pydantic import ConfigDict, ImportString, JsonValue, TypeAdapter, with_config -from typing_extensions import ReadOnly, Required, TypedDict +from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from uvicorn import Config, Server from uvicorn import config as uvicorn_config from uvicorn.supervisors import ChangeReload, Multiprocess @@ -42,6 +42,7 @@ class ServerOptions(TypedDict, total=False): @with_config(ConfigDict(extra="allow")) class ReloadOptions(ServerOptions): app: Required[ReadOnly[str]] + factory: NotRequired[ReadOnly[bool]] reload: ReadOnly[bool] reload_dirs: ReadOnly[tuple[str, ...]] reload_includes: ReadOnly[tuple[str, ...]] @@ -92,7 +93,9 @@ def load_worker_app(import_path: str, worker_app_factory: bool) -> Callable[..., def pool_config(options: ReloadOptions, workers: int, worker_overrides: Mapping[str, object] | None = None) -> Config: overrides: Final = worker_overrides or {} worker_app: Final = TypeAdapter(str).validate_python(overrides.get("app", options["app"])) - worker_app_factory: Final = TypeAdapter(bool).validate_python(overrides.get("factory", False)) + worker_app_factory: Final = TypeAdapter(bool).validate_python( + overrides.get("factory", options.get("factory", False)) + ) return Config( partial(load_worker_app, worker_app, worker_app_factory), factory=True, diff --git a/tests/unit/proxy/test_dev_reload.py b/tests/unit/proxy/test_dev_reload.py index 60a7c53820b..6650c1cd3e3 100644 --- a/tests/unit/proxy/test_dev_reload.py +++ b/tests/unit/proxy/test_dev_reload.py @@ -86,6 +86,39 @@ def test_a_broken_factory_exits_the_worker_as_a_startup_failure( assert exit_info.value.code == uvicorn_config.STARTUP_FAILURE +@pytest.mark.parametrize("worker_overrides", [None, {"factory": True}]) +def test_a_factory_flag_in_the_options_reaches_the_workers( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, worker_overrides: dict[str, object] | None +) -> None: + (tmp_path / "reload_factory_flag_app.py").write_text( + "async def served(scope, receive, send):\n" + " await send({'type': 'http.response.start', 'status': 204, 'headers': []})\n" + " await send({'type': 'http.response.body', 'body': b'', 'more_body': False})\n" + "\n" + "def factory():\n" + " return served\n" + ) + monkeypatch.syspath_prepend(str(tmp_path)) + config: Final = pool_config( + RELOAD_OPTIONS.validate_python({**_options("reload_factory_flag_app:factory"), "factory": True}), + workers=POOL_WORKERS, + worker_overrides=worker_overrides, + ) + config.load() + loaded_app: Final = config.loaded_app + assert loaded_app is not None + sent: Final[list[Message]] = [] + + async def receive() -> Message: + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(message: Message) -> None: + sent.append(message) + + asyncio.run(loaded_app({"type": "http", "method": "GET", "path": "/", "headers": []}, receive, send)) + assert sent[0] == {"type": "http.response.start", "status": 204, "headers": []} + + @REQUIRES_UVICORN_STARTUP_FAILURE @pytest.mark.parametrize( ("module", "source"), @@ -118,7 +151,9 @@ def test_the_reload_pool_defaults_the_worker_healthcheck_and_keeps_an_explicit_o explicit: Final = RELOAD_WORKER_HEALTHCHECK_SECONDS + 7 assert pool_config(_options("app:app"), workers=2).timeout_worker_healthcheck == RELOAD_WORKER_HEALTHCHECK_SECONDS assert ( - pool_config({**_options("app:app"), "timeout_worker_healthcheck": explicit}, workers=2).timeout_worker_healthcheck + pool_config( + {**_options("app:app"), "timeout_worker_healthcheck": explicit}, workers=2 + ).timeout_worker_healthcheck == explicit ) @@ -142,7 +177,9 @@ def test_the_reload_pool_accepts_and_keeps_every_uvicorn_arg_the_proxy_cli_build def test_the_reload_pool_passes_a_uvicorn_arg_it_does_not_list_through_to_the_workers() -> None: - config: Final = pool_config(RELOAD_OPTIONS.validate_python({**_options("app:app"), "backlog": 123}), workers=POOL_WORKERS) + config: Final = pool_config( + RELOAD_OPTIONS.validate_python({**_options("app:app"), "backlog": 123}), workers=POOL_WORKERS + ) assert (config.backlog, config.reload) == (123, False) From f4cd7133917a59a9690d76240cfbc6ec99bc2a9e Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 15:12:30 +0900 Subject: [PATCH 16/17] test(proxy): cover each traffic shape through the Rust forwarding gateway Co-Authored-By: Claude Opus 5.5 --- .../test_rust_forward_traffic.py | 457 ++++++++++++++++++ 1 file changed, 457 insertions(+) create mode 100644 tests/integration/configuration/test_rust_forward_traffic.py diff --git a/tests/integration/configuration/test_rust_forward_traffic.py b/tests/integration/configuration/test_rust_forward_traffic.py new file mode 100644 index 00000000000..eeeeea366d8 --- /dev/null +++ b/tests/integration/configuration/test_rust_forward_traffic.py @@ -0,0 +1,457 @@ +import asyncio +import json +import os +import socket +import subprocess +import sys +import threading +import time +from collections.abc import AsyncIterator, Callable, Iterator, Mapping +from contextlib import AbstractContextManager, ExitStack +from dataclasses import dataclass +from pathlib import Path +from queue import SimpleQueue +from types import MappingProxyType +from typing import Final + +import anyio +import httpx +import openai +import pytest +from pydantic import BaseModel, JsonValue +from starlette.applications import Starlette +from starlette.requests import Request as StarletteRequest +from starlette.responses import StreamingResponse +from starlette.routing import Route, WebSocketRoute +from starlette.websockets import WebSocket, WebSocketDisconnect +from websockets.sync.client import connect + +from tests.integration._support.asgi import asgi_server +from tests.integration._support.client import eventually +from tests.integration._support.mcp import mcp_peer +from tests.integration._support.ports import allocate_port +from tests.integration._support.process import stop_root_process +from tests.integration._support.wire import Reply, Request, wire_server + +MASTER_KEY: Final = "sk-rust-forward-traffic-key-not-a-secret" +KEEPALIVE_SECONDS: Final = 1 +FILES_MODEL: Final = "rust-forward-files" +FILE_ID: Final = "file-rust-forward" +CHAT_MODEL: Final = "rust-forward-chat" +REALTIME_MODEL: Final = "rust-forward-realtime" +MCP_HEADERS: Final = MappingProxyType( + { + "Authorization": f"Bearer {MASTER_KEY}", + "Accept": "application/json, text/event-stream", + "mcp-protocol-version": "2025-06-18", + } +) + + +@dataclass(frozen=True, slots=True) +class ScriptedProvider: + url: str + released: threading.Event + hung_up: SimpleQueue[str] + + +@dataclass(frozen=True, slots=True) +class RustProxy: + url: str + port: int + log: Path + provider: ScriptedProvider + + +class _Message(BaseModel): + content: str + + +class _ChatRequest(BaseModel): + messages: tuple[_Message, ...] + + +class _InputText(BaseModel): + text: str + + +class _Item(BaseModel): + content: tuple[_InputText, ...] = () + + +class _RealtimeFrame(BaseModel): + type: str + item: _Item | None = None + + +def _sse(content: str | None, finish_reason: str | None = None) -> bytes: + chunk: Final = { + "id": "chatcmpl-rust-forward", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "delta": {} if content is None else {"content": content}, "finish_reason": finish_reason} + ], + } + return b"data: " + json.dumps(chunk).encode() + b"\n\n" + + +async def _gated_stream(released: threading.Event) -> AsyncIterator[bytes]: + yield _sse("before-gate") + await anyio.to_thread.run_sync(released.wait, 30) + yield _sse("after-gate") + yield _sse(None, "stop") + yield b"data: [DONE]\n\n" + + +async def _endless_stream(marker: str, hung_up: SimpleQueue[str]) -> AsyncIterator[bytes]: + try: + for index in range(600): + yield _sse(f"tick-{index} ") + await asyncio.sleep(0.1) + except (asyncio.CancelledError, GeneratorExit): + hung_up.put(marker) + raise + + +def _provider_app(released: threading.Event, hung_up: SimpleQueue[str]) -> Starlette: + async def chat(request: StarletteRequest) -> StreamingResponse: + prompt: Final = _ChatRequest.model_validate_json(await request.body()).messages[-1].content + return StreamingResponse( + _gated_stream(released) if prompt == "gated" else _endless_stream(prompt, hung_up), + media_type="text/event-stream", + ) + + async def realtime(websocket: WebSocket) -> None: + await websocket.accept() + await websocket.send_json({"type": "session.created", "session": {"id": "sess_rust_forward"}}) + try: + while True: + frame = _RealtimeFrame.model_validate_json(await websocket.receive_text()) # rebind-ok: per frame + if frame.item is not None: + text = "".join(part.text for part in frame.item.content) # rebind-ok: per frame + await websocket.send_json({"type": "response.output_text.delta", "delta": f"echo:{text}"}) + except WebSocketDisconnect: + return + + return Starlette( + routes=[ + Route("/v1/chat/completions", chat, methods=["POST"]), + WebSocketRoute("/v1/realtime", realtime), + ] + ) + + +def _routable_address() -> str: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe: + probe.connect(("192.0.2.1", 9)) + address: Final[str] = probe.getsockname()[0] + assert not address.startswith("127."), address + return address + + +def _ready(client: httpx.Client) -> bool: + try: + return client.get("/health/liveliness").status_code == 200 + except httpx.TransportError: + return False + + +def _files_peer(content: bytes) -> Callable[[Request], Reply]: + file_object: Final = json.dumps( + { + "id": FILE_ID, + "object": "file", + "purpose": "batch", + "bytes": len(content), + "created_at": 1, + "filename": "upload.jsonl", + "status": "processed", + } + ).encode() + + def respond(request: Request) -> Reply: + if request.method == "POST" and request.target.startswith("/v1/files"): + return Reply(body=file_object) + if request.method == "GET" and request.target.startswith(f"/v1/files/{FILE_ID}/content"): + return Reply( + content_type="application/octet-stream", body=content, chunks=(content[:65536], content[65536:]) + ) + return Reply(status=404, body=b'{"error": {"message": "not found"}}') + + return respond + + +UPLOAD: Final = b"".join( + json.dumps( + { + "custom_id": f"row-{index}", + "method": "POST", + "url": "/v1/chat/completions", + "body": {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": os.urandom(48).hex()}]}, + } + ).encode() + + b"\n" + for index in range(30_000) +) + + +@pytest.fixture(scope="module") +def files_upstream() -> Iterator[str]: + with wire_server(_files_peer(UPLOAD), keep_alive=True) as wire: + yield wire.url + + +@pytest.fixture(scope="module") +def rust_proxy(tmp_path_factory: pytest.TempPathFactory, files_upstream: str) -> Iterator[RustProxy]: + directory: Final = tmp_path_factory.mktemp("rust-forward-traffic") + log: Final = directory / "proxy.log" + port: Final = allocate_port() + released: Final = threading.Event() + hung_up: Final[SimpleQueue[str]] = SimpleQueue() + with ExitStack() as stack: + mcp: Final = stack.enter_context(mcp_peer()) + provider: Final = ScriptedProvider( + stack.enter_context(asgi_server(_provider_app(released, hung_up))), released, hung_up + ) + config: Final = directory / "config.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": CHAT_MODEL, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{provider.url}/v1", + "api_key": "integration-provider-key", + }, + }, + { + "model_name": REALTIME_MODEL, + "litellm_params": { + "model": "openai/gpt-realtime", + "api_base": provider.url, + "api_key": "integration-provider-key", + }, + "model_info": {"mode": "realtime"}, + }, + { + "model_name": FILES_MODEL, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{files_upstream}/v1", + "api_key": "integration-provider-key", + }, + }, + ], + "mcp_servers": {"math": {**mcp.registration(), "available_on_public_internet": False}}, + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "mcp_internal_ip_ranges": ["127.0.0.0/8"], + }, + } + ) + ) + environment: Final = { + **{ + key: value + for key, value in os.environ.items() + if not key.startswith("DATABASE_") and key != "DIRECT_URL" + }, + "LITELLM_MASTER_KEY": MASTER_KEY, + "LITELLM_RUST": "true", + "STORE_MODEL_IN_DB": "False", + "PYTHON_DOTENV_DISABLED": "1", + } + output: Final = stack.enter_context(log.open("wb")) + supervisor: Final = stack.enter_context( + subprocess.Popen( + ( + sys.executable, + "-I", + "-m", + "litellm.proxy.proxy_cli", + "--config", + str(config), + "--host", + "0.0.0.0", + "--port", + str(port), + "--num_workers", + "2", + "--keepalive_timeout", + str(KEEPALIVE_SECONDS), + "--timeout_worker_healthcheck", + "120", + ), + env=environment, + stdout=output, + stderr=output, + ) + ) + try: + with httpx.Client(base_url=f"http://127.0.0.1:{port}", trust_env=False, timeout=5) as probe: + assert eventually(lambda: _ready(probe), bool, seconds=180), log.read_text() + yield RustProxy(f"http://127.0.0.1:{port}", port, log, provider) + finally: + released.set() + assert stop_root_process(supervisor), log.read_text() + assert supervisor.returncode == 0, log.read_text() + + +def _still_open(stream: httpx.Response) -> bool: + try: + next(stream.iter_raw()) + except httpx.ReadTimeout: + return True + except (StopIteration, httpx.TransportError): + return False + return True + + +def _initialize(client: httpx.Client, headers: Mapping[str, str]) -> httpx.Response: + return client.post( + "/mcp", + headers={**MCP_HEADERS, **headers}, + json={ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "clientInfo": {"name": "rust-forward", "version": "1"}, + }, + }, + ) + + +def _mcp_post(client: httpx.Client, session: str, body: Mapping[str, JsonValue]) -> httpx.Response: + return client.post("/mcp", headers={**MCP_HEADERS, "mcp-session-id": session}, json=body) + + +def test_mcp_session_keeps_working_while_its_get_stream_sits_idle(rust_proxy: RustProxy) -> None: + idle_seconds: Final = 8 * KEEPALIVE_SECONDS + with httpx.Client(base_url=rust_proxy.url, trust_env=False, timeout=30) as client: + initialized: Final = _initialize(client, {}) + assert initialized.status_code == 200, initialized.text + session: Final = initialized.headers["mcp-session-id"] + assert _mcp_post(client, session, {"jsonrpc": "2.0", "method": "notifications/initialized"}).status_code == 202 + with httpx.Client(base_url=rust_proxy.url, trust_env=False, timeout=httpx.Timeout(5, read=2)) as listener: + with listener.stream( + "GET", "/mcp", headers={**MCP_HEADERS, "Accept": "text/event-stream", "mcp-session-id": session} + ) as notifications: + assert notifications.status_code == 200 + assert notifications.headers["content-type"].startswith("text/event-stream") + time.sleep(idle_seconds) + called: Final = _mcp_post( + client, + session, + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "math-add", "arguments": {"a": 2, "b": 3}}, + }, + ) + assert called.status_code == 200, called.text + assert '"text":"5"' in called.text.replace(" ", ""), called.text + assert _still_open(notifications) + + +def test_uploaded_file_and_its_download_cross_byte_for_byte(rust_proxy: RustProxy) -> None: + with openai.OpenAI(api_key=MASTER_KEY, base_url=rust_proxy.url, max_retries=0, timeout=60) as client: + created: Final = client.files.create( + file=("upload.jsonl", UPLOAD), purpose="batch", extra_body={"model": FILES_MODEL} + ) + downloaded: Final = client.files.content(created.id, extra_query={"model": FILES_MODEL}).content + assert len(UPLOAD) > 4 * 1024 * 1024 + assert downloaded == UPLOAD, (len(downloaded), len(UPLOAD)) + + +def test_internal_only_mcp_server_judges_the_connecting_peer_not_its_forwarded_for_claim( + rust_proxy: RustProxy, +) -> None: + outsider: Final = _routable_address() + with httpx.Client( + base_url=f"http://{outsider}:{rust_proxy.port}", + transport=httpx.HTTPTransport(local_address=outsider), + trust_env=False, + timeout=10, + ) as spoofing: + spoofed: Final = _initialize(spoofing, {"X-Forwarded-For": "127.0.0.1"}) + with httpx.Client(base_url=rust_proxy.url, trust_env=False, timeout=10) as internal: + admitted: Final = _initialize(internal, {}) + assert admitted.status_code == 200, admitted.text + assert spoofed.status_code == 403, spoofed.text + assert "client IP" in spoofed.text, spoofed.text + + +def test_oauth_metadata_names_the_origin_the_client_used(rust_proxy: RustProxy) -> None: + with httpx.Client(base_url=rust_proxy.url, trust_env=False, timeout=10) as client: + direct: Final = client.get("/.well-known/oauth-protected-resource") + balanced: Final = client.get( + "/.well-known/oauth-protected-resource", + headers={"Host": "gateway.example.test", "X-Forwarded-Proto": "https"}, + ) + assert direct.status_code == 200, direct.text + assert direct.json()["resource"] == rust_proxy.url + assert direct.json()["authorization_servers"] == [f"{rust_proxy.url}/mcp"] + assert balanced.status_code == 200, balanced.text + assert balanced.json()["resource"] == "https://gateway.example.test" + + +def _chat_stream(client: httpx.Client, prompt: str) -> AbstractContextManager[httpx.Response]: + return client.stream( + "POST", + "/v1/chat/completions", + headers={"Authorization": f"Bearer {MASTER_KEY}"}, + json={"model": CHAT_MODEL, "stream": True, "messages": [{"role": "user", "content": prompt}]}, + ) + + +def _content(lines: Iterator[str]) -> str: + return next(line for line in lines if '"content"' in line) + + +def test_stream_chunk_reaches_the_client_before_the_provider_sends_the_next(rust_proxy: RustProxy) -> None: + with httpx.Client(base_url=rust_proxy.url, trust_env=False, timeout=httpx.Timeout(10, read=5)) as client: + with _chat_stream(client, "gated") as response: + assert response.status_code == 200, response.read() + lines: Final = response.iter_lines() + first: Final = _content(lines) + rust_proxy.provider.released.set() + rest: Final = tuple(line for line in lines if line) + assert "before-gate" in first, first + assert any("after-gate" in line for line in rest), rest + assert rest[-1] == "data: [DONE]", rest + + +def test_client_hang_up_mid_stream_closes_the_provider_stream(rust_proxy: RustProxy) -> None: + marker: Final = f"endless-{time.monotonic_ns()}" + with httpx.Client(base_url=rust_proxy.url, trust_env=False, timeout=httpx.Timeout(10, read=5)) as client: + with _chat_stream(client, marker) as response: + assert response.status_code == 200, response.read() + assert "tick-0" in _content(response.iter_lines()) + assert rust_proxy.provider.hung_up.get(timeout=15) == marker + + +def test_realtime_websocket_frames_cross_both_ways(rust_proxy: RustProxy) -> None: + marker: Final = f"realtime-{time.monotonic_ns()}" + with connect( + f"ws://127.0.0.1:{rust_proxy.port}/v1/realtime?model={REALTIME_MODEL}", + additional_headers={"Authorization": f"Bearer {MASTER_KEY}"}, + open_timeout=10, + ) as session: + session.send( + json.dumps( + { + "type": "conversation.item.create", + "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": marker}]}, + } + ) + ) + replies: Final = tuple(session.recv(timeout=10) for _ in range(2)) + assert any(f"echo:{marker}" in reply for reply in replies), replies From 55f221297d8bd06529ba1d58e000a9ed200d923b Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 15:52:00 +0900 Subject: [PATCH 17/17] refactor(proxy): keep every Rust forwarding decision inside rust_forward Entry points now describe what they would serve with as a PythonServer and hand it to supervisor.supervise, which owns the startup check, the child environments and how a refusal is reported. Supervised children merge identity-when-disabled options from mode, and the app wrap moves into the worker factories, so asgi.py and the metrics host variable go away Co-Authored-By: Claude Opus 5.5 --- gateway/launch.py | 75 +- litellm/proxy/proxy_cli.py | 126 ++-- litellm/proxy/rust_forward/AGENTS.md | 12 +- litellm/proxy/rust_forward/asgi.py | 10 - litellm/proxy/rust_forward/listener.py | 46 -- litellm/proxy/rust_forward/mode.py | 65 +- litellm/proxy/rust_forward/server.py | 30 + litellm/proxy/rust_forward/supervisor.py | 86 ++- litellm_cli/main.py | 8 +- tests/unit/gateway/test_launch.py | 72 +- .../unit/proxy/rust_forward/test_listener.py | 74 -- tests/unit/proxy/rust_forward/test_mode.py | 75 +- tests/unit/proxy/rust_forward/test_peer.py | 15 +- .../proxy/rust_forward/test_supervisor.py | 83 ++- tests/unit/proxy/test_proxy_cli.py | 668 ++++++------------ 15 files changed, 638 insertions(+), 807 deletions(-) delete mode 100644 litellm/proxy/rust_forward/asgi.py delete mode 100644 litellm/proxy/rust_forward/listener.py create mode 100644 litellm/proxy/rust_forward/server.py delete mode 100644 tests/unit/proxy/rust_forward/test_listener.py diff --git a/gateway/launch.py b/gateway/launch.py index 89341a4cb89..24f1ab2a388 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -19,7 +19,6 @@ Run with: import os import sys -import sysconfig from collections.abc import Callable, Mapping, Sequence from typing import TYPE_CHECKING, Final @@ -31,31 +30,19 @@ from litellm_cli.main import without_gc_pauses if TYPE_CHECKING: from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings - from litellm.proxy.rust_forward.listener import PublicListener - from litellm.proxy.rust_forward.mode import ServerOverrides from litellm.proxy.rust_forward.peer import ASGIApp + from litellm.proxy.rust_forward.server import PythonServer GATEWAY_APP: Final = "gateway.launch:worker_app" KEEPALIVE_FLAG: Final = "--timeout-keep-alive" -def uvicorn_argv( - argv: Sequence[str], environ: Mapping[str, str], forwarded: "ServerOverrides | None" -) -> tuple[str, ...]: +def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str, ...]: """Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly.""" keepalive: Final = environ.get("KEEPALIVE_TIMEOUT") - inherited_socket: Final = ( - () - if forwarded is None - else ( - "--fd", - str(forwarded.fd), - "--proxy-headers" if forwarded.proxy_headers else "--no-proxy-headers", - ) - ) if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv): - return (GATEWAY_APP, "--factory", *argv, *inherited_socket) - return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive, *inherited_socket) + return (GATEWAY_APP, "--factory", *argv) + return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive) def pool_database_url( @@ -91,36 +78,40 @@ def _serve(argv: Sequence[str]) -> None: uvicorn_main(tuple(argv), prog_name="uvicorn") -def public_listener(argv: Sequence[str], binary: str | None) -> "tuple[str, PublicListener]": - """Validate the uvicorn flags for Rust forwarding and return the binary and the public listener.""" +def python_server(argv: Sequence[str]) -> "PythonServer": + """What uvicorn would serve the public port with, read from the same flags and environment it reads.""" from pydantic import TypeAdapter - from litellm.proxy.rust_forward.listener import PublicListener, StartupError, shutdown_drain_seconds, startup_check + from litellm.proxy.rust_forward.server import Listener, PythonServer, Server from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager params: Final[Mapping[str, object]] = uvicorn_main.make_context( - "uvicorn", list(uvicorn_argv(argv, os.environ, None)) + "uvicorn", list(uvicorn_argv(argv, os.environ)) ).params - checked: Final = startup_check( - binary, - { - "--ssl-keyfile": params["ssl_keyfile"], - "--ssl-certfile": params["ssl_certfile"], - "--ssl-ciphers": any(arg.split("=")[0] == "--ssl-ciphers" for arg in argv), - "--uds": params["uds"] is not None, - "--fd": params["fd"] is not None, - }, + tls: Final = ( + params["ssl_keyfile"] is not None + or params["ssl_certfile"] is not None + or any(arg.split("=")[0] == "--ssl-ciphers" for arg in argv) + ) + listener: Final = ( + Listener.TLS + if tls + else Listener.UNIX_SOCKET + if params["uds"] is not None + else Listener.INHERITED_FD + if params["fd"] is not None + else Listener.TCP ) - if isinstance(checked, StartupError): - sys.exit(checked.reason) graceful_timeout: Final = TypeAdapter[int | None](int | None).validate_python(params["timeout_graceful_shutdown"]) - return checked, PublicListener( + return PythonServer( + server=Server.UVICORN, + listener=listener, host=TypeAdapter(str).validate_python(params["host"]), port=TypeAdapter(int).validate_python(params["port"]), keepalive_seconds=TypeAdapter(int).validate_python(params["timeout_keep_alive"]), - shutdown_drain_seconds=shutdown_drain_seconds( - GracefulShutdownManager.get_timeout() if graceful_timeout is None else graceful_timeout - ), + graceful_timeout_seconds=GracefulShutdownManager.get_timeout() + if graceful_timeout is None + else graceful_timeout, proxy_headers=TypeAdapter(bool).validate_python(params["proxy_headers"]), trusted_proxies=TypeAdapter[str | None](str | None).validate_python(params["forwarded_allow_ips"]), ) @@ -140,21 +131,19 @@ def configure_database() -> None: def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: - from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides - from litellm.proxy.rust_forward.supervisor import gateway_binary, supervise + from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, uvicorn_flags + from litellm.proxy.rust_forward.supervisor import supervise mode: Final = rust_forward_mode(os.environ) match mode: case Disabled(): configure_database() - serve(uvicorn_argv(argv, os.environ, None)) + serve(uvicorn_argv(argv, os.environ)) case Supervisor(): - binary, listener = public_listener(argv, gateway_binary(sysconfig.get_path("scripts"))) - configure_database() command: Final = (sys.executable, "-m", "gateway.launch", *argv) - sys.exit(supervise(binary, command, listener, os.environ)) + sys.exit(supervise(python_server(argv), command, os.environ, before_start=configure_database)) case Supervised(): - serve(uvicorn_argv(argv, os.environ, server_overrides(mode))) + serve((*uvicorn_argv(argv, os.environ), *uvicorn_flags(mode))) case _: assert_never(mode) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index cb8e936f3c5..8d2acfc4694 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -8,7 +8,6 @@ import random import re import subprocess import sys -import sysconfig import urllib.parse as urlparse from collections.abc import Callable, Iterable, Mapping, Sequence from pathlib import Path @@ -37,9 +36,8 @@ from litellm.types.llms.base import LiteLLMBaseModel if TYPE_CHECKING: from fastapi import FastAPI - from litellm.proxy.rust_forward.listener import PublicListener - from litellm.proxy.rust_forward.mode import ServerOverrides from litellm.proxy.rust_forward.peer import ASGIApp + from litellm.proxy.rust_forward.server import PythonServer else: FastAPI = Any @@ -266,7 +264,6 @@ def _apply_config_logging_settings(litellm_settings: StartupOutputSettings | Non WORKER_APP: Final = "litellm_cli.main:proxy_worker_app" -FORWARD_APP: Final = "litellm.proxy.rust_forward.asgi:app" WORKER_LOG_CONFIG_ENV: Final = "LITELLM_WORKER_LOG_CONFIG" _LOG_CONFIG: Final = TypeAdapter[str | Mapping[str, object]](str | Mapping[str, object]) _LOG_CONFIG_FILE: Final = TypeAdapter[Mapping[str, object]](Mapping[str, object]) @@ -395,12 +392,12 @@ class ProxyInitializationHelpers: logging.config.fileConfig(log_config, disable_existing_loggers=False) @staticmethod - def child_process_uvicorn_overrides(log_config: object, app: str = WORKER_APP) -> Mapping[str, object]: + def child_process_uvicorn_overrides(log_config: object) -> Mapping[str, object]: if log_config is None: - return MappingProxyType({"app": app, "factory": True}) + return MappingProxyType({"app": WORKER_APP, "factory": True}) ProxyInitializationHelpers.apply_log_config(_LOG_CONFIG.validate_python(log_config)) os.environ[WORKER_LOG_CONFIG_ENV] = json.dumps(log_config) - return MappingProxyType({"app": app, "factory": True, "log_config": None}) + return MappingProxyType({"app": WORKER_APP, "factory": True, "log_config": None}) @staticmethod def apply_worker_log_config(environ: Mapping[str, str]) -> None: @@ -415,7 +412,6 @@ class ProxyInitializationHelpers: log_config: str | None = None, keepalive_timeout: int | None = None, timeout_worker_healthcheck: int | None = None, - forwarded: "ServerOverrides | None" = None, ) -> dict: """ Get the arguments for `uvicorn` worker @@ -434,16 +430,6 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), - **( - {} - if forwarded is None - else { - "app": FORWARD_APP, - "factory": True, - "fd": forwarded.fd, - "proxy_headers": forwarded.proxy_headers, - } - ), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -697,7 +683,8 @@ class ProxyInitializationHelpers: ssl_keyfile_path: str, max_requests_before_restart: int | None = None, max_requests_before_restart_jitter: int | None = None, - forwarded: "ServerOverrides | None" = None, + bind: str | None = None, + proxy_headers: bool = True, ): """ Run litellm with `gunicorn` @@ -770,10 +757,10 @@ class ProxyInitializationHelpers: ) graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout() gunicorn_options: Final = { - "bind": f"{host}:{port}" if forwarded is None else f"fd://{forwarded.fd}", + "bind": f"{host}:{port}" if bind is None else bind, "workers": num_workers, # default is 1 "worker_class": build_graceful_shutdown_uvicorn_worker( - graceful_shutdown_timeout, proxy_headers=True if forwarded is None else forwarded.proxy_headers + graceful_shutdown_timeout, proxy_headers=proxy_headers ), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, "preload": True, # Add the preload flag, @@ -844,29 +831,49 @@ class ProxyInitializationHelpers: _is_port_in_use = is_port_in_use @staticmethod - def public_listener(host: str, port: int, keepalive_timeout: int | None, run_gunicorn: bool) -> "PublicListener": + def serving_port(port: int) -> int: + return random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port + + @staticmethod + def python_server( + host: str, + port: int, + keepalive_timeout: int | None, + run_gunicorn: bool, + run_hypercorn: bool, + run_granian: bool, + tls: bool, + ) -> "PythonServer": """``run_gunicorn_server`` never hands ``--keepalive_timeout`` to gunicorn, so its workers keep gunicorn's.""" + import uvicorn from gunicorn.config import Config as GunicornConfig - from litellm.proxy.rust_forward.listener import ( - PublicListener, - shutdown_drain_seconds, - uvicorn_default_keepalive, - ) + from litellm.proxy.rust_forward.server import Listener, PythonServer, Server from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager + server: Final = ( + Server.HYPERCORN + if run_hypercorn + else Server.GRANIAN + if run_granian + else Server.GUNICORN + if run_gunicorn + else Server.UVICORN + ) keepalive: Final = ( TypeAdapter(int).validate_python(GunicornConfig().keepalive) - if run_gunicorn - else (uvicorn_default_keepalive() if keepalive_timeout is None else keepalive_timeout) + if server is Server.GUNICORN + else uvicorn.Config(app="", log_config=None).timeout_keep_alive + if keepalive_timeout is None + else keepalive_timeout ) - return PublicListener( + return PythonServer( + server=server, + listener=Listener.TLS if tls else Listener.TCP, host=host, - port=random.randint(1024, 49152) - if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) - else port, + port=ProxyInitializationHelpers.serving_port(port), keepalive_seconds=keepalive, - shutdown_drain_seconds=shutdown_drain_seconds(GracefulShutdownManager.get_timeout()), + graceful_timeout_seconds=GracefulShutdownManager.get_timeout(), proxy_headers=True, trusted_proxies=None, ) @@ -1328,30 +1335,32 @@ def run_server( run_setup_wizard() return - from litellm.proxy.rust_forward.mode import Supervised, Supervisor, rust_forward_mode, server_overrides + from litellm.proxy.rust_forward.mode import ( + Supervisor, + gunicorn_bind, + rust_forward_mode, + server_proxy_headers, + uvicorn_options, + ) forward_mode: Final = rust_forward_mode(os.environ) if isinstance(forward_mode, Supervisor) and not any( (version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data) ): - from litellm.proxy.rust_forward.listener import StartupError, startup_check - from litellm.proxy.rust_forward.supervisor import cli_command, gateway_binary, supervise + from litellm.proxy.rust_forward.supervisor import supervise - binary: Final = startup_check( - gateway_binary(sysconfig.get_path("scripts")), - { - "--ssl_keyfile_path": ssl_keyfile_path, - "--ssl_certfile_path": ssl_certfile_path, - "--ciphers": ciphers, - "--run_hypercorn": run_hypercorn, - "--run_granian": run_granian, - }, + server: Final = ProxyInitializationHelpers.python_server( + host, + port, + keepalive_timeout, + run_gunicorn=run_gunicorn, + run_hypercorn=run_hypercorn, + run_granian=run_granian, + tls=any((ssl_keyfile_path, ssl_certfile_path, ciphers)), + ) + raise SystemExit( + supervise(server, (sys.executable, "-m", "litellm.proxy.proxy_cli", *sys.argv[1:]), os.environ) ) - if isinstance(binary, StartupError): - raise click.ClickException(binary.reason) - listener: Final = ProxyInitializationHelpers.public_listener(host, port, keepalive_timeout, run_gunicorn) - raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), listener, os.environ)) - forwarded: Final = server_overrides(forward_mode) if isinstance(forward_mode, Supervised) else None args: Final = locals() try: @@ -1772,8 +1781,7 @@ def run_server( ) sys.exit(1) export_pooled_database_url(pooled_database_url) - if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port): - port = random.randint(1024, 49152) + port = ProxyInitializationHelpers.serving_port(port) if prometheus_metrics_port == port: raise click.UsageError("--prometheus_metrics_port must differ from --port") @@ -1802,15 +1810,14 @@ def run_server( if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None: from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process - metrics_bind_host: Final = host if forwarded is None else forwarded.metrics_host try: metrics_process: Final = start_metrics_server_process( - host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir + host=host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir ) except MetricsServerStartupError as error: raise click.ClickException(str(error)) from error print_startup_message( - f"\033[1;32mLiteLLM: Serving Prometheus metrics on {metrics_bind_host}:{prometheus_metrics_port}/metrics " + f"\033[1;32mLiteLLM: Serving Prometheus metrics on {host}:{prometheus_metrics_port}/metrics " f"(pid {metrics_process.pid})\033[0m" ) @@ -1821,8 +1828,8 @@ def run_server( log_config=log_config, keepalive_timeout=keepalive_timeout, timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None), - forwarded=forwarded, ) + uvicorn_args.update(uvicorn_options(forward_mode, WORKER_APP)) # Optional: recycle uvicorn workers after N requests if max_requests_before_restart is not None: uvicorn_args["limit_max_requests"] = max_requests_before_restart @@ -1861,7 +1868,7 @@ def run_server( start_query_engine_reaper() if num_workers > 1 or reload: worker_overrides: Final = ProxyInitializationHelpers.child_process_uvicorn_overrides( - uvicorn_args.get("log_config"), WORKER_APP if forwarded is None else FORWARD_APP + uvicorn_args.get("log_config") ) if reload and num_workers > 1: from litellm.proxy.dev_reload import ( @@ -1898,7 +1905,8 @@ def run_server( ssl_keyfile_path=ssl_keyfile_path, max_requests_before_restart=max_requests_before_restart, max_requests_before_restart_jitter=max_requests_before_restart_jitter, - forwarded=forwarded, + bind=gunicorn_bind(forward_mode, f"{host}:{port}"), + proxy_headers=server_proxy_headers(forward_mode), ) elif run_hypercorn is True: ProxyInitializationHelpers.init_hypercorn_server( diff --git a/litellm/proxy/rust_forward/AGENTS.md b/litellm/proxy/rust_forward/AGENTS.md index a2ab93cde5c..8bb6d87e5ef 100644 --- a/litellm/proxy/rust_forward/AGENTS.md +++ b/litellm/proxy/rust_forward/AGENTS.md @@ -2,7 +2,11 @@ With `LITELLM_RUST` on, the supervisor here runs the Rust gateway on the public port and Python on a private loopback listener. This package is the Python half of forwarding and pairs with the crate `litellm-rust/crates/gateway-forward`. Once Rust serves every route, both get deleted together, along with one dispatch branch in each entry point -- Nothing outside this package imports Rust forwarding concepts, except the dispatch, the flag adapter and the app wrap in `litellm/proxy/proxy_cli.py` and `gateway/launch.py`. The proxy app itself never knows Rust is in front of it -- Python must see each request as if the client had connected directly. `x-litellm-rust-peer` is the only Rust-specific input, and `peer.py` honors it only from the loopback hop -- Python owns every setting Rust needs. The supervisor resolves the host, port, keepalive and shutdown drain Python actually runs with and hands them to Rust as the `LITELLM_FORWARD_*` variables in `rust_spec`. Rust never reads Python's own settings, so a new value Rust needs gets a new variable there -- Entry points turn their own flags into a `PublicListener`, apply `server_overrides` in the supervised child, and wrap the app with `peer.wrap` where they load it +Every forwarding decision lives here. Entry points (`litellm/proxy/proxy_cli.py`, `gateway/launch.py`, `litellm_cli/main.py`) only translate between their own flags and this package: + +- In the supervisor they describe what they would serve with as a `server.PythonServer` and hand it to `supervisor.supervise`, which decides whether forwarding can start, what Rust and the Python child receive, and how to report a refusal +- In the supervised child they merge `mode.uvicorn_options`, `mode.uvicorn_flags` or `mode.gunicorn_bind` and `mode.server_proxy_headers` into their server settings, and wrap the app with `peer.wrap` in their worker factory. These are identities when forwarding is off, so entry points never branch on it + +Each environment contract is written and read in one place. `mode.supervised_environment` and `mode.rust_forward_mode` are the two halves of the Python child's, and `supervisor.rust_spec` writes the `LITELLM_FORWARD_*` variables that `gateway-forward`'s `settings.rs` reads. Python owns every value Rust needs, so a new one gets a new variable in `rust_spec` + +Python must see each request as if the client had connected directly. `x-litellm-rust-peer` is the only Rust-specific input, and `peer.py` honors it only from the loopback hop. The proxy app itself never knows Rust is in front of it diff --git a/litellm/proxy/rust_forward/asgi.py b/litellm/proxy/rust_forward/asgi.py deleted file mode 100644 index ff9bdc57cf8..00000000000 --- a/litellm/proxy/rust_forward/asgi.py +++ /dev/null @@ -1,10 +0,0 @@ -import os - -from litellm.proxy.rust_forward.mode import rust_forward_mode -from litellm.proxy.rust_forward.peer import ASGIApp, wrap - - -def app() -> ASGIApp: - from litellm_cli.main import proxy_worker_app - - return wrap(proxy_worker_app(), rust_forward_mode(os.environ)) diff --git a/litellm/proxy/rust_forward/listener.py b/litellm/proxy/rust_forward/listener.py deleted file mode 100644 index 28595a4099e..00000000000 --- a/litellm/proxy/rust_forward/listener.py +++ /dev/null @@ -1,46 +0,0 @@ -import os -from collections.abc import Mapping -from dataclasses import dataclass -from typing import Final - -from litellm.rust_bridge.configuration import ENV_NAME - - -@dataclass(frozen=True, slots=True) -class PublicListener: - host: str - port: int - keepalive_seconds: int - shutdown_drain_seconds: float - proxy_headers: bool - trusted_proxies: str | None - - -@dataclass(frozen=True, slots=True) -class StartupError: - reason: str - - -def startup_check(binary: str | None, unsupported: Mapping[str, object]) -> str | StartupError: - flag: Final = next((name for name, value in unsupported.items() if value), None) - if flag is not None: - return StartupError(f"{ENV_NAME} does not support {flag}") - if os.name != "posix": - return StartupError(f"{ENV_NAME} requires POSIX inherited sockets and process signals") - if binary is None: - return StartupError( - f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment" - ) - return binary - - -def uvicorn_default_keepalive() -> int: - import uvicorn - - return uvicorn.Config(app="", log_config=None).timeout_keep_alive - - -def shutdown_drain_seconds(graceful_timeout: float) -> float: - from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS - - return graceful_timeout + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS diff --git a/litellm/proxy/rust_forward/mode.py b/litellm/proxy/rust_forward/mode.py index a75287688ea..2e035550c36 100644 --- a/litellm/proxy/rust_forward/mode.py +++ b/litellm/proxy/rust_forward/mode.py @@ -1,13 +1,14 @@ from collections.abc import Mapping from dataclasses import dataclass +from types import MappingProxyType from typing import Final, TypeAlias from pydantic import TypeAdapter +from typing_extensions import assert_never from litellm.rust_bridge.configuration import environment_switch GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" -METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1" @@ -30,7 +31,6 @@ class Supervisor: @dataclass(frozen=True, slots=True) class Supervised: listener_fd: int - metrics_host: str proxy_headers: bool trusted_proxies: str @@ -38,12 +38,21 @@ class Supervised: RustForwardMode: TypeAlias = Disabled | Supervisor | Supervised +def supervised_environment(listener_fd: int, proxy_headers: bool, trusted_proxies: str | None) -> Mapping[str, str]: + return MappingProxyType( + { + GATEWAY_FD: str(listener_fd), + PROXY_HEADERS: str(proxy_headers).lower(), + **({} if trusted_proxies is None else {TRUSTED_PROXIES: trusted_proxies}), + } + ) + + def rust_forward_mode(environ: Mapping[str, str]) -> RustForwardMode: fd: Final = environ.get(GATEWAY_FD) if fd is not None: return Supervised( _FD.validate_python(fd), - environ[METRICS_HOST], _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), ) @@ -52,17 +61,45 @@ def rust_forward_mode(environ: Mapping[str, str]) -> RustForwardMode: return Disabled() -@dataclass(frozen=True, slots=True) -class ServerOverrides: - fd: int - proxy_headers: bool - metrics_host: str +# A supervised server's TCP peer is always Rust on loopback, so server-level proxy headers would trust any +# client's forwarding headers. They stay off there and ``peer.wrap`` applies the operator's trust instead. -def server_overrides(mode: Supervised) -> ServerOverrides: - """The private listener the supervised server binds instead of its own flags. +def uvicorn_options(mode: RustForwardMode, factory: str) -> Mapping[str, object]: + match mode: + case Supervised(listener_fd): + return MappingProxyType({"app": factory, "factory": True, "fd": listener_fd, "proxy_headers": False}) + case Disabled() | Supervisor(): + return MappingProxyType({}) + case _: + assert_never(mode) - Proxy headers stay off at the server, since it would judge Rust's loopback address instead of the real peer. - ``peer.wrap`` applies the operator's trust settings to the restored peer. - """ - return ServerOverrides(fd=mode.listener_fd, proxy_headers=False, metrics_host=mode.metrics_host) + +def uvicorn_flags(mode: RustForwardMode) -> tuple[str, ...]: + match mode: + case Supervised(listener_fd): + return ("--fd", str(listener_fd), "--no-proxy-headers") + case Disabled() | Supervisor(): + return () + case _: + assert_never(mode) + + +def gunicorn_bind(mode: RustForwardMode, public: str) -> str: + match mode: + case Supervised(listener_fd): + return f"fd://{listener_fd}" + case Disabled() | Supervisor(): + return public + case _: + assert_never(mode) + + +def server_proxy_headers(mode: RustForwardMode) -> bool: + match mode: + case Supervised(): + return False + case Disabled() | Supervisor(): + return True + case _: + assert_never(mode) diff --git a/litellm/proxy/rust_forward/server.py b/litellm/proxy/rust_forward/server.py new file mode 100644 index 00000000000..8398cbda533 --- /dev/null +++ b/litellm/proxy/rust_forward/server.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass +from enum import Enum + + +class Server(Enum): + UVICORN = "uvicorn" + GUNICORN = "gunicorn" + HYPERCORN = "Hypercorn" + GRANIAN = "Granian" + + +class Listener(Enum): + TCP = "plain TCP" + TLS = "TLS" + UNIX_SOCKET = "a Unix socket" + INHERITED_FD = "an inherited file descriptor" + + +@dataclass(frozen=True, slots=True) +class PythonServer: + """What an entry point would serve with on the public port if Rust were not in front of it.""" + + server: Server + listener: Listener + host: str + port: int + keepalive_seconds: int + graceful_timeout_seconds: float + proxy_headers: bool + trusted_proxies: str | None diff --git a/litellm/proxy/rust_forward/supervisor.py b/litellm/proxy/rust_forward/supervisor.py index 6ab4b86dca3..25d443ed35a 100644 --- a/litellm/proxy/rust_forward/supervisor.py +++ b/litellm/proxy/rust_forward/supervisor.py @@ -5,7 +5,8 @@ import shutil import signal import socket import sys -from collections.abc import Mapping, Sequence +import sysconfig +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from enum import Enum, IntEnum from types import MappingProxyType @@ -14,8 +15,10 @@ from typing import Final, TypeAlias from pydantic import TypeAdapter from typing_extensions import assert_never -from litellm.proxy.rust_forward.listener import PublicListener -from litellm.proxy.rust_forward.mode import GATEWAY_FD, METRICS_HOST, PROXY_HEADERS, RUST_HOP, TRUSTED_PROXIES +from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS +from litellm.proxy.rust_forward.mode import RUST_HOP, supervised_environment +from litellm.proxy.rust_forward.server import Listener, PythonServer, Server +from litellm.rust_bridge.configuration import ENV_NAME _ADDRESS: Final = TypeAdapter(tuple[str, int]) @@ -25,6 +28,25 @@ def gateway_binary(scripts_dir: str) -> str | None: return shutil.which(name, path=scripts_dir) +@dataclass(frozen=True, slots=True) +class StartupError: + reason: str + + +def startup_check(server: PythonServer, binary: str | None) -> str | StartupError: + if server.server in (Server.HYPERCORN, Server.GRANIAN): + return StartupError(f"{ENV_NAME} does not support {server.server.value}") + if server.listener is not Listener.TCP: + return StartupError(f"{ENV_NAME} does not support {server.listener.value} on the public port") + if os.name != "posix": + return StartupError(f"{ENV_NAME} requires POSIX inherited sockets and process signals") + if binary is None: + return StartupError( + f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment" + ) + return binary + + class Child(Enum): RUST = "rust" PYTHON = "python" @@ -116,34 +138,30 @@ class ChildSpec: def python_spec( - command: Sequence[str], listener_fd: int, upstream_port: int, listener: PublicListener, environ: Mapping[str, str] + command: Sequence[str], listener_fd: int, upstream_port: int, server: PythonServer, environ: Mapping[str, str] ) -> ChildSpec: return ChildSpec( - argv=(*command, "--host", RUST_HOP, "--port", str(upstream_port)), + argv=(*command, "--port", str(upstream_port)), env=MappingProxyType( - { - **environ, - GATEWAY_FD: str(listener_fd), - METRICS_HOST: listener.host, - PROXY_HEADERS: str(listener.proxy_headers).lower(), - **({} if listener.trusted_proxies is None else {TRUSTED_PROXIES: listener.trusted_proxies}), - } + {**environ, **supervised_environment(listener_fd, server.proxy_headers, server.trusted_proxies)} ), pass_fds=(listener_fd,), ) -def rust_spec(binary: str, upstream_port: int, listener: PublicListener, environ: Mapping[str, str]) -> ChildSpec: +def rust_spec(binary: str, upstream_port: int, server: PythonServer, environ: Mapping[str, str]) -> ChildSpec: return ChildSpec( argv=(binary,), env=MappingProxyType( { **environ, "LITELLM_FORWARD_UPSTREAM": f"http://{RUST_HOP}:{upstream_port}", - "LITELLM_FORWARD_HOST": listener.host, - "LITELLM_FORWARD_PORT": str(listener.port), - "LITELLM_FORWARD_UPSTREAM_KEEPALIVE": str(listener.keepalive_seconds), - "LITELLM_FORWARD_SHUTDOWN_DRAIN": str(listener.shutdown_drain_seconds), + "LITELLM_FORWARD_HOST": server.host, + "LITELLM_FORWARD_PORT": str(server.port), + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE": str(server.keepalive_seconds), + "LITELLM_FORWARD_SHUTDOWN_DRAIN": str( + server.graceful_timeout_seconds + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS + ), } ), pass_fds=(), @@ -177,9 +195,7 @@ async def _drive( return await _drive(following, events, children) -async def _supervise( - binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str] -) -> ExitCode: +async def _supervise(binary: str, command: Sequence[str], server: PythonServer, environ: Mapping[str, str]) -> ExitCode: loop: Final = asyncio.get_running_loop() events: Final[asyncio.Queue[Event]] = asyncio.Queue() for signum in (signal.SIGTERM, signal.SIGINT): @@ -188,9 +204,9 @@ async def _supervise( private.bind((RUST_HOP, 0)) private.listen() _, upstream_port = _ADDRESS.validate_python(private.getsockname()) - python: Final = await _spawn(python_spec(command, private.fileno(), upstream_port, listener, environ)) + python: Final = await _spawn(python_spec(command, private.fileno(), upstream_port, server, environ)) try: - rust: Final = await _spawn(rust_spec(binary, upstream_port, listener, environ)) + rust: Final = await _spawn(rust_spec(binary, upstream_port, server, environ)) except OSError: _terminate_group(python) await python.wait() @@ -207,13 +223,27 @@ async def _supervise( await asyncio.gather(*waiters) -def supervise(binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str]) -> int: +def _nothing_to_prepare() -> None: + pass + + +def supervise( + server: PythonServer, + command: Sequence[str], + environ: Mapping[str, str], + before_start: Callable[[], None] = _nothing_to_prepare, +) -> int: + """Serve ``server``'s public port from Rust and run ``command`` behind it, or say why that cannot start. + + ``before_start`` runs only once forwarding is known to be possible, so a refused start leaves nothing behind. + """ + checked: Final = startup_check(server, gateway_binary(sysconfig.get_path("scripts"))) + if isinstance(checked, StartupError): + sys.stderr.write(f"{checked.reason}\n") + return ExitCode.CHILD_FAILED + before_start() try: - return asyncio.run(_supervise(binary, command, listener, environ)) + return asyncio.run(_supervise(checked, command, server, environ)) except OSError as error: sys.stderr.write(f"Rust gateway startup failed: {error}\n") return ExitCode.CHILD_FAILED - - -def cli_command(argv: Sequence[str]) -> tuple[str, ...]: - return (sys.executable, "-m", "litellm.proxy.proxy_cli", *argv) diff --git a/litellm_cli/main.py b/litellm_cli/main.py index a9dfa4be179..ea170603d08 100644 --- a/litellm_cli/main.py +++ b/litellm_cli/main.py @@ -33,7 +33,7 @@ from contextlib import contextmanager from typing import TYPE_CHECKING, Final if TYPE_CHECKING: - from fastapi import FastAPI + from litellm.proxy.rust_forward.peer import ASGIApp SDK_PACKAGE: Final = "litellm" CLI_PACKAGE: Final = "litellm.proxy.client" @@ -214,10 +214,12 @@ def without_gc_pauses() -> Generator[None, None, None]: gc.enable() -def proxy_worker_app() -> "FastAPI": +def proxy_worker_app() -> "ASGIApp": with without_gc_pauses(): from litellm.proxy.proxy_cli import ProxyInitializationHelpers ProxyInitializationHelpers.apply_worker_log_config(os.environ) from litellm.proxy.proxy_server import app - return app + from litellm.proxy.rust_forward.mode import rust_forward_mode + from litellm.proxy.rust_forward.peer import wrap + return wrap(app, rust_forward_mode(os.environ)) diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index 36eaabfb029..37541953e42 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -16,13 +16,12 @@ import uvicorn from uvicorn.importer import import_from_string from uvicorn.main import main as uvicorn_main -from gateway.launch import GATEWAY_APP, main, pool_database_url, public_listener, uvicorn_argv -from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS +from gateway.launch import GATEWAY_APP, main, pool_database_url, python_server, uvicorn_argv from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings from litellm.proxy.proxy_server import app as proxy_app -from litellm.proxy.rust_forward.mode import Supervised, server_overrides -from litellm.proxy.rust_forward.supervisor import rust_spec +from litellm.proxy.rust_forward.mode import Supervised, uvicorn_flags +from litellm.proxy.rust_forward.server import Listener from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager DB_ENV: Final = { @@ -120,13 +119,32 @@ def _uvicorn_params(argv: tuple[str, ...]) -> dict[str, object]: class TestUvicornArgv: @pytest.mark.parametrize( - "arguments", [("--uds", "/tmp/gateway.sock"), ("--fd", "0"), ("--fd=17",), ("--ssl-keyfile", "unused")] + ("arguments", "listener"), + [ + ((), Listener.TCP), + (("--uds", "/tmp/gateway.sock"), Listener.UNIX_SOCKET), + (("--fd", "0"), Listener.INHERITED_FD), + (("--fd=17",), Listener.INHERITED_FD), + (("--ssl-keyfile", "unused"), Listener.TLS), + (("--ssl-ciphers", "TLSv1.2"), Listener.TLS), + ], ) - def test_forwarding_rejects_unsupported_listeners_before_starting_processes( - self, arguments: tuple[str, ...] + def test_forwarding_sees_the_public_listener_uvicorn_would_open( + self, arguments: tuple[str, ...], listener: Listener ) -> None: - with pytest.raises(SystemExit, match=f"does not support {arguments[0].split('=')[0]}"): - public_listener(arguments, "gateway") + assert python_server(arguments).listener is listener + + def test_an_unsupported_listener_stops_forwarding_before_the_database_is_prepared( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + monkeypatch.setenv("LITELLM_RUST", "true") + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true") + monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", "/nonexistent/pgbouncer") + with pytest.raises(SystemExit) as stopped: + main(("--uds", "/tmp/gateway.sock"), serve=lambda argv: None) + assert stopped.value.code == 1 + assert capsys.readouterr().err == "LITELLM_RUST does not support a Unix socket on the public port\n" @pytest.mark.parametrize( ("argv", "trust"), @@ -141,9 +159,9 @@ class TestUvicornArgv: self, argv: tuple[str, ...], trust: tuple[bool, str | None], monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.delenv("FORWARDED_ALLOW_IPS", raising=False) - binary, listener = public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") - assert (binary, listener.host, listener.port) == ("gateway", "0.0.0.0", 4000) - assert (listener.proxy_headers, listener.trusted_proxies) == trust + server: Final = python_server(("--host", "0.0.0.0", "--port", "4000", *argv)) + assert (server.host, server.port) == ("0.0.0.0", 4000) + assert (server.proxy_headers, server.trusted_proxies) == trust @pytest.mark.parametrize( ("argv", "environ"), @@ -155,53 +173,49 @@ class TestUvicornArgv: monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) for name, value in environ.items(): monkeypatch.setenv(name, value) - _, listener = public_listener(argv, "gateway") - sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"] - assert sent == str(_uvicorn_params(uvicorn_argv(argv, os.environ, None))["timeout_keep_alive"]) + served: Final = _uvicorn_params(uvicorn_argv(argv, os.environ))["timeout_keep_alive"] + assert python_server(argv).keepalive_seconds == served def test_without_a_keepalive_rust_follows_uvicorns_own_default(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) - _, listener = public_listener((), "gateway") - assert listener.keepalive_seconds == uvicorn.Config(app="", log_config=None).timeout_keep_alive + assert python_server(()).keepalive_seconds == uvicorn.Config(app="", log_config=None).timeout_keep_alive @pytest.mark.parametrize("argv", [(), ("--timeout-graceful-shutdown", "12")]) def test_rust_drains_for_pythons_graceful_timeout_plus_teardown( self, argv: tuple[str, ...], monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "7.5") - _, listener = public_listener(argv, "gateway") - flag: Final = _uvicorn_params(uvicorn_argv(argv, {}, None))["timeout_graceful_shutdown"] + flag: Final = _uvicorn_params(uvicorn_argv(argv, {}))["timeout_graceful_shutdown"] graceful: Final = GracefulShutdownManager.get_timeout() if flag is None else float(str(flag)) - sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"] - assert float(sent) == graceful + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS + assert python_server(argv).graceful_timeout_seconds == graceful def test_supervised_listener_uses_the_inherited_loopback_socket(self): - supervised: Final = server_overrides(Supervised(17, "0.0.0.0", True, "127.0.0.1")) - params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised)) + supervised: Final = uvicorn_flags(Supervised(17, True, "127.0.0.1")) + params: Final = _uvicorn_params((*uvicorn_argv(("--workers", "2"), {}), *supervised)) assert params["fd"] == 17 assert params["workers"] == 2 @pytest.mark.parametrize("argv", [(), ("--proxy-headers",), ("--forwarded-allow-ips=*",)]) def test_private_hop_leaves_proxy_headers_to_the_app(self, argv: tuple[str, ...]): - params: Final = _uvicorn_params(uvicorn_argv(argv, {}, server_overrides(Supervised(17, "0.0.0.0", True, "*")))) + params: Final = _uvicorn_params((*uvicorn_argv(argv, {}), *uvicorn_flags(Supervised(17, True, "*")))) assert params["proxy_headers"] is False def test_direct_listener_keeps_explicit_proxy_header_options(self): - params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {}, None)) + params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {})) assert params["proxy_headers"] is False assert params["forwarded_allow_ips"] == "*" def test_keepalive_env_reaches_uvicorn(self): - params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"}, None)) + params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"})) assert params["app"] == GATEWAY_APP assert params["workers"] == 4 assert params["timeout_keep_alive"] == 75 def test_unset_env_keeps_the_uvicorn_default(self): - assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}, None))["timeout_keep_alive"] == 5 + assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}))["timeout_keep_alive"] == 5 def test_an_explicit_flag_wins_over_the_env(self): - argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}, None) + argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}) assert _uvicorn_params(argv)["timeout_keep_alive"] == 30 def test_the_app_uvicorn_is_told_to_serve_is_the_trimmed_gateway(self, monkeypatch: pytest.MonkeyPatch) -> None: @@ -218,7 +232,7 @@ class TestUvicornArgv: ): monkeypatch.delenv(key, raising=False) - params: Final = _uvicorn_params(uvicorn_argv((), {}, None)) + params: Final = _uvicorn_params(uvicorn_argv((), {})) assert params["factory"] is True try: served: Final = import_from_string(cast(str, params["app"]))() diff --git a/tests/unit/proxy/rust_forward/test_listener.py b/tests/unit/proxy/rust_forward/test_listener.py deleted file mode 100644 index d17b0e74e14..00000000000 --- a/tests/unit/proxy/rust_forward/test_listener.py +++ /dev/null @@ -1,74 +0,0 @@ -from collections.abc import Mapping -from typing import Final - -import pytest -import uvicorn -from click.testing import CliRunner -from gunicorn.config import Config as GunicornConfig - -from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS -from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server -from litellm.proxy.rust_forward.listener import PublicListener, StartupError, startup_check -from litellm.proxy.rust_forward.supervisor import rust_spec - - -@pytest.mark.parametrize( - "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] -) -def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("LITELLM_RUST", "true") - monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) - values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] - result: Final = CliRunner().invoke(run_server, [flag, *values]) - assert result.exit_code != 0 - assert f"LITELLM_RUST does not support {flag}" in result.output - - -def test_missing_binary_gives_install_instruction() -> None: - assert startup_check(None, {}) == StartupError( - "LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment" - ) - - -def _cli_listener(argv: list[str]) -> tuple[Mapping[str, object], PublicListener]: - params: Final = run_server.make_context("litellm", ["--port", "45999", *argv]).params - return params, ProxyInitializationHelpers.public_listener( - str(params["host"]), int(str(params["port"])), params["keepalive_timeout"], params["run_gunicorn"] is True - ) - - -def _uvicorn_keepalive(options: Mapping[str, object]) -> int: - configured: Final = {"timeout_keep_alive": options["timeout_keep_alive"]} if "timeout_keep_alive" in options else {} - return uvicorn.Config(app="", log_config=None, **configured).timeout_keep_alive - - -@pytest.mark.parametrize( - ("argv", "environ"), - [(["--keepalive_timeout", "2"], {}), ([], {"KEEPALIVE_TIMEOUT": "7"}), ([], {})], -) -def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with( - argv: list[str], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) - for name, value in environ.items(): - monkeypatch.setenv(name, value) - params, listener = _cli_listener(argv) - uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( - "127.0.0.1", 45555, keepalive_timeout=params["keepalive_timeout"] - ) - sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"] - assert sent == str(_uvicorn_keepalive(uvicorn_options)) - - -def test_gunicorn_workers_keep_gunicorns_keepalive_and_rust_follows(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) - _, listener = _cli_listener(["--run_gunicorn", "--keepalive_timeout", "9"]) - assert listener.keepalive_seconds == GunicornConfig().keepalive - - -def test_rust_drains_for_pythons_graceful_timeout_plus_teardown(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "12.5") - _, listener = _cli_listener([]) - uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555) - sent: Final = float(rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"]) - assert sent == uvicorn_options["timeout_graceful_shutdown"] + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS diff --git a/tests/unit/proxy/rust_forward/test_mode.py b/tests/unit/proxy/rust_forward/test_mode.py index 20005a21255..b503da56f4a 100644 --- a/tests/unit/proxy/rust_forward/test_mode.py +++ b/tests/unit/proxy/rust_forward/test_mode.py @@ -1,9 +1,20 @@ from typing import Final import pytest +import uvicorn -from litellm.proxy.proxy_cli import ProxyInitializationHelpers -from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides +from litellm.proxy.proxy_cli import WORKER_APP, ProxyInitializationHelpers +from litellm.proxy.rust_forward.mode import ( + Disabled, + RustForwardMode, + Supervised, + Supervisor, + gunicorn_bind, + rust_forward_mode, + server_proxy_headers, + supervised_environment, + uvicorn_options, +) @pytest.mark.parametrize( @@ -20,24 +31,54 @@ from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, ru ({"LITELLM_RUST": " 1 "}, Supervisor()), ({"LITELLM_RUST": "enabled"}, Disabled()), ( - { - "LITELLM_RUST": "true", - "_LITELLM_GATEWAY_FD": "17", - "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", - "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", - }, - Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"), + {"LITELLM_RUST": "true", "_LITELLM_GATEWAY_FD": "17", "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5"}, + Supervised(17, True, "127.0.0.1,10.0.0.5"), ), ], ) -def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None: +def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: RustForwardMode) -> None: assert rust_forward_mode(environ) == expected -@pytest.mark.parametrize("mode", [None, Supervised(17, "0.0.0.0", True, "127.0.0.1")]) -def test_inherited_socket_reaches_uvicorn(mode: Supervised | None) -> None: - options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( - "127.0.0.1", 45555, forwarded=None if mode is None else server_overrides(mode) - ) - assert options.get("fd") == (None if mode is None else mode.listener_fd) - assert options.get("proxy_headers", True) is (mode is None) +@pytest.mark.parametrize( + ("proxy_headers", "trusted_proxies", "inherited", "expected"), + [ + (True, None, {}, Supervised(17, True, "127.0.0.1")), + (True, None, {"FORWARDED_ALLOW_IPS": "10.0.0.5"}, Supervised(17, True, "10.0.0.5")), + (True, "*", {"FORWARDED_ALLOW_IPS": "10.0.0.5"}, Supervised(17, True, "*")), + (False, None, {}, Supervised(17, False, "127.0.0.1")), + ], +) +def test_child_reads_back_the_trust_the_supervisor_wrote( + proxy_headers: bool, trusted_proxies: str | None, inherited: dict[str, str], expected: Supervised +) -> None: + written: Final = supervised_environment(17, proxy_headers, trusted_proxies) + assert rust_forward_mode({**inherited, **written}) == expected + + +def _served_uvicorn_config(mode: RustForwardMode) -> uvicorn.Config: + options: Final = { + **ProxyInitializationHelpers.get_default_unvicorn_init_args("0.0.0.0", 45555), + **uvicorn_options(mode, WORKER_APP), + } + return uvicorn.Config(**options) + + +def test_supervised_uvicorn_serves_the_inherited_socket_without_trusting_the_rust_hop() -> None: + config: Final = _served_uvicorn_config(Supervised(17, True, "*")) + assert (config.fd, config.proxy_headers) == (17, False) + assert (config.app, config.factory) == (WORKER_APP, True) + + +@pytest.mark.parametrize("mode", [Disabled(), Supervisor()]) +def test_unsupervised_servers_keep_their_own_socket_and_proxy_headers(mode: RustForwardMode) -> None: + config: Final = _served_uvicorn_config(mode) + assert (config.fd, config.proxy_headers, config.port) == (None, True, 45555) + assert gunicorn_bind(mode, "0.0.0.0:4000") == "0.0.0.0:4000" + assert server_proxy_headers(mode) is True + + +def test_supervised_gunicorn_binds_the_inherited_socket_without_trusting_the_rust_hop() -> None: + mode: Final = Supervised(17, True, "*") + assert gunicorn_bind(mode, "0.0.0.0:4000") == "fd://17" + assert server_proxy_headers(mode) is False diff --git a/tests/unit/proxy/rust_forward/test_peer.py b/tests/unit/proxy/rust_forward/test_peer.py index e96d9cf2b6a..29d32b29b76 100644 --- a/tests/unit/proxy/rust_forward/test_peer.py +++ b/tests/unit/proxy/rust_forward/test_peer.py @@ -79,10 +79,10 @@ FORWARDED: Final[Headers] = ( @pytest.mark.parametrize( "mode", [ - Supervised(17, "0.0.0.0", True, "127.0.0.1"), - Supervised(17, "0.0.0.0", True, "10.0.0.0/8"), - Supervised(17, "0.0.0.0", True, "*"), - Supervised(17, "0.0.0.0", False, "10.0.0.0/8"), + Supervised(17, True, "127.0.0.1"), + Supervised(17, True, "10.0.0.0/8"), + Supervised(17, True, "*"), + Supervised(17, False, "10.0.0.0/8"), ], ) async def test_python_behind_rust_sees_what_a_direct_connection_would( @@ -101,7 +101,7 @@ async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None: True, ("127.0.0.1", 60000), ((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")), - Supervised(17, "0.0.0.0", True, "127.0.0.1"), + Supervised(17, True, "127.0.0.1"), ) assert (client, scheme) == (("203.0.113.7", 51234), "http") @@ -111,7 +111,7 @@ async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() True, ("192.0.2.9", 1000), ((b"x-litellm-rust-peer", b"10.0.12.34:1"),), - Supervised(17, "0.0.0.0", False, "127.0.0.1"), + Supervised(17, False, "127.0.0.1"), ) assert (client, headers) == (("192.0.2.9", 1000), ()) @@ -121,12 +121,11 @@ async def _peer_probe(request: Request) -> dict[str, object]: return {"client": [request.client.host, request.client.port], "peer": request.headers.get("x-litellm-rust-peer")} -@pytest.mark.parametrize("entry_point", ["litellm.proxy.rust_forward.asgi:app", "gateway.launch:worker_app"]) +@pytest.mark.parametrize("entry_point", ["litellm_cli.main:proxy_worker_app", "gateway.launch:worker_app"]) async def test_every_entry_point_hands_routes_the_peer_rust_saw( entry_point: str, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17") - monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", "0.0.0.0") monkeypatch.setattr(proxy_app.router, "lifespan_context", proxy_app.router.lifespan_context) monkeypatch.setattr( proxy_app.router, "routes", [APIRoute("/__rust_forward_peer_probe", _peer_probe), *proxy_app.router.routes] diff --git a/tests/unit/proxy/rust_forward/test_supervisor.py b/tests/unit/proxy/rust_forward/test_supervisor.py index 224d97f349d..2d9e646c728 100644 --- a/tests/unit/proxy/rust_forward/test_supervisor.py +++ b/tests/unit/proxy/rust_forward/test_supervisor.py @@ -1,14 +1,14 @@ import functools import signal -import sys from pathlib import Path from typing import Final import pytest +from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS from litellm.proxy.proxy_cli import run_server -from litellm.proxy.rust_forward.listener import PublicListener from litellm.proxy.rust_forward.mode import Supervised, rust_forward_mode +from litellm.proxy.rust_forward.server import Listener, PythonServer, Server from litellm.proxy.rust_forward.supervisor import ( Child, Event, @@ -17,55 +17,43 @@ from litellm.proxy.rust_forward.supervisor import ( Finished, Running, Signal, + StartupError, State, Stop, - cli_command, gateway_binary, python_spec, rust_spec, + startup_check, step, + supervise, ) TERM: Final = signal.SIGTERM INT: Final = signal.SIGINT -def _listener(proxy_headers: bool = True, trusted_proxies: str | None = None) -> PublicListener: - return PublicListener("0.0.0.0", 4000, 5, 40.0, proxy_headers, trusted_proxies) +def _server( + server: Server = Server.UVICORN, listener: Listener = Listener.TCP, trusted_proxies: str | None = None +) -> PythonServer: + return PythonServer(server, listener, "0.0.0.0", 4000, 2, 2.5, False, trusted_proxies) -@pytest.mark.parametrize( - ("environ", "listener", "expected"), - [ - ({}, _listener(), Supervised(17, "0.0.0.0", True, "127.0.0.1")), - ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(), Supervised(17, "0.0.0.0", True, "10.0.0.5")), - ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(True, "*"), Supervised(17, "0.0.0.0", True, "*")), - ({}, _listener(False), Supervised(17, "0.0.0.0", False, "127.0.0.1")), - ], -) -def test_python_child_keeps_the_trust_python_alone_would_use( - environ: dict[str, str], listener: PublicListener, expected: Supervised -) -> None: - spec: Final = python_spec(("python",), 17, 45555, listener, environ) - assert rust_forward_mode(spec.env) == expected +def test_python_child_serves_the_private_socket_with_the_operators_trust() -> None: + spec: Final = python_spec(("python",), 17, 45555, _server(trusted_proxies="10.0.0.5"), {"KEEP": "1"}) + assert rust_forward_mode(spec.env) == Supervised(17, False, "10.0.0.5") + assert (spec.env["KEEP"], spec.pass_fds) == ("1", (17,)) -def test_python_child_listens_privately_and_keeps_options() -> None: +def test_python_child_keeps_the_public_host_and_its_options() -> None: arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") - spec: Final = python_spec(arguments, 17, 45555, _listener(), {}) + spec: Final = python_spec(arguments, 17, 45555, _server(), {}) params: Final = run_server.make_context("litellm", list(spec.argv)).params - assert params["host"] == "127.0.0.1" - assert params["port"] == 45555 - assert params["num_workers"] == 2 - assert params["reload"] is True - assert spec.pass_fds == (17,) - assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli") + assert (params["host"], params["port"]) == ("0.0.0.0", 45555) + assert (params["num_workers"], params["reload"]) == (2, True) def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None: - spec: Final = rust_spec( - "gateway", 45555, PublicListener("0.0.0.0", 4000, 2, 12.5, True, None), {"RUST_LOG": "info"} - ) + spec: Final = rust_spec("gateway", 45555, _server(), {"RUST_LOG": "info"}) assert spec.argv == ("gateway",) assert spec.pass_fds == () assert dict(spec.env) == { @@ -74,10 +62,43 @@ def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> Non "LITELLM_FORWARD_HOST": "0.0.0.0", "LITELLM_FORWARD_PORT": "4000", "LITELLM_FORWARD_UPSTREAM_KEEPALIVE": "2", - "LITELLM_FORWARD_SHUTDOWN_DRAIN": "12.5", + "LITELLM_FORWARD_SHUTDOWN_DRAIN": str(2.5 + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS), } +@pytest.mark.parametrize( + ("server", "reason"), + [ + (_server(Server.HYPERCORN), "LITELLM_RUST does not support Hypercorn"), + (_server(Server.GRANIAN), "LITELLM_RUST does not support Granian"), + (_server(listener=Listener.TLS), "LITELLM_RUST does not support TLS on the public port"), + (_server(listener=Listener.UNIX_SOCKET), "LITELLM_RUST does not support a Unix socket on the public port"), + ( + _server(Server.GUNICORN, Listener.INHERITED_FD), + "LITELLM_RUST does not support an inherited file descriptor on the public port", + ), + ], +) +def test_unsupported_servers_are_refused_before_anything_starts( + server: PythonServer, reason: str, capsys: pytest.CaptureFixture[str] +) -> None: + assert startup_check(server, "gateway") == StartupError(reason) + prepared: Final[list[bool]] = [] + assert supervise(server, ("python",), {}, before_start=lambda: prepared.append(True)) == ExitCode.CHILD_FAILED + assert (capsys.readouterr().err, prepared) == (f"{reason}\n", []) + + +@pytest.mark.parametrize("server", [Server.UVICORN, Server.GUNICORN]) +def test_plain_tcp_uvicorn_and_gunicorn_can_be_forwarded(server: Server) -> None: + assert startup_check(_server(server), "gateway") == "gateway" + + +def test_missing_binary_gives_install_instruction() -> None: + assert startup_check(_server(), None) == StartupError( + "LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment" + ) + + def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: scripts: Final = tmp_path / "scripts" elsewhere: Final = tmp_path / "elsewhere" diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index 0ce5cc58191..821e70e5925 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -39,9 +39,11 @@ from litellm.proxy.shutdown.graceful_shutdown_manager import DEFAULT_GRACEFUL_SH @pytest.fixture(autouse=True) def fork_reservation(): """Reserving is irreversible: it would forbid native routes in this pytest worker for good""" - with patch( # test-quality-ok: process-global native state, a real reservation would poison every later test in the worker - "litellm.rust_bridge.fork_guard.reserve_process_for_forking" - ) as reserve: + with ( + patch( # test-quality-ok: process-global native state, a real reservation would poison every later test in the worker + "litellm.rust_bridge.fork_guard.reserve_process_for_forking" + ) as reserve + ): yield reserve @@ -98,42 +100,30 @@ class TestProxyInitializationHelpers: # Execute with pytest.raises(ValueError, match="Invalid test value"): - ProxyInitializationHelpers._run_test_chat_completion( - "localhost", 8000, "gpt-3.5-turbo", True - ) + ProxyInitializationHelpers._run_test_chat_completion("localhost", 8000, "gpt-3.5-turbo", True) # Test with valid string test value - ProxyInitializationHelpers._run_test_chat_completion( - "localhost", 8000, "gpt-3.5-turbo", "http://test-url" - ) + ProxyInitializationHelpers._run_test_chat_completion("localhost", 8000, "gpt-3.5-turbo", "http://test-url") # Assert - mock_openai.assert_called_once_with( - api_key="My API Key", base_url="http://test-url" - ) + mock_openai.assert_called_once_with(api_key="My API Key", base_url="http://test-url") mock_client.chat.completions.create.assert_called() def test_get_default_unvicorn_init_args(self): # Test without log_config - args = ProxyInitializationHelpers._get_default_unvicorn_init_args( - "localhost", 8000 - ) + args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000) assert args["app"] == "litellm.proxy.proxy_server:app" assert args["host"] == "localhost" assert args["port"] == 8000 assert args["server_header"] is False # Test with log_config - args = ProxyInitializationHelpers._get_default_unvicorn_init_args( - "localhost", 8000, "log_config.json" - ) + args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000, "log_config.json") assert args["log_config"] == "log_config.json" # Test with json_logs=True with patch("litellm.json_logs", True): - args = ProxyInitializationHelpers._get_default_unvicorn_init_args( - "localhost", 8000 - ) + args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000) # When json_logs is True, log_config should be set to the JSON log config dict assert args["log_config"] is not None assert isinstance(args["log_config"], dict) @@ -141,15 +131,11 @@ class TestProxyInitializationHelpers: assert "formatters" in args["log_config"] # Test with keepalive_timeout - args = ProxyInitializationHelpers._get_default_unvicorn_init_args( - "localhost", 8000, None, 60 - ) + args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000, None, 60) assert args["timeout_keep_alive"] == 60 # Test with both log_config and keepalive_timeout - args = ProxyInitializationHelpers._get_default_unvicorn_init_args( - "localhost", 8000, "log_config.json", 120 - ) + args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000, "log_config.json", 120) assert args["log_config"] == "log_config.json" assert args["timeout_keep_alive"] == 120 @@ -280,14 +266,10 @@ class TestProxyInitializationHelpers: py_file = tmp_path / "module.py" py_file.write_text("x = 1\n") - applied = ProxyInitializationHelpers._patch_statreload_extra_paths( - [str(config_file)] - ) + applied = ProxyInitializationHelpers._patch_statreload_extra_paths([str(config_file)]) assert applied is True - fake_self = types.SimpleNamespace( - config=types.SimpleNamespace(reload_dirs=[tmp_path]) - ) + fake_self = types.SimpleNamespace(config=types.SimpleNamespace(reload_dirs=[tmp_path])) yielded_paths = {Path(p).resolve() for p in StatReload.iter_py_files(fake_self)} assert config_file.resolve() in yielded_paths @@ -304,14 +286,10 @@ class TestProxyInitializationHelpers: env_file = tmp_path / ".env" env_file.write_text("FOO=bar\n") - applied = ProxyInitializationHelpers._patch_statreload_extra_paths( - [str(env_file)] - ) + applied = ProxyInitializationHelpers._patch_statreload_extra_paths([str(env_file)]) assert applied is True - fake_self = types.SimpleNamespace( - config=types.SimpleNamespace(reload_dirs=[tmp_path]) - ) + fake_self = types.SimpleNamespace(config=types.SimpleNamespace(reload_dirs=[tmp_path])) yielded_paths = {Path(p).resolve() for p in StatReload.iter_py_files(fake_self)} assert env_file.resolve() in yielded_paths @@ -323,10 +301,7 @@ class TestProxyInitializationHelpers: StatReload._litellm_patched_config_paths.clear() assert ProxyInitializationHelpers._patch_statreload_extra_paths([]) is False - assert ( - ProxyInitializationHelpers._patch_statreload_extra_paths([None, ""]) - is False - ) + assert ProxyInitializationHelpers._patch_statreload_extra_paths([None, ""]) is False def test_patch_statreload_extra_paths_is_idempotent(self, tmp_path): from pathlib import Path @@ -344,18 +319,14 @@ class TestProxyInitializationHelpers: for _ in range(3): ProxyInitializationHelpers._patch_statreload_extra_paths([str(config_file)]) - fake_self = types.SimpleNamespace( - config=types.SimpleNamespace(reload_dirs=[tmp_path]) - ) + fake_self = types.SimpleNamespace(config=types.SimpleNamespace(reload_dirs=[tmp_path])) yielded = list(StatReload.iter_py_files(fake_self)) assert len(yielded) == len(set(map(str, yielded))) yielded_paths = {Path(p).resolve() for p in yielded} assert config_file.resolve() in yielded_paths assert py_file.resolve() in yielded_paths - def test_configure_dev_reload_watches_env_and_sets_override_flag( - self, tmp_path, monkeypatch - ): + def test_configure_dev_reload_watches_env_and_sets_override_flag(self, tmp_path, monkeypatch): from pathlib import Path from uvicorn.supervisors.statreload import StatReload @@ -372,9 +343,7 @@ class TestProxyInitializationHelpers: uvicorn_args: dict = {} with patch("litellm._logging.verbose_proxy_logger.warning") as mock_warning: - ProxyInitializationHelpers._configure_dev_reload( - uvicorn_args, str(config_file) - ) + ProxyInitializationHelpers._configure_dev_reload(uvicorn_args, str(config_file)) assert os.environ["LITELLM_DEV_ENV_HOT_RELOAD"] == "True" assert uvicorn_args["reload"] is True @@ -385,9 +354,7 @@ class TestProxyInitializationHelpers: assert "override" in warning_text assert ".env" in warning_text - fake_self = types.SimpleNamespace( - config=types.SimpleNamespace(reload_dirs=[tmp_path]) - ) + fake_self = types.SimpleNamespace(config=types.SimpleNamespace(reload_dirs=[tmp_path])) yielded_paths = {Path(p).resolve() for p in StatReload.iter_py_files(fake_self)} assert env_file.resolve() in yielded_paths assert config_file.resolve() in yielded_paths @@ -411,17 +378,13 @@ class TestProxyInitializationHelpers: mock_app = MagicMock() # Execute - ProxyInitializationHelpers._init_hypercorn_server( - mock_app, "localhost", 8000, None, None, None - ) + ProxyInitializationHelpers._init_hypercorn_server(mock_app, "localhost", 8000, None, None, None) # Assert mock_asyncio_run.assert_called_once() # Test with SSL - ProxyInitializationHelpers._init_hypercorn_server( - mock_app, "localhost", 8000, "cert.pem", "key.pem", "ECDHE" - ) + ProxyInitializationHelpers._init_hypercorn_server(mock_app, "localhost", 8000, "cert.pem", "key.pem", "ECDHE") @patch("granian.Granian") @patch("builtins.print") @@ -590,10 +553,14 @@ class TestProxyInitializationHelpers: database_name_enc = urllib.parse.quote_plus(database_name) # Construct DATABASE_URL from the provided variables - database_url = f"postgresql://{database_username_enc}:{database_password_enc}@{database_host}/{database_name_enc}" + database_url = ( + f"postgresql://{database_username_enc}:{database_password_enc}@{database_host}/{database_name_enc}" + ) # Assert the correct URL was constructed with properly escaped characters - expected_url = "postgresql://user%40with%2Bspecial:test-password-special-chars@localhost:5432/db_name%2Ftest" + expected_url = ( + "postgresql://user%40with%2Bspecial:test-password-special-chars@localhost:5432/db_name%2Ftest" + ) assert database_url == expected_url # Test appending query parameters @@ -611,9 +578,7 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("atexit.register") # critical @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_skip_server_startup( self, mock_should_update, mock_setup_db, mock_atexit_register, mock_uvicorn_run, tmp_path: Path ): @@ -631,11 +596,7 @@ class TestProxyInitializationHelpers: ) # Remove DATABASE_URL/DIRECT_URL so the CLI doesn't attempt # real prisma operations when these are set in CI. - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env["PROMETHEUS_MULTIPROC_DIR"] = str(tmp_path) live_proxy_samples = tmp_path / "counter_123.db" live_proxy_samples.write_bytes(b"samples of a proxy that is still running") @@ -655,9 +616,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -668,18 +627,12 @@ class TestProxyInitializationHelpers: # --- skip startup --- result = runner.invoke(run_server, ["--local", "--skip_server_startup"]) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" assert "Skipping server startup" in result.output mock_uvicorn_run.assert_not_called() - result = runner.invoke( - run_server, ["--local", "--skip_server_startup", "--telemetry", "False"] - ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + result = runner.invoke(run_server, ["--local", "--skip_server_startup", "--telemetry", "False"]) + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" assert "Skipping server startup" in result.output assert "telemetry" not in runner.invoke(run_server, ["--help"]).output assert live_proxy_samples.exists() @@ -689,9 +642,7 @@ class TestProxyInitializationHelpers: result = runner.invoke(run_server, ["--local"]) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() assert not live_proxy_samples.exists() @@ -728,9 +679,7 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_limit_concurrency_passed_to_uvicorn( self, mock_should_update, mock_setup_db, mock_atexit_register, mock_uvicorn_run ): @@ -747,11 +696,7 @@ class TestProxyInitializationHelpers: KeyManagementSettings=MagicMock(), save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -761,9 +706,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.side_effect = lambda *a, **k: { "app": "litellm.proxy.proxy_server:app", @@ -771,20 +714,14 @@ class TestProxyInitializationHelpers: "port": 8000, } - result = runner.invoke( - run_server, ["--local", "--limit_concurrency", "250"] - ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + result = runner.invoke(run_server, ["--local", "--limit_concurrency", "250"]) + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() assert mock_uvicorn_run.call_args.kwargs.get("limit_concurrency") == 250 mock_uvicorn_run.reset_mock() result = runner.invoke(run_server, ["--local"]) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() assert "limit_concurrency" not in mock_uvicorn_run.call_args.kwargs @@ -802,7 +739,9 @@ class TestProxyInitializationHelpers: @patch("httpx.HTTPTransport.handle_request") @patch("atexit.register") @patch("subprocess.Popen") - @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + @patch( + "litellm.proxy.db.prisma_client.PrismaManager.setup_database" + ) # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above @patch( # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False ) @@ -840,9 +779,7 @@ class TestProxyInitializationHelpers: save_worker_config=MagicMock(), ) clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL", "PROMETHEUS_METRICS_PORT") + k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL", "PROMETHEUS_METRICS_PORT") } clean_env["PROMETHEUS_MULTIPROC_DIR"] = str(tmp_path) with ( @@ -868,9 +805,7 @@ class TestProxyInitializationHelpers: run_server, ["--local", "--host", "127.0.0.1", "--port", "4000", "--prometheus_metrics_port", "4001"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_popen.assert_called_once() spawned = list(mock_popen.call_args.args[0]) assert spawned[1:3] == ["-m", "litellm.proxy.prometheus_metrics_server"] @@ -893,9 +828,7 @@ class TestProxyInitializationHelpers: mock_popen.reset_mock() mock_uvicorn_run.reset_mock() result = runner.invoke(run_server, ["--local"]) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_popen.assert_not_called() mock_uvicorn_run.assert_called_once() @@ -909,9 +842,7 @@ class TestProxyInitializationHelpers: mock_popen.assert_not_called() mock_uvicorn_run.assert_not_called() - result = runner.invoke( - run_server, ["--local", "--prometheus_metrics_port", "0"] - ) + result = runner.invoke(run_server, ["--local", "--prometheus_metrics_port", "0"]) assert result.exit_code == 2 assert "Invalid value for '--prometheus_metrics_port'" in result.output mock_popen.assert_not_called() @@ -933,9 +864,7 @@ class TestProxyInitializationHelpers: @patch("subprocess.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_db_timeout_settings_are_forwarded_to_pool_timeout( self, mock_should_update, @@ -968,11 +897,7 @@ class TestProxyInitializationHelpers: } ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), @@ -983,9 +908,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, patch( "litellm.proxy.proxy_cli.append_query_params", side_effect=lambda url, params: ( @@ -1004,9 +927,7 @@ class TestProxyInitializationHelpers: ["--local", "--config", "test-config.yaml", "--skip_server_startup"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_append_query_params.assert_called() appended_params = mock_append_query_params.call_args.args[1] assert appended_params["connection_limit"] == 5 @@ -1061,9 +982,7 @@ class TestProxyInitializationHelpers: @patch("subprocess.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_db_connection_extra_params_forwarded_to_url( self, mock_should_update, @@ -1098,11 +1017,7 @@ class TestProxyInitializationHelpers: } ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), @@ -1113,9 +1028,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, patch( "litellm.proxy.proxy_cli.append_query_params", side_effect=lambda url, params: str(url), @@ -1132,9 +1045,7 @@ class TestProxyInitializationHelpers: ["--local", "--config", "test-config.yaml", "--skip_server_startup"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_append_query_params.assert_called() appended_params = mock_append_query_params.call_args.args[1] assert appended_params["connect_timeout"] == 15 @@ -1185,9 +1096,7 @@ class TestProxyInitializationHelpers: @patch("subprocess.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_disable_prepared_statements_forwarded_to_url( self, mock_should_update, @@ -1219,11 +1128,7 @@ class TestProxyInitializationHelpers: } ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), @@ -1234,9 +1139,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, patch( "litellm.proxy.proxy_cli.append_query_params", side_effect=lambda url, params: str(url), @@ -1253,9 +1156,7 @@ class TestProxyInitializationHelpers: ["--local", "--config", "test-config.yaml", "--skip_server_startup"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_append_query_params.assert_called() appended_params = mock_append_query_params.call_args.args[1] if expect_pgbouncer: @@ -1277,9 +1178,7 @@ class TestProxyInitializationHelpers: @patch("subprocess.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_disable_prepared_statements_env_var_forwarded_to_url( self, mock_should_update, @@ -1327,9 +1226,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, patch( "litellm.proxy.proxy_cli.append_query_params", side_effect=lambda url, params: str(url), @@ -1346,9 +1243,7 @@ class TestProxyInitializationHelpers: ["--local", "--config", "test-config.yaml", "--skip_server_startup"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" appended_params = mock_append_query_params.call_args.args[1] if expect_pgbouncer: assert appended_params["pgbouncer"] == "true", appended_params @@ -1358,9 +1253,7 @@ class TestProxyInitializationHelpers: @patch("subprocess.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_malformed_disable_prepared_statements_env_var_is_rejected_even_when_config_enables_it( self, mock_should_update, @@ -1406,9 +1299,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, patch( "litellm.proxy.proxy_cli.append_query_params", side_effect=lambda url, params: str(url), @@ -1432,9 +1323,7 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_proxy_default_api_version_uses_azure_default( self, mock_should_update, mock_setup_db, mock_atexit_register, mock_uvicorn_run ): @@ -1451,11 +1340,7 @@ class TestProxyInitializationHelpers: KeyManagementSettings=MagicMock(), save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -1465,9 +1350,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -1475,9 +1358,7 @@ class TestProxyInitializationHelpers: "port": 8000, } result = runner.invoke(run_server, ["--local", "--skip_server_startup"]) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_proxy_module.save_worker_config.assert_called_once() call_kwargs = mock_proxy_module.save_worker_config.call_args[1] assert call_kwargs["api_version"] == litellm.AZURE_DEFAULT_API_VERSION @@ -1485,12 +1366,8 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("builtins.print") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) - def test_keepalive_timeout_flag( - self, mock_should_update, mock_setup_db, mock_print, mock_uvicorn_run - ): + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) + def test_keepalive_timeout_flag(self, mock_should_update, mock_setup_db, mock_print, mock_uvicorn_run): """Test that the keepalive_timeout flag is properly passed to uvicorn""" from click.testing import CliRunner @@ -1507,11 +1384,7 @@ class TestProxyInitializationHelpers: # DB-setup block (un-timeout'd `subprocess.run(["prisma"])` + # migrate-deploy retry loop) — same isolation every other run_server # test in this file uses. - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), @@ -1526,9 +1399,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, patch( "litellm.proxy.proxy_cli.ProxyInitializationHelpers.is_port_in_use", return_value=False, @@ -1550,7 +1421,6 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=30, timeout_worker_healthcheck=None, - forwarded=None, ) mock_uvicorn_run.assert_called_once() @@ -1561,12 +1431,8 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("builtins.print") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) - def test_timeout_worker_healthcheck_flag( - self, mock_should_update, mock_setup_db, mock_print, mock_uvicorn_run - ): + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) + def test_timeout_worker_healthcheck_flag(self, mock_should_update, mock_setup_db, mock_print, mock_uvicorn_run): """Test that the --timeout_worker_healthcheck flag is threaded through to the uvicorn init helper.""" from click.testing import CliRunner @@ -1583,11 +1449,7 @@ class TestProxyInitializationHelpers: # DB-setup block (un-timeout'd `subprocess.run(["prisma"])` + # migrate-deploy retry loop) — same isolation every other run_server # test in this file uses. - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), @@ -1602,9 +1464,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, patch( "litellm.proxy.proxy_cli.ProxyInitializationHelpers.is_port_in_use", return_value=False, @@ -1616,9 +1476,7 @@ class TestProxyInitializationHelpers: "port": 8000, } - result = runner.invoke( - run_server, ["--local", "--timeout_worker_healthcheck", "15"] - ) + result = runner.invoke(run_server, ["--local", "--timeout_worker_healthcheck", "15"]) assert result.exit_code == 0 mock_get_args.assert_called_once_with( @@ -1627,15 +1485,12 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=None, timeout_worker_healthcheck=15, - forwarded=None, ) @patch("uvicorn.run") @patch("builtins.print") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - def test_max_requests_before_restart_flag( - self, mock_setup_db, mock_print, mock_uvicorn_run - ): + def test_max_requests_before_restart_flag(self, mock_setup_db, mock_print, mock_uvicorn_run): """Test that the max_requests_before_restart flag is passed to uvicorn as limit_max_requests""" from click.testing import CliRunner @@ -1648,11 +1503,7 @@ class TestProxyInitializationHelpers: mock_key_mgmt = MagicMock() mock_save_worker_config = MagicMock() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict( os.environ, @@ -1670,9 +1521,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -1680,13 +1529,9 @@ class TestProxyInitializationHelpers: "port": 8000, } - result = runner.invoke( - run_server, ["--local", "--max_requests_before_restart", "123"] - ) + result = runner.invoke(run_server, ["--local", "--max_requests_before_restart", "123"]) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() # Check that uvicorn.run was called with limit_max_requests parameter @@ -1696,9 +1541,7 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("builtins.print") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - def test_max_requests_before_restart_jitter_flag( - self, mock_setup_db, mock_print, mock_uvicorn_run - ): + def test_max_requests_before_restart_jitter_flag(self, mock_setup_db, mock_print, mock_uvicorn_run): """--max_requests_before_restart_jitter maps to uvicorn limit_max_requests_jitter""" from click.testing import CliRunner @@ -1709,11 +1552,7 @@ class TestProxyInitializationHelpers: pass runner = CliRunner() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch("uvicorn.Config", _NewUvicornConfig), @@ -1728,9 +1567,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -1749,9 +1586,7 @@ class TestProxyInitializationHelpers: ], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() call_args = mock_uvicorn_run.call_args assert call_args[1]["limit_max_requests"] == 1000 @@ -1770,11 +1605,7 @@ class TestProxyInitializationHelpers: from litellm.proxy.proxy_cli import run_server runner = CliRunner() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -1788,9 +1619,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -1810,9 +1639,7 @@ class TestProxyInitializationHelpers: ], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_not_called() mock_run_gunicorn.assert_called_once() g_kwargs = mock_run_gunicorn.call_args[1] @@ -1858,20 +1685,14 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - def test_single_worker_max_requests_warning_printed_on_uvicorn( - self, mock_setup_db, mock_uvicorn_run - ): + def test_single_worker_max_requests_warning_printed_on_uvicorn(self, mock_setup_db, mock_uvicorn_run): """uvicorn single-worker startup prints the does-not-recycle warning""" from click.testing import CliRunner from litellm.proxy.proxy_cli import run_server runner = CliRunner() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -1885,9 +1706,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -1895,13 +1714,9 @@ class TestProxyInitializationHelpers: "port": 8000, } - result = runner.invoke( - run_server, ["--local", "--max_requests_before_restart", "123"] - ) + result = runner.invoke(run_server, ["--local", "--max_requests_before_restart", "123"]) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" assert "--max_requests_before_restart 123 with --num_workers 1" in result.output assert "uvicorn exits the process" in result.output @@ -1918,11 +1733,7 @@ class TestProxyInitializationHelpers: from litellm.proxy.proxy_cli import run_server runner = CliRunner() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -1936,9 +1747,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -1951,12 +1760,8 @@ class TestProxyInitializationHelpers: ["--local", "--num_workers", "2", "--max_requests_before_restart", "123"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" - assert not any( - "does not recycle a worker" in str(c) for c in mock_print.call_args_list - ) + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" + assert not any("does not recycle a worker" in str(c) for c in mock_print.call_args_list) @patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.run_gunicorn_server") @patch("uvicorn.run") @@ -1970,11 +1775,7 @@ class TestProxyInitializationHelpers: from litellm.proxy.proxy_cli import run_server runner = CliRunner() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -1988,9 +1789,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -2003,9 +1802,7 @@ class TestProxyInitializationHelpers: ["--local", "--run_gunicorn", "--max_requests_before_restart", "123"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" assert "--max_requests_before_restart 123 with --num_workers 1" in result.output assert "gunicorn accepts no connections" in result.output @@ -2022,11 +1819,7 @@ class TestProxyInitializationHelpers: from litellm.proxy.proxy_cli import run_server runner = CliRunner() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -2040,9 +1833,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -2055,13 +1846,9 @@ class TestProxyInitializationHelpers: ["--local", "--run_hypercorn", "--max_requests_before_restart", "5"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_init_hypercorn.assert_called_once() - assert not any( - "does not recycle a worker" in str(c) for c in mock_print.call_args_list - ) + assert not any("does not recycle a worker" in str(c) for c in mock_print.call_args_list) @pytest.mark.skipif(os.name == "nt", reason="gunicorn server path skips Windows") def test_gunicorn_options_include_max_requests_jitter(self): @@ -2262,7 +2049,9 @@ class TestProxyInitializationHelpers: with patch.dict(os.environ, test_env_special): result = construct_database_url_from_env_vars() - expected_url = "postgresql://user%40with%2Bspecial:test-password-special-chars@localhost:5432/db_name%2Ftest" + expected_url = ( + "postgresql://user%40with%2Bspecial:test-password-special-chars@localhost:5432/db_name%2Ftest" + ) assert result == expected_url # Test without password (should still work) @@ -2390,11 +2179,7 @@ class TestQueryEngineReaperWiring: from litellm.proxy.proxy_cli import run_server runner = CliRunner() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -2409,12 +2194,8 @@ class TestQueryEngineReaperWiring: }, ), patch("uvicorn.run") as mock_uvicorn_run, - patch( - "litellm.proxy.proxy_cli.start_query_engine_reaper" - ) as mock_start_reaper, - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.start_query_engine_reaper") as mock_start_reaper, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -2425,17 +2206,13 @@ class TestQueryEngineReaperWiring: return result, mock_uvicorn_run, mock_start_reaper def test_multi_worker_uvicorn_starts_reaper(self): - result, mock_uvicorn_run, mock_start_reaper = self._invoke_run_server( - ["--local", "--num_workers", "2"] - ) + result, mock_uvicorn_run, mock_start_reaper = self._invoke_run_server(["--local", "--num_workers", "2"]) assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() mock_start_reaper.assert_called_once() def test_single_worker_uvicorn_does_not_start_reaper(self): - result, mock_uvicorn_run, mock_start_reaper = self._invoke_run_server( - ["--local", "--num_workers", "1"] - ) + result, mock_uvicorn_run, mock_start_reaper = self._invoke_run_server(["--local", "--num_workers", "1"]) assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() mock_start_reaper.assert_not_called() @@ -2446,9 +2223,7 @@ class TestQueryEngineReaperWiring: with ( patch("gunicorn.app.base.BaseApplication.run"), - patch( - "litellm.proxy.proxy_cli.start_query_engine_reaper" - ) as mock_start_reaper, + patch("litellm.proxy.proxy_cli.start_query_engine_reaper") as mock_start_reaper, ): ProxyInitializationHelpers._run_gunicorn_server( host="127.0.0.1", @@ -2496,11 +2271,7 @@ class TestRunServerDbSetup: save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" with ( @@ -2512,9 +2283,7 @@ class TestRunServerDbSetup: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -2545,10 +2314,18 @@ class TestRunServerDbSetup: mock_setup_database.assert_called_with(use_migrate=False, use_v2_resolver=True) @patch("atexit.register") - @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above - @patch("litellm.proxy.db.prisma_client.PrismaManager.build_request_log_indexes") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above - @patch("litellm.proxy.db.check_migration.check_prisma_schema_diff") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above - @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + @patch( + "litellm.proxy.db.prisma_client.PrismaManager.setup_database" + ) # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + @patch( + "litellm.proxy.db.prisma_client.PrismaManager.build_request_log_indexes" + ) # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + @patch( + "litellm.proxy.db.check_migration.check_prisma_schema_diff" + ) # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + @patch( + "litellm.proxy.db.prisma_client.should_update_prisma_schema" + ) # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above def test_migrations_run_when_the_prisma_cli_is_not_on_path( self, mock_should_update_schema, @@ -2572,11 +2349,7 @@ class TestRunServerDbSetup: save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" clean_env["PATH"] = str(empty_bin) @@ -2736,11 +2509,7 @@ class TestRunServerDbSetup: save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" with ( @@ -2752,9 +2521,7 @@ class TestRunServerDbSetup: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -2797,11 +2564,7 @@ class TestRunServerDbSetup: save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env["DATABASE_URL"] = "sqlite:///data/litellm.db" with ( @@ -2815,9 +2578,7 @@ class TestRunServerDbSetup: ), ): with pytest.raises(SystemExit) as exc_info: - run_server.main( - ["--local", "--skip_server_startup"], standalone_mode=False - ) + run_server.main(["--local", "--skip_server_startup"], standalone_mode=False) assert exc_info.value.code == 1 mock_setup_database.assert_not_called() err: Final = capsys.readouterr().err @@ -2859,11 +2620,7 @@ class TestRunServerDbSetup: save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" clean_env["USE_V2_MIGRATION_RESOLVER"] = "true" @@ -2877,9 +2634,7 @@ class TestRunServerDbSetup: }, ), ): - run_server.main( - ["--local", "--skip_server_startup"], standalone_mode=False - ) + run_server.main(["--local", "--skip_server_startup"], standalone_mode=False) mock_setup_database.assert_called_once_with(use_migrate=True, use_v2_resolver=True) assert "--use_v2_migration_resolver is deprecated" not in capsys.readouterr().out @@ -2922,9 +2677,7 @@ class TestRunServerDbSetup: {"proxy_server": mock_proxy_module, "litellm.proxy.proxy_server": mock_proxy_module}, ), ): - run_server.main( - ["--local", "--skip_server_startup", "--use_v2_migration_resolver"], standalone_mode=False - ) + run_server.main(["--local", "--skip_server_startup", "--use_v2_migration_resolver"], standalone_mode=False) mock_setup_database.assert_called_once_with(use_migrate=True, use_v2_resolver=True) assert "--use_v2_migration_resolver is deprecated and has no effect" in capsys.readouterr().out @@ -3061,12 +2814,7 @@ class TestRunServerDbSetup: def test_resolve_v2_migration_resolver(self, use_legacy_flag, env_value, expected): from litellm.proxy.proxy_cli import resolve_v2_migration_resolver - assert ( - resolve_v2_migration_resolver( - use_legacy_flag=use_legacy_flag, env_value=env_value - ) - is expected - ) + assert resolve_v2_migration_resolver(use_legacy_flag=use_legacy_flag, env_value=env_value) is expected def test_deprecated_v2_flag_not_reported_outside_a_cli_invocation(self): from litellm.proxy.proxy_cli import deprecated_v2_flag_passed_on_cli @@ -3107,9 +2855,7 @@ class TestRunServerDbSetup: ) clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL", "USE_V2_MIGRATION_RESOLVER") + k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL", "USE_V2_MIGRATION_RESOLVER") } clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" @@ -3190,9 +2936,7 @@ class TestRunServerDbSetup: "sys.modules", {"proxy_server": mock_proxy_module, "litellm.proxy.proxy_server": mock_proxy_module}, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, outcome as exc_info, ): mock_get_args.return_value = {"app": "litellm.proxy.proxy_server:app", "host": "localhost", "port": 8000} @@ -3244,11 +2988,7 @@ class TestWorkerStartupHooks: "LITELLM_WORKER_STARTUP_HOOKS": "tests.unit.proxy.test_proxy_cli:_dummy_hook", } # Remove DATABASE_URL to avoid real DB setup - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env.update(env_overrides) with patch.dict(os.environ, clean_env, clear=True): @@ -3271,11 +3011,7 @@ class TestWorkerStartupHooks: env_overrides = { "LITELLM_WORKER_STARTUP_HOOKS": "tests.unit.proxy.test_proxy_cli:_dummy_async_hook", } - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env.update(env_overrides) with patch.dict(os.environ, clean_env, clear=True): @@ -3295,11 +3031,7 @@ class TestWorkerStartupHooks: env_overrides = { "LITELLM_WORKER_STARTUP_HOOKS": "tests.unit.proxy.test_proxy_cli:_failing_hook", } - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env.update(env_overrides) with patch.dict(os.environ, clean_env, clear=True): @@ -3328,18 +3060,11 @@ class TestWorkerStartupHooks: from litellm.proxy.proxy_server import proxy_startup_event - hooks = ( - "tests.unit.proxy.test_proxy_cli:_dummy_hook," - "tests.unit.proxy.test_proxy_cli:_dummy_async_hook" - ) + hooks = "tests.unit.proxy.test_proxy_cli:_dummy_hook,tests.unit.proxy.test_proxy_cli:_dummy_async_hook" env_overrides = { "LITELLM_WORKER_STARTUP_HOOKS": hooks, } - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env.update(env_overrides) with patch.dict(os.environ, clean_env, clear=True): @@ -3485,13 +3210,9 @@ class TestPostgresStatementTimeoutOptions: import yaml config_path = tmp_path / "config.yaml" - config_path.write_text( - yaml.dump({"model_list": [], "general_settings": {"database_statement_timeout": 60}}) - ) + config_path.write_text(yaml.dump({"model_list": [], "general_settings": {"database_statement_timeout": 60}})) - captured = _run_server_and_capture_urls( - str(config_path), direct_url="postgresql://t:t@localhost:5432/t" - ) + captured = _run_server_and_capture_urls(str(config_path), direct_url="postgresql://t:t@localhost:5432/t") assert "options" in urlparse.parse_qs(urlparse.urlparse(captured["DATABASE_URL"]).query) assert "options" not in urlparse.parse_qs(urlparse.urlparse(captured["DIRECT_URL"]).query) @@ -3663,9 +3384,7 @@ class TestReadReplicaConnectionParams: import yaml config_path: Final = tmp_path / "config.yaml" - config_path.write_text( - yaml.dump({"model_list": [], "general_settings": {"database_connection_pool_limit": 3}}) - ) + config_path.write_text(yaml.dump({"model_list": [], "general_settings": {"database_connection_pool_limit": 3}})) _run_server_and_capture_urls( str(config_path), @@ -3685,9 +3404,7 @@ class TestReadReplicaConnectionParams: import yaml config_path: Final = tmp_path / "config.yaml" - config_path.write_text( - yaml.dump({"model_list": [], "general_settings": {"database_connection_pool_limit": 3}}) - ) + config_path.write_text(yaml.dump({"model_list": [], "general_settings": {"database_connection_pool_limit": 3}})) _run_server_and_capture_urls( str(config_path), @@ -3882,9 +3599,7 @@ class TestTokenAuthCliFlags: patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False), patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database"), patch("uvicorn.run"), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -3898,9 +3613,7 @@ class TestTokenAuthCliFlags: return result, database_url, toggle def test_azure_flag_assembles_a_token_bearing_database_url(self): - result, database_url, toggle = self._invoke_with_azure_host( - ["--local", "--azure_postgresql_auth"] - ) + result, database_url, toggle = self._invoke_with_azure_host(["--local", "--azure_postgresql_auth"]) assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" assert database_url is not None @@ -4128,7 +3841,11 @@ def probe_logger() -> Iterator[logging.Logger]: def _log_config_source(kind: str, tmp_path: Path) -> str | dict[str, object]: - config: Final[dict[str, object]] = {"version": 1, "incremental": True, "loggers": {_PROBE_LOGGER: {"level": "ERROR"}}} + config: Final[dict[str, object]] = { + "version": 1, + "incremental": True, + "loggers": {_PROBE_LOGGER: {"level": "ERROR"}}, + } if kind == "dict": return config written: Final = tmp_path / f"log_config.{kind}" @@ -4457,3 +4174,72 @@ def test_migration_data_check_requires_database_url(monkeypatch): with pytest.raises(click.UsageError, match="requires DATABASE_URL"): ProxyInitializationHelpers.run_migration_data_check(use_migrate=True) + + +class TestRustForwardServer: + """The public listener ``run_server`` reports to the Rust supervisor matches what it would serve itself.""" + + @staticmethod + def _server(argv: list[str]): + params: Final = run_server.make_context("litellm", ["--port", "45999", *argv]).params + return params, ProxyInitializationHelpers.python_server( + str(params["host"]), + int(str(params["port"])), + params["keepalive_timeout"], + run_gunicorn=params["run_gunicorn"] is True, + run_hypercorn=params["run_hypercorn"] is True, + run_granian=params["run_granian"] is True, + tls=any((params["ssl_keyfile_path"], params["ssl_certfile_path"], params["ciphers"])), + ) + + @pytest.mark.parametrize( + ("argv", "reason"), + [ + (["--ssl_keyfile_path", "unused"], "does not support TLS"), + (["--ssl_certfile_path", "unused"], "does not support TLS"), + (["--ciphers", "unused"], "does not support TLS"), + (["--run_hypercorn"], "does not support Hypercorn"), + (["--run_granian"], "does not support Granian"), + ], + ) + def test_unsupported_servers_fail_before_starting_children( + self, argv: list[str], reason: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + from click.testing import CliRunner + + monkeypatch.setenv("LITELLM_RUST", "true") + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + result: Final = CliRunner().invoke(run_server, argv) + assert result.exit_code == 1 + assert f"LITELLM_RUST {reason}" in result.output + + @pytest.mark.parametrize( + ("argv", "environ"), + [(["--keepalive_timeout", "2"], {}), ([], {"KEEPALIVE_TIMEOUT": "7"}), ([], {})], + ) + def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with( + self, argv: list[str], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + for name, value in environ.items(): + monkeypatch.setenv(name, value) + params, server = self._server(argv) + served: Final = uvicorn.Config( + **ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 45555, keepalive_timeout=params["keepalive_timeout"] + ) + ) + assert server.keepalive_seconds == served.timeout_keep_alive + + def test_gunicorn_workers_keep_gunicorns_keepalive_and_rust_follows(self, monkeypatch: pytest.MonkeyPatch) -> None: + from gunicorn.config import Config as GunicornConfig + + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + _, server = self._server(["--run_gunicorn", "--keepalive_timeout", "9"]) + assert server.keepalive_seconds == GunicornConfig().keepalive + + def test_rust_drains_for_the_graceful_timeout_uvicorn_runs_with(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "12.5") + _, server = self._server([]) + served: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555) + assert server.graceful_timeout_seconds == served["timeout_graceful_shutdown"]