From 608a5c3e575bcdb2dd6128acc45e3dd1354c75e3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:11:18 +0000 Subject: [PATCH 01/15] feat(proxy): supervise the opt-in Rust forwarding server --- litellm/proxy/proxy_cli.py | 35 +++++++- litellm/proxy/rust_gateway.py | 115 ++++++++++++++++++++++++++ tests/unit/proxy/test_rust_gateway.py | 62 ++++++++++++++ 3 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 litellm/proxy/rust_gateway.py create mode 100644 tests/unit/proxy/test_rust_gateway.py diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 0fbfa0b06a7..94878bd1af2 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -419,6 +419,7 @@ class ProxyInitializationHelpers: import litellm from litellm._logging import _get_uvicorn_json_log_config, resolve_log_level + from litellm.proxy.rust_gateway import inherited_fd from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager uvicorn_args: Final = { @@ -427,6 +428,7 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), + **({"fd": inherited_fd()} if inherited_fd() is not None else {}), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -752,7 +754,9 @@ class ProxyInitializationHelpers: ) graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout() gunicorn_options: Final = { - "bind": f"{host}:{port}", + "bind": ( + f"fd://{os.environ['_LITELLM_GATEWAY_FD']}" if "_LITELLM_GATEWAY_FD" in os.environ else f"{host}:{port}" + ), "workers": num_workers, # default is 1 "worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, @@ -1039,6 +1043,13 @@ class ProxyInitializationHelpers: is_flag=True, help="Starts proxy via gunicorn, instead of uvicorn (better for managing multiple workers)", ) +@click.option( + "--rust_gateway", + envvar="LITELLM_RUST_GATEWAY", + default=False, + is_flag=True, + help="Run the Rust forwarding server on the public port, with Python on loopback", +) @click.option( "--run_hypercorn", default=False, @@ -1241,6 +1252,7 @@ def run_server( setup, version, run_gunicorn, + rust_gateway, run_hypercorn, run_granian, ssl_keyfile_path, @@ -1280,6 +1292,27 @@ def run_server( run_setup_wizard() return + if rust_gateway and not any( + (version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data) + ): + from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise + + binary: Final = gateway_binary() + error: Final = startup_error( + binary, + { + "--ssl_keyfile_path": ssl_keyfile_path, + "--ssl_certfile_path": ssl_certfile_path, + "--ciphers": ciphers, + "--run_hypercorn": run_hypercorn, + "--run_granian": run_granian, + }, + ) + if error is not None: + raise click.ClickException(error) + assert binary is not None + raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, port)) + args: Final = locals() try: from litellm.proxy.proxy_server import ( diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py new file mode 100644 index 00000000000..830a4ab3d3d --- /dev/null +++ b/litellm/proxy/rust_gateway.py @@ -0,0 +1,115 @@ +import asyncio +import os +import shutil +import signal +import socket +import sys +import sysconfig +from collections.abc import Mapping, Sequence +from typing import Final + +from pydantic import TypeAdapter + +GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" + + +def gateway_binary() -> str | None: + name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway" + return shutil.which(name, path=sysconfig.get_path("scripts")) or shutil.which(name) + + +def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: + flag: Final = next((name for name, value in unsupported.items() if value), None) + if flag is not None: + return f"--rust_gateway does not support {flag}" + if os.name != "posix": + return "--rust_gateway requires POSIX inherited sockets and process signals" + if binary is None: + return "--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment" + return None + + +def child_argv(argv: Sequence[str], port: int) -> tuple[str, ...]: + return (*(arg for arg in argv if arg != "--rust_gateway"), "--host", "127.0.0.1", "--port", str(port)) + + +def inherited_fd() -> int | None: + value: Final = os.environ.get(GATEWAY_FD) + return int(value) if value is not None else None + + +def terminate_group(process: asyncio.subprocess.Process) -> None: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + + +async def _supervise(binary: str, command: Sequence[str], host: str, port: int) -> int: + loop: Final = asyncio.get_running_loop() + stopped: Final[asyncio.Future[signal.Signals]] = loop.create_future() + + def stop(signum: signal.Signals) -> None: + if not stopped.done(): + stopped.set_result(signum) + + for signum in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(signum, stop, signum) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen() + _, upstream_port = TypeAdapter(tuple[str, int]).validate_python(listener.getsockname()) + python: Final = await asyncio.create_subprocess_exec( + *child_argv(command, upstream_port), + env={ + **os.environ, + "LITELLM_RUST_GATEWAY": "false", + GATEWAY_FD: str(listener.fileno()), + "FORWARDED_ALLOW_IPS": "127.0.0.1", + }, + pass_fds=(listener.fileno(),), + start_new_session=True, + ) + try: + rust: Final = await asyncio.create_subprocess_exec( + binary, + env={ + **os.environ, + "HOST": host, + "PORT": str(port), + "LITELLM_FORWARD_UPSTREAM": f"http://127.0.0.1:{upstream_port}", + }, + start_new_session=True, + ) + except OSError: + terminate_group(python) + await python.wait() + raise + python_done: Final = asyncio.create_task(python.wait()) + rust_done: Final = asyncio.create_task(rust.wait()) + try: + done, _ = await asyncio.wait((stopped, python_done, rust_done), return_when=asyncio.FIRST_COMPLETED) + if python_done in done or rust_done in done: + return 1 + rust.send_signal(stopped.result()) + draining, _ = await asyncio.wait((python_done, rust_done), return_when=asyncio.FIRST_COMPLETED) + if python_done in draining or rust_done.result() != 0: + return 1 + python.send_signal(stopped.result()) + return 0 if await python_done == 0 else 1 + finally: + terminate_group(rust) + terminate_group(python) + await asyncio.gather(python_done, rust_done) + + +def supervise(binary: str, command: Sequence[str], host: str, port: int) -> int: + try: + return asyncio.run(_supervise(binary, command, host, port)) + except OSError as error: + sys.stderr.write(f"Rust gateway startup failed: {error}\n") + return 1 + + +def cli_command(argv: Sequence[str]) -> tuple[str, ...]: + return (sys.executable, "-m", "litellm.proxy.proxy_cli", *argv) diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py new file mode 100644 index 00000000000..c9992d07d14 --- /dev/null +++ b/tests/unit/proxy/test_rust_gateway.py @@ -0,0 +1,62 @@ +import sys +from pathlib import Path +from typing import Final + +import pytest +from click.testing import CliRunner + +from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server +from litellm.proxy.rust_gateway import child_argv, cli_command, gateway_binary, inherited_fd, startup_error + + +@pytest.mark.parametrize( + "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] +) +@pytest.mark.parametrize("enable", ["flag", "environment"]) +def test_unsupported_modes_fail_before_starting_children( + flag: str, enable: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("LITELLM_RUST_GATEWAY", "true" if enable == "environment" else "false") + arguments: Final = ([] if enable == "environment" else ["--rust_gateway"]) + [flag] + values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] + result: Final = CliRunner().invoke(run_server, arguments + values) + assert result.exit_code != 0 + assert f"--rust_gateway does not support {flag}" in result.output + + +def test_child_arguments_preserve_options_and_force_private_listener() -> None: + arguments: Final = ("--rust_gateway", "--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") + params: Final = run_server.make_context("litellm", list(child_argv(arguments, 45555))).params + assert params["host"] == "127.0.0.1" + assert params["port"] == 45555 + assert params["num_workers"] == 2 + assert params["reload"] is True + assert params["rust_gateway"] is False + assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli") + + +def test_missing_binary_gives_install_instruction() -> None: + assert startup_error(None, {}) == ( + "--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment" + ) + + +def test_current_environment_binary_wins_over_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + binary: Final = tmp_path / "litellm-gateway" + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o755) + monkeypatch.setattr("sysconfig.get_path", lambda _: str(tmp_path)) + monkeypatch.setenv("PATH", "") + assert gateway_binary() == str(binary) + + +@pytest.mark.parametrize("fd", [None, "17"]) +def test_inherited_socket_reaches_uvicorn(fd: str | None, monkeypatch: pytest.MonkeyPatch) -> None: + if fd is None: + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + else: + monkeypatch.setenv("_LITELLM_GATEWAY_FD", fd) + expected: Final = int(fd) if fd is not None else None + options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555) + assert inherited_fd() == expected + assert options.get("fd") == expected From c0ec3485833ea0bed5c82ea2784ccff69933eba4 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:39:14 +0000 Subject: [PATCH 02/15] fix(proxy): preserve gateway metrics and default-port fallback --- litellm/proxy/proxy_cli.py | 25 +++--- litellm/proxy/rust_gateway.py | 5 ++ .../configuration/test_rust_gateway.py | 89 +++++++++++++++++++ tests/unit/proxy/test_rust_gateway.py | 22 ++++- 4 files changed, 130 insertions(+), 11 deletions(-) create mode 100644 tests/integration/configuration/test_rust_gateway.py diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 94878bd1af2..203358cba97 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -819,7 +819,7 @@ class ProxyInitializationHelpers: _run_ollama_serve = run_ollama_serve @staticmethod - def is_port_in_use(port): + def is_port_in_use(port: int): import socket with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: @@ -1221,8 +1221,8 @@ class ProxyInitializationHelpers: ) def run_server( cli_args, - host, - port, + host: str, + port: int, api_base, api_version, model, @@ -1252,7 +1252,7 @@ def run_server( setup, version, run_gunicorn, - rust_gateway, + rust_gateway: bool, run_hypercorn, run_granian, ssl_keyfile_path, @@ -1298,7 +1298,7 @@ def run_server( from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise binary: Final = gateway_binary() - error: Final = startup_error( + gateway_error: Final = startup_error( binary, { "--ssl_keyfile_path": ssl_keyfile_path, @@ -1308,10 +1308,13 @@ def run_server( "--run_granian": run_granian, }, ) - if error is not None: - raise click.ClickException(error) + if gateway_error is not None: + raise click.ClickException(gateway_error) assert binary is not None - raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, port)) + public_port: Final = ( + random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port + ) + raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port)) args: Final = locals() try: @@ -1761,15 +1764,17 @@ def run_server( if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None: from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process + from litellm.proxy.rust_gateway import metrics_host + metrics_bind_host: Final = metrics_host(host) try: metrics_process: Final = start_metrics_server_process( - host=host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir + host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir ) except MetricsServerStartupError as error: raise click.ClickException(str(error)) from error print_startup_message( - f"\033[1;32mLiteLLM: Serving Prometheus metrics on {host}:{prometheus_metrics_port}/metrics " + f"\033[1;32mLiteLLM: Serving Prometheus metrics on {metrics_bind_host}:{prometheus_metrics_port}/metrics " f"(pid {metrics_process.pid})\033[0m" ) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 830a4ab3d3d..9b7ba98c161 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -38,6 +38,10 @@ def inherited_fd() -> int | None: return int(value) if value is not None else None +def metrics_host(host: str) -> str: + return os.environ.get("_LITELLM_GATEWAY_METRICS_HOST", host) if inherited_fd() is not None else host + + def terminate_group(process: asyncio.subprocess.Process) -> None: try: os.killpg(process.pid, signal.SIGTERM) @@ -65,6 +69,7 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int) **os.environ, "LITELLM_RUST_GATEWAY": "false", GATEWAY_FD: str(listener.fileno()), + "_LITELLM_GATEWAY_METRICS_HOST": host, "FORWARDED_ALLOW_IPS": "127.0.0.1", }, pass_fds=(listener.fileno(),), diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py new file mode 100644 index 00000000000..bb0119f9398 --- /dev/null +++ b/tests/integration/configuration/test_rust_gateway.py @@ -0,0 +1,89 @@ +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Final + +import httpx +import pytest + +from tests.integration._support.client import Gateway, eventually +from tests.integration._support.ports import allocate_port +from tests.integration._support.process import stop_root_process + + +def ready(client: httpx.Client) -> bool: + try: + return client.get("/health/liveliness").status_code == 200 + except httpx.TransportError: + return False + + +@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) +def test_gateway_preserves_metrics_bind_and_mcp_client_ip( + gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...] +) -> None: + config: Final = tmp_path / "config.yaml" + config.write_text( + json.dumps( + { + "model_list": [], + "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY", "use_x_forwarded_for": False}, + } + ) + ) + metrics_port: Final = allocate_port() + public_port: Final = allocate_port() + environment: Final = { + **{key: value for key, value in os.environ.items() if key not in ("DATABASE_URL", "DATABASE_URL_READ_REPLICA")}, + "LITELLM_MASTER_KEY": gateway.key, + "STORE_MODEL_IN_DB": "False", + "PYTHON_DOTENV_DISABLED": "1", + } + with (tmp_path / "proxy.log").open("wb") as output: + with subprocess.Popen( + ( + sys.executable, + "-I", + "-m", + "litellm.proxy.proxy_cli", + "--rust_gateway", + "--config", + str(config), + "--host", + "0.0.0.0", + "--port", + str(public_port), + "--num_workers", + "2", + "--timeout_worker_healthcheck", + "120", + "--prometheus_metrics_port", + str(metrics_port), + *server_arguments, + ), + env=environment, + stdout=output, + stderr=output, + ) as supervisor: + try: + with httpx.Client( + base_url=f"http://127.0.0.1:{public_port}", + transport=httpx.HTTPTransport(local_address="127.0.0.2"), + trust_env=False, + follow_redirects=True, + timeout=5, + ) as external: + assert eventually(lambda: ready(external), bool, seconds=180), (tmp_path / "proxy.log").read_text() + response: Final = external.get(f"http://127.0.0.2:{metrics_port}/metrics") + assert response.status_code == 200, response.text + client_ip: Final = external.get( + "/v1/mcp/network/client-ip", + headers={"Authorization": f"Bearer {gateway.key}", "X-Forwarded-For": "1.2.3.4"}, + ) + assert client_ip.status_code == 200, client_ip.text + assert client_ip.json() == {"ip": "127.0.0.2"} + finally: + assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text() + assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text() diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index c9992d07d14..c1d03d696b9 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -6,7 +6,14 @@ import pytest from click.testing import CliRunner from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server -from litellm.proxy.rust_gateway import child_argv, cli_command, gateway_binary, inherited_fd, startup_error +from litellm.proxy.rust_gateway import ( + child_argv, + cli_command, + gateway_binary, + inherited_fd, + metrics_host, + startup_error, +) @pytest.mark.parametrize( @@ -41,6 +48,19 @@ def test_missing_binary_gives_install_instruction() -> None: ) +@pytest.mark.parametrize("supervised", [False, True]) +@pytest.mark.parametrize("host", ["0.0.0.0", "192.0.2.1", "::"]) +def test_metrics_keep_configured_host_separate_from_private_listener( + supervised: bool, host: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", host) + if supervised: + monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17") + else: + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + assert metrics_host("127.0.0.1") == (host if supervised else "127.0.0.1") + + def test_current_environment_binary_wins_over_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: binary: Final = tmp_path / "litellm-gateway" binary.write_text("#!/bin/sh\nexit 0\n") From 85e87e25e20fb6a8a7b1d7ea631cedf503ad8628 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:49:20 +0000 Subject: [PATCH 03/15] test(proxy): cover gateway shutdown and child failures --- tests/unit/proxy/test_rust_gateway.py | 96 +++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index c1d03d696b9..969cfd12da4 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -1,6 +1,10 @@ +import asyncio +import signal +import socket import sys from pathlib import Path from typing import Final +from unittest.mock import MagicMock, Mock, call, patch import pytest from click.testing import CliRunner @@ -13,6 +17,7 @@ from litellm.proxy.rust_gateway import ( inherited_fd, metrics_host, startup_error, + supervise, ) @@ -80,3 +85,94 @@ def test_inherited_socket_reaches_uvicorn(fd: str | None, monkeypatch: pytest.Mo options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555) assert inherited_fd() == expected assert options.get("fd") == expected + + +@pytest.mark.parametrize( + ("event", "python_exit", "rust_exit", "expected"), + [ + ("signal", 0, 0, 0), + ("signal", 1, 0, 1), + ("signal", 0, 1, 1), + ("python-exit", 0, 0, 1), + ("rust-exit", 0, 0, 1), + ("python-during-drain", 1, 0, 1), + ("spawn-error", 0, 0, 1), + ], +) +def test_supervisor_drains_before_stopping_python_and_reaps_both_children( + event: str, python_exit: int, rust_exit: int, expected: int, capsys: pytest.CaptureFixture[str] +) -> None: + loop: Final = asyncio.new_event_loop() + handlers: Final = Mock() + listener: Final = MagicMock(spec=socket.socket) + listener.__enter__.return_value = listener + listener.getsockname.return_value = ("127.0.0.1", 45555) + listener.fileno.return_value = 17 + children: Final = {} + sent: Final = Mock() + + def current_loop(): + return Mock(wraps=loop, add_signal_handler=handlers) + + async def spawn(*argv, **kwargs): + done: Final = asyncio.events.get_running_loop().create_future() + pid: Final = 101 if argv[0] == "python" else 102 + + async def wait() -> int: + return await done + + def send_signal(signum: signal.Signals) -> None: + sent(pid, signum) + if pid == 102 and event == "python-during-drain": + children[101][1].set_result(python_exit) + return + if pid == 101: + assert children[102][1].done(), "Python must remain alive until Rust has drained" + done.set_result(python_exit if pid == 101 else rust_exit) + + process: Final = Mock(spec=asyncio.subprocess.Process, pid=pid, wait=wait, send_signal=send_signal) + if pid == 102 and event == "spawn-error": + raise OSError("gateway cannot start") + children[pid] = (process, done) + if pid == 102: + if event == "python-exit": + children[101][1].set_result(python_exit) + elif event == "rust-exit": + done.set_result(rust_exit) + else: + _, stop, signum = handlers.call_args_list[0].args + stop(signum) + stop(signum) + return process + + def kill_group(pid: int, signum: signal.Signals) -> None: + assert signum == signal.SIGTERM + _, done = children[pid] + if done.done(): + raise ProcessLookupError + done.set_result(1) + + with ( + patch("asyncio.events.new_event_loop", return_value=loop), + patch("asyncio.get_running_loop", side_effect=current_loop), + patch("asyncio.create_subprocess_exec", side_effect=spawn) as launch, + patch("socket.socket", return_value=listener), + patch("os.killpg", side_effect=kill_group) as kill, + ): + assert supervise("gateway", ("python", "--rust_gateway"), "0.0.0.0", 4000) == expected + + assert all(done.done() for _, done in children.values()) + assert call(101, signal.SIGTERM) in kill.call_args_list + python_call, rust_call = launch.call_args_list + assert python_call.args == ("python", "--host", "127.0.0.1", "--port", "45555") + assert python_call.kwargs["pass_fds"] == (17,) + assert python_call.kwargs["env"]["LITELLM_RUST_GATEWAY"] == "false" + assert python_call.kwargs["env"]["FORWARDED_ALLOW_IPS"] == "127.0.0.1" + assert rust_call.kwargs["env"]["LITELLM_FORWARD_UPSTREAM"] == "http://127.0.0.1:45555" + assert rust_call.kwargs["env"]["PORT"] == "4000" + if event == "spawn-error": + assert "Rust gateway startup failed: gateway cannot start" in capsys.readouterr().err + elif event == "signal" and rust_exit == 0: + assert sent.call_args_list == [call(102, signal.SIGTERM), call(101, signal.SIGTERM)] + else: + assert call(101, signal.SIGTERM) not in sent.call_args_list From 21b2087981ca6aabb384309e0a456b52e1746462 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 02:43:32 +0000 Subject: [PATCH 04/15] fix(proxy): retain configured trusted proxies behind Rust --- litellm/proxy/rust_gateway.py | 9 ++++++++- .../configuration/test_rust_gateway.py | 8 +++++--- tests/unit/proxy/test_rust_gateway.py | 19 ++++++++++++++++++- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 9b7ba98c161..85a850978e8 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -18,6 +18,13 @@ def gateway_binary() -> str | None: return shutil.which(name, path=sysconfig.get_path("scripts")) or shutil.which(name) +def forwarded_allow_ips(trusted: str | None) -> str: + if trusted == "*": + return trusted + hosts: Final = tuple(host.strip() for host in (trusted or "").split(",") if host.strip()) + return ",".join(dict.fromkeys(("127.0.0.1", *hosts))) + + def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: flag: Final = next((name for name, value in unsupported.items() if value), None) if flag is not None: @@ -70,7 +77,7 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int) "LITELLM_RUST_GATEWAY": "false", GATEWAY_FD: str(listener.fileno()), "_LITELLM_GATEWAY_METRICS_HOST": host, - "FORWARDED_ALLOW_IPS": "127.0.0.1", + "FORWARDED_ALLOW_IPS": forwarded_allow_ips(os.environ.get("FORWARDED_ALLOW_IPS")), }, pass_fds=(listener.fileno(),), start_new_session=True, diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index bb0119f9398..aa5ce9825ae 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -21,8 +21,9 @@ def ready(client: httpx.Client) -> bool: @pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) +@pytest.mark.parametrize(("trusted_proxies", "expected_ip"), [("", "127.0.0.2"), ("127.0.0.2", "198.51.100.10")]) def test_gateway_preserves_metrics_bind_and_mcp_client_ip( - gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...] + gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...], trusted_proxies: str, expected_ip: str ) -> None: config: Final = tmp_path / "config.yaml" config.write_text( @@ -40,6 +41,7 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( "LITELLM_MASTER_KEY": gateway.key, "STORE_MODEL_IN_DB": "False", "PYTHON_DOTENV_DISABLED": "1", + "FORWARDED_ALLOW_IPS": trusted_proxies, } with (tmp_path / "proxy.log").open("wb") as output: with subprocess.Popen( @@ -80,10 +82,10 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( assert response.status_code == 200, response.text client_ip: Final = external.get( "/v1/mcp/network/client-ip", - headers={"Authorization": f"Bearer {gateway.key}", "X-Forwarded-For": "1.2.3.4"}, + headers={"Authorization": f"Bearer {gateway.key}", "X-Forwarded-For": "198.51.100.10"}, ) assert client_ip.status_code == 200, client_ip.text - assert client_ip.json() == {"ip": "127.0.0.2"} + assert client_ip.json() == {"ip": expected_ip} finally: assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text() assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text() diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index 969cfd12da4..44771ff9616 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -13,6 +13,7 @@ from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server from litellm.proxy.rust_gateway import ( child_argv, cli_command, + forwarded_allow_ips, gateway_binary, inherited_fd, metrics_host, @@ -53,6 +54,21 @@ def test_missing_binary_gives_install_instruction() -> None: ) +@pytest.mark.parametrize( + ("trusted", "expected"), + [ + (None, "127.0.0.1"), + ("", "127.0.0.1"), + ("192.0.2.1", "127.0.0.1,192.0.2.1"), + ("127.0.0.1, 192.0.2.0/24", "127.0.0.1,192.0.2.0/24"), + ("*", "*"), + ("127.0.0.1,*", "127.0.0.1,*"), + ], +) +def test_private_hop_preserves_configured_trusted_proxies(trusted: str | None, expected: str) -> None: + assert forwarded_allow_ips(trusted) == expected + + @pytest.mark.parametrize("supervised", [False, True]) @pytest.mark.parametrize("host", ["0.0.0.0", "192.0.2.1", "::"]) def test_metrics_keep_configured_host_separate_from_private_listener( @@ -158,6 +174,7 @@ def test_supervisor_drains_before_stopping_python_and_reaps_both_children( patch("asyncio.create_subprocess_exec", side_effect=spawn) as launch, patch("socket.socket", return_value=listener), patch("os.killpg", side_effect=kill_group) as kill, + patch.dict("os.environ", {"FORWARDED_ALLOW_IPS": "192.0.2.1"}), ): assert supervise("gateway", ("python", "--rust_gateway"), "0.0.0.0", 4000) == expected @@ -167,7 +184,7 @@ def test_supervisor_drains_before_stopping_python_and_reaps_both_children( assert python_call.args == ("python", "--host", "127.0.0.1", "--port", "45555") assert python_call.kwargs["pass_fds"] == (17,) assert python_call.kwargs["env"]["LITELLM_RUST_GATEWAY"] == "false" - assert python_call.kwargs["env"]["FORWARDED_ALLOW_IPS"] == "127.0.0.1" + assert python_call.kwargs["env"]["FORWARDED_ALLOW_IPS"] == "127.0.0.1,192.0.2.1" assert rust_call.kwargs["env"]["LITELLM_FORWARD_UPSTREAM"] == "http://127.0.0.1:45555" assert rust_call.kwargs["env"]["PORT"] == "4000" if event == "spawn-error": From 48997c340cc0d177ff385436499e5923c5c7770e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 02:49:29 +0000 Subject: [PATCH 05/15] fix(proxy): enforce specified loopback-only forwarding trust --- litellm/proxy/rust_gateway.py | 9 +-------- .../configuration/test_rust_gateway.py | 8 +++----- tests/unit/proxy/test_rust_gateway.py | 19 +------------------ 3 files changed, 5 insertions(+), 31 deletions(-) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 85a850978e8..9b7ba98c161 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -18,13 +18,6 @@ def gateway_binary() -> str | None: return shutil.which(name, path=sysconfig.get_path("scripts")) or shutil.which(name) -def forwarded_allow_ips(trusted: str | None) -> str: - if trusted == "*": - return trusted - hosts: Final = tuple(host.strip() for host in (trusted or "").split(",") if host.strip()) - return ",".join(dict.fromkeys(("127.0.0.1", *hosts))) - - def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: flag: Final = next((name for name, value in unsupported.items() if value), None) if flag is not None: @@ -77,7 +70,7 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int) "LITELLM_RUST_GATEWAY": "false", GATEWAY_FD: str(listener.fileno()), "_LITELLM_GATEWAY_METRICS_HOST": host, - "FORWARDED_ALLOW_IPS": forwarded_allow_ips(os.environ.get("FORWARDED_ALLOW_IPS")), + "FORWARDED_ALLOW_IPS": "127.0.0.1", }, pass_fds=(listener.fileno(),), start_new_session=True, diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index aa5ce9825ae..bb0119f9398 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -21,9 +21,8 @@ def ready(client: httpx.Client) -> bool: @pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) -@pytest.mark.parametrize(("trusted_proxies", "expected_ip"), [("", "127.0.0.2"), ("127.0.0.2", "198.51.100.10")]) def test_gateway_preserves_metrics_bind_and_mcp_client_ip( - gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...], trusted_proxies: str, expected_ip: str + gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...] ) -> None: config: Final = tmp_path / "config.yaml" config.write_text( @@ -41,7 +40,6 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( "LITELLM_MASTER_KEY": gateway.key, "STORE_MODEL_IN_DB": "False", "PYTHON_DOTENV_DISABLED": "1", - "FORWARDED_ALLOW_IPS": trusted_proxies, } with (tmp_path / "proxy.log").open("wb") as output: with subprocess.Popen( @@ -82,10 +80,10 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( assert response.status_code == 200, response.text client_ip: Final = external.get( "/v1/mcp/network/client-ip", - headers={"Authorization": f"Bearer {gateway.key}", "X-Forwarded-For": "198.51.100.10"}, + headers={"Authorization": f"Bearer {gateway.key}", "X-Forwarded-For": "1.2.3.4"}, ) assert client_ip.status_code == 200, client_ip.text - assert client_ip.json() == {"ip": expected_ip} + assert client_ip.json() == {"ip": "127.0.0.2"} finally: assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text() assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text() diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index 44771ff9616..969cfd12da4 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -13,7 +13,6 @@ from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server from litellm.proxy.rust_gateway import ( child_argv, cli_command, - forwarded_allow_ips, gateway_binary, inherited_fd, metrics_host, @@ -54,21 +53,6 @@ def test_missing_binary_gives_install_instruction() -> None: ) -@pytest.mark.parametrize( - ("trusted", "expected"), - [ - (None, "127.0.0.1"), - ("", "127.0.0.1"), - ("192.0.2.1", "127.0.0.1,192.0.2.1"), - ("127.0.0.1, 192.0.2.0/24", "127.0.0.1,192.0.2.0/24"), - ("*", "*"), - ("127.0.0.1,*", "127.0.0.1,*"), - ], -) -def test_private_hop_preserves_configured_trusted_proxies(trusted: str | None, expected: str) -> None: - assert forwarded_allow_ips(trusted) == expected - - @pytest.mark.parametrize("supervised", [False, True]) @pytest.mark.parametrize("host", ["0.0.0.0", "192.0.2.1", "::"]) def test_metrics_keep_configured_host_separate_from_private_listener( @@ -174,7 +158,6 @@ def test_supervisor_drains_before_stopping_python_and_reaps_both_children( patch("asyncio.create_subprocess_exec", side_effect=spawn) as launch, patch("socket.socket", return_value=listener), patch("os.killpg", side_effect=kill_group) as kill, - patch.dict("os.environ", {"FORWARDED_ALLOW_IPS": "192.0.2.1"}), ): assert supervise("gateway", ("python", "--rust_gateway"), "0.0.0.0", 4000) == expected @@ -184,7 +167,7 @@ def test_supervisor_drains_before_stopping_python_and_reaps_both_children( assert python_call.args == ("python", "--host", "127.0.0.1", "--port", "45555") assert python_call.kwargs["pass_fds"] == (17,) assert python_call.kwargs["env"]["LITELLM_RUST_GATEWAY"] == "false" - assert python_call.kwargs["env"]["FORWARDED_ALLOW_IPS"] == "127.0.0.1,192.0.2.1" + assert python_call.kwargs["env"]["FORWARDED_ALLOW_IPS"] == "127.0.0.1" assert rust_call.kwargs["env"]["LITELLM_FORWARD_UPSTREAM"] == "http://127.0.0.1:45555" assert rust_call.kwargs["env"]["PORT"] == "4000" if event == "spawn-error": From d0e1bebce0e0ff502e5743cdac7a60b82bc59e07 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 02:51:32 +0000 Subject: [PATCH 06/15] fix(proxy): reject incompatible forwarding trust configuration --- litellm/proxy/rust_gateway.py | 6 ++++++ tests/unit/proxy/test_rust_gateway.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 9b7ba98c161..cd02be26c77 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -18,6 +18,10 @@ def gateway_binary() -> str | None: return shutil.which(name, path=sysconfig.get_path("scripts")) or shutil.which(name) +def unsupported_proxy_trust(value: str | None) -> bool: + return any(host.strip() not in ("", "127.0.0.1") for host in (value or "").split(",")) + + def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: flag: Final = next((name for name, value in unsupported.items() if value), None) if flag is not None: @@ -26,6 +30,8 @@ def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str return "--rust_gateway requires POSIX inherited sockets and process signals" if binary is None: return "--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment" + if unsupported_proxy_trust(os.environ.get("FORWARDED_ALLOW_IPS")): + return "--rust_gateway does not support non-loopback FORWARDED_ALLOW_IPS; disable Rust forwarding for this deployment" return None diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index 969cfd12da4..27b2e9b54e9 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -53,6 +53,20 @@ def test_missing_binary_gives_install_instruction() -> None: ) +@pytest.mark.parametrize("trusted", [None, "", "127.0.0.1", "192.0.2.1", "*", "127.0.0.1,192.0.2.1"]) +def test_configured_proxy_trust_is_not_silently_discarded(trusted: str | None, monkeypatch: pytest.MonkeyPatch) -> None: + if trusted is None: + monkeypatch.delenv("FORWARDED_ALLOW_IPS", raising=False) + else: + monkeypatch.setenv("FORWARDED_ALLOW_IPS", trusted) + expected: Final = ( + None + if trusted in (None, "", "127.0.0.1") + else "--rust_gateway does not support non-loopback FORWARDED_ALLOW_IPS; disable Rust forwarding for this deployment" + ) + assert startup_error("gateway", {}) == expected + + @pytest.mark.parametrize("supervised", [False, True]) @pytest.mark.parametrize("host", ["0.0.0.0", "192.0.2.1", "::"]) def test_metrics_keep_configured_host_separate_from_private_listener( @@ -158,6 +172,7 @@ def test_supervisor_drains_before_stopping_python_and_reaps_both_children( patch("asyncio.create_subprocess_exec", side_effect=spawn) as launch, patch("socket.socket", return_value=listener), patch("os.killpg", side_effect=kill_group) as kill, + patch.dict("os.environ", {"FORWARDED_ALLOW_IPS": "*"}), ): assert supervise("gateway", ("python", "--rust_gateway"), "0.0.0.0", 4000) == expected From 077d9032115807662712460c1cd4b998eea46a43 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 06:31:41 +0900 Subject: [PATCH 07/15] refactor(proxy): resolve the rust gateway mode once and make the supervisor policy a pure state machine LITELLM_RUST_GATEWAY is now the only switch. rust_gateway_mode reads the environment once and the entry points pass the result down. Shutdown order and exit codes live in step(), child commands in python_spec and rust_spec. The binary is looked up in the environment's scripts directory only, and an operator's FORWARDED_ALLOW_IPS is kept with the Rust hop added instead of being rejected. Co-Authored-By: Claude Opus 5.5 --- litellm/proxy/proxy_cli.py | 31 +- litellm/proxy/rust_gateway.py | 280 ++++++++++++---- .../configuration/test_rust_gateway.py | 34 +- tests/unit/proxy/test_rust_gateway.py | 303 +++++++++--------- 4 files changed, 406 insertions(+), 242 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 203358cba97..8a3d85c81b3 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -8,6 +8,7 @@ import random import re import subprocess import sys +import sysconfig import urllib.parse as urlparse from collections.abc import Callable, Iterable, Mapping, Sequence from pathlib import Path @@ -409,6 +410,7 @@ class ProxyInitializationHelpers: log_config: str | None = None, keepalive_timeout: int | None = None, timeout_worker_healthcheck: int | None = None, + listener_fd: int | None = None, ) -> dict: """ Get the arguments for `uvicorn` worker @@ -419,7 +421,6 @@ class ProxyInitializationHelpers: import litellm from litellm._logging import _get_uvicorn_json_log_config, resolve_log_level - from litellm.proxy.rust_gateway import inherited_fd from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager uvicorn_args: Final = { @@ -428,7 +429,7 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), - **({"fd": inherited_fd()} if inherited_fd() is not None else {}), + **({"fd": listener_fd} if listener_fd is not None else {}), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -682,6 +683,7 @@ class ProxyInitializationHelpers: ssl_keyfile_path: str, max_requests_before_restart: int | None = None, max_requests_before_restart_jitter: int | None = None, + listener_fd: int | None = None, ): """ Run litellm with `gunicorn` @@ -754,9 +756,7 @@ class ProxyInitializationHelpers: ) graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout() gunicorn_options: Final = { - "bind": ( - f"fd://{os.environ['_LITELLM_GATEWAY_FD']}" if "_LITELLM_GATEWAY_FD" in os.environ else f"{host}:{port}" - ), + "bind": f"fd://{listener_fd}" if listener_fd is not None else f"{host}:{port}", "workers": num_workers, # default is 1 "worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, @@ -1043,13 +1043,6 @@ class ProxyInitializationHelpers: is_flag=True, help="Starts proxy via gunicorn, instead of uvicorn (better for managing multiple workers)", ) -@click.option( - "--rust_gateway", - envvar="LITELLM_RUST_GATEWAY", - default=False, - is_flag=True, - help="Run the Rust forwarding server on the public port, with Python on loopback", -) @click.option( "--run_hypercorn", default=False, @@ -1252,7 +1245,6 @@ def run_server( setup, version, run_gunicorn, - rust_gateway: bool, run_hypercorn, run_granian, ssl_keyfile_path, @@ -1292,12 +1284,15 @@ def run_server( run_setup_wizard() return - if rust_gateway and not any( + from litellm.proxy.rust_gateway import Supervised, Supervisor, rust_gateway_mode + + gateway_mode: Final = rust_gateway_mode(os.environ) + if isinstance(gateway_mode, Supervisor) and not any( (version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data) ): from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise - binary: Final = gateway_binary() + binary: Final = gateway_binary(sysconfig.get_path("scripts")) gateway_error: Final = startup_error( binary, { @@ -1315,6 +1310,7 @@ def run_server( random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port ) raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port)) + supervised: Final = gateway_mode if isinstance(gateway_mode, Supervised) else None args: Final = locals() try: @@ -1764,9 +1760,8 @@ def run_server( if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None: from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process - from litellm.proxy.rust_gateway import metrics_host - metrics_bind_host: Final = metrics_host(host) + metrics_bind_host: Final = supervised.metrics_host if supervised is not None else host try: metrics_process: Final = start_metrics_server_process( host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir @@ -1785,6 +1780,7 @@ def run_server( log_config=log_config, keepalive_timeout=keepalive_timeout, timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None), + listener_fd=supervised.listener_fd if supervised is not None else None, ) # Optional: recycle uvicorn workers after N requests if max_requests_before_restart is not None: @@ -1859,6 +1855,7 @@ def run_server( ssl_keyfile_path=ssl_keyfile_path, max_requests_before_restart=max_requests_before_restart, max_requests_before_restart_jitter=max_requests_before_restart_jitter, + listener_fd=supervised.listener_fd if supervised is not None else None, ) elif run_hypercorn is True: ProxyInitializationHelpers.init_hypercorn_server( diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index cd02be26c77..9a2d61f09f7 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -1,117 +1,255 @@ import asyncio +import contextlib import os import shutil import signal import socket import sys -import sysconfig from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from enum import Enum, IntEnum +from types import MappingProxyType from typing import Final from pydantic import TypeAdapter +from typing_extensions import assert_never +ENABLE: Final = "LITELLM_RUST_GATEWAY" GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" +METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" +TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" +RUST_HOP: Final = "127.0.0.1" + +_FD: Final = TypeAdapter(int) +_ENABLED: Final = TypeAdapter(bool) +_ADDRESS: Final = TypeAdapter(tuple[str, int]) -def gateway_binary() -> str | None: +@dataclass(frozen=True, slots=True) +class Disabled: + pass + + +@dataclass(frozen=True, slots=True) +class Supervisor: + pass + + +@dataclass(frozen=True, slots=True) +class Supervised: + listener_fd: int + metrics_host: str + trusted_proxies: tuple[str, ...] + + +RustGatewayMode = Disabled | Supervisor | Supervised + + +def trusted_proxies(operator_value: str | None) -> tuple[str, ...]: + configured: Final = tuple(host.strip() for host in (operator_value or "").split(",") if host.strip()) + return ("*",) if "*" in configured else (RUST_HOP, *(host for host in configured if host != RUST_HOP)) + + +def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode: + fd: Final = environ.get(GATEWAY_FD) + if fd is not None: + return Supervised(_FD.validate_python(fd), environ[METRICS_HOST], trusted_proxies(environ.get(TRUSTED_PROXIES))) + if _ENABLED.validate_python(environ.get(ENABLE, "false")): + return Supervisor() + return Disabled() + + +def gateway_binary(scripts_dir: str) -> str | None: name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway" - return shutil.which(name, path=sysconfig.get_path("scripts")) or shutil.which(name) - - -def unsupported_proxy_trust(value: str | None) -> bool: - return any(host.strip() not in ("", "127.0.0.1") for host in (value or "").split(",")) + return shutil.which(name, path=scripts_dir) def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: flag: Final = next((name for name, value in unsupported.items() if value), None) if flag is not None: - return f"--rust_gateway does not support {flag}" + return f"{ENABLE} does not support {flag}" if os.name != "posix": - return "--rust_gateway requires POSIX inherited sockets and process signals" + return f"{ENABLE} requires POSIX inherited sockets and process signals" if binary is None: - return "--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment" - if unsupported_proxy_trust(os.environ.get("FORWARDED_ALLOW_IPS")): - return "--rust_gateway does not support non-loopback FORWARDED_ALLOW_IPS; disable Rust forwarding for this deployment" + return f"{ENABLE} requires the litellm-gateway binary. Install litellm[proxy] in this environment" return None -def child_argv(argv: Sequence[str], port: int) -> tuple[str, ...]: - return (*(arg for arg in argv if arg != "--rust_gateway"), "--host", "127.0.0.1", "--port", str(port)) +class Child(Enum): + RUST = "rust" + PYTHON = "python" -def inherited_fd() -> int | None: - value: Final = os.environ.get(GATEWAY_FD) - return int(value) if value is not None else None +DRAIN_ORDER: Final = (Child.RUST, Child.PYTHON) -def metrics_host(host: str) -> str: - return os.environ.get("_LITELLM_GATEWAY_METRICS_HOST", host) if inherited_fd() is not None else host +class ExitCode(IntEnum): + CLEAN = 0 + CHILD_FAILED = 1 -def terminate_group(process: asyncio.subprocess.Process) -> None: - try: +@dataclass(frozen=True, slots=True) +class Stop: + signum: signal.Signals + + +@dataclass(frozen=True, slots=True) +class Exited: + child: Child + code: int + + +@dataclass(frozen=True, slots=True) +class Running: + pass + + +@dataclass(frozen=True, slots=True) +class Draining: + signum: signal.Signals + signalled: Child + remaining: tuple[Child, ...] + + +@dataclass(frozen=True, slots=True) +class Finished: + code: ExitCode + + +@dataclass(frozen=True, slots=True) +class Signal: + child: Child + signum: signal.Signals + + +Event = Stop | Exited +State = Running | Draining | Finished + + +def _signal_next(signum: signal.Signals, order: tuple[Child, ...]) -> tuple[State, Signal | None]: + if not order: + return Finished(ExitCode.CLEAN), None + return Draining(signum, order[0], order[1:]), Signal(order[0], signum) + + +def step(state: State, event: Event) -> tuple[State, Signal | None]: + match state: + case Running(): + match event: + case Stop(signum): + return _signal_next(signum, DRAIN_ORDER) + case Exited(): + return Finished(ExitCode.CHILD_FAILED), None + case _: + assert_never(event) + case Draining(signum, signalled, remaining): + match event: + case Stop(): + return state, None + case Exited(child, code) if child is signalled and code == 0: + return _signal_next(signum, remaining) + case Exited(): + return Finished(ExitCode.CHILD_FAILED), None + case _: + assert_never(event) + case Finished(): + return state, None + case _: + assert_never(state) + + +@dataclass(frozen=True, slots=True) +class ChildSpec: + argv: tuple[str, ...] + env: Mapping[str, str] + pass_fds: tuple[int, ...] + + +def python_spec( + command: Sequence[str], listener_fd: int, upstream_port: int, public_host: str, environ: Mapping[str, str] +) -> ChildSpec: + return ChildSpec( + argv=(*command, "--host", RUST_HOP, "--port", str(upstream_port)), + env=MappingProxyType( + { + **environ, + GATEWAY_FD: str(listener_fd), + METRICS_HOST: public_host, + TRUSTED_PROXIES: ",".join(trusted_proxies(environ.get(TRUSTED_PROXIES))), + } + ), + pass_fds=(listener_fd,), + ) + + +def rust_spec(binary: str, upstream_port: int, host: str, port: int, environ: Mapping[str, str]) -> ChildSpec: + return ChildSpec( + argv=(binary,), + env=MappingProxyType( + { + **environ, + "HOST": host, + "PORT": str(port), + "LITELLM_FORWARD_UPSTREAM": f"http://{RUST_HOP}:{upstream_port}", + } + ), + pass_fds=(), + ) + + +async def _spawn(spec: ChildSpec) -> asyncio.subprocess.Process: + return await asyncio.create_subprocess_exec( + *spec.argv, env=dict(spec.env), pass_fds=spec.pass_fds, start_new_session=True + ) + + +def _terminate_group(process: asyncio.subprocess.Process) -> None: + with contextlib.suppress(ProcessLookupError): os.killpg(process.pid, signal.SIGTERM) - except ProcessLookupError: - pass -async def _supervise(binary: str, command: Sequence[str], host: str, port: int) -> int: +async def _report_exit(child: Child, process: asyncio.subprocess.Process, events: asyncio.Queue[Event]) -> None: + events.put_nowait(Exited(child, await process.wait())) + + +async def _drive( + state: State, events: asyncio.Queue[Event], children: Mapping[Child, asyncio.subprocess.Process] +) -> ExitCode: + if isinstance(state, Finished): + return state.code + following, action = step(state, await events.get()) + if action is not None: + with contextlib.suppress(ProcessLookupError): + children[action.child].send_signal(action.signum) + return await _drive(following, events, children) + + +async def _supervise(binary: str, command: Sequence[str], host: str, port: int) -> ExitCode: loop: Final = asyncio.get_running_loop() - stopped: Final[asyncio.Future[signal.Signals]] = loop.create_future() - - def stop(signum: signal.Signals) -> None: - if not stopped.done(): - stopped.set_result(signum) - + events: Final[asyncio.Queue[Event]] = asyncio.Queue() for signum in (signal.SIGTERM, signal.SIGINT): - loop.add_signal_handler(signum, stop, signum) + loop.add_signal_handler(signum, events.put_nowait, Stop(signum)) with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: - listener.bind(("127.0.0.1", 0)) + listener.bind((RUST_HOP, 0)) listener.listen() - _, upstream_port = TypeAdapter(tuple[str, int]).validate_python(listener.getsockname()) - python: Final = await asyncio.create_subprocess_exec( - *child_argv(command, upstream_port), - env={ - **os.environ, - "LITELLM_RUST_GATEWAY": "false", - GATEWAY_FD: str(listener.fileno()), - "_LITELLM_GATEWAY_METRICS_HOST": host, - "FORWARDED_ALLOW_IPS": "127.0.0.1", - }, - pass_fds=(listener.fileno(),), - start_new_session=True, - ) + _, upstream_port = _ADDRESS.validate_python(listener.getsockname()) + python: Final = await _spawn(python_spec(command, listener.fileno(), upstream_port, host, os.environ)) try: - rust: Final = await asyncio.create_subprocess_exec( - binary, - env={ - **os.environ, - "HOST": host, - "PORT": str(port), - "LITELLM_FORWARD_UPSTREAM": f"http://127.0.0.1:{upstream_port}", - }, - start_new_session=True, - ) + rust: Final = await _spawn(rust_spec(binary, upstream_port, host, port, os.environ)) except OSError: - terminate_group(python) + _terminate_group(python) await python.wait() raise - python_done: Final = asyncio.create_task(python.wait()) - rust_done: Final = asyncio.create_task(rust.wait()) + children: Final = MappingProxyType({Child.RUST: rust, Child.PYTHON: python}) + waiters: Final = tuple( + asyncio.create_task(_report_exit(child, process, events)) for child, process in children.items() + ) try: - done, _ = await asyncio.wait((stopped, python_done, rust_done), return_when=asyncio.FIRST_COMPLETED) - if python_done in done or rust_done in done: - return 1 - rust.send_signal(stopped.result()) - draining, _ = await asyncio.wait((python_done, rust_done), return_when=asyncio.FIRST_COMPLETED) - if python_done in draining or rust_done.result() != 0: - return 1 - python.send_signal(stopped.result()) - return 0 if await python_done == 0 else 1 + return await _drive(Running(), events, children) finally: - terminate_group(rust) - terminate_group(python) - await asyncio.gather(python_done, rust_done) + for process in children.values(): + _terminate_group(process) + await asyncio.gather(*waiters) def supervise(binary: str, command: Sequence[str], host: str, port: int) -> int: @@ -119,7 +257,7 @@ def supervise(binary: str, command: Sequence[str], host: str, port: int) -> int: return asyncio.run(_supervise(binary, command, host, port)) except OSError as error: sys.stderr.write(f"Rust gateway startup failed: {error}\n") - return 1 + return ExitCode.CHILD_FAILED def cli_command(argv: Sequence[str]) -> tuple[str, ...]: diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index bb0119f9398..bbcb20cfa8c 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -12,6 +12,8 @@ from tests.integration._support.client import Gateway, eventually from tests.integration._support.ports import allocate_port from tests.integration._support.process import stop_root_process +TRUSTED_BALANCER: Final = "127.0.0.3" + def ready(client: httpx.Client) -> bool: try: @@ -21,6 +23,19 @@ def ready(client: httpx.Client) -> bool: @pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) +def client_ip(base_url: str, local_address: str, key: str, forwarded_for: str) -> httpx.Response: + with httpx.Client( + base_url=base_url, + transport=httpx.HTTPTransport(local_address=local_address), + trust_env=False, + timeout=5, + ) as client: + return client.get( + "/v1/mcp/network/client-ip", + headers={"Authorization": f"Bearer {key}", "X-Forwarded-For": forwarded_for}, + ) + + def test_gateway_preserves_metrics_bind_and_mcp_client_ip( gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...] ) -> None: @@ -40,6 +55,8 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( "LITELLM_MASTER_KEY": gateway.key, "STORE_MODEL_IN_DB": "False", "PYTHON_DOTENV_DISABLED": "1", + "LITELLM_RUST_GATEWAY": "true", + "FORWARDED_ALLOW_IPS": TRUSTED_BALANCER, } with (tmp_path / "proxy.log").open("wb") as output: with subprocess.Popen( @@ -48,7 +65,6 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( "-I", "-m", "litellm.proxy.proxy_cli", - "--rust_gateway", "--config", str(config), "--host", @@ -78,12 +94,16 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( assert eventually(lambda: ready(external), bool, seconds=180), (tmp_path / "proxy.log").read_text() response: Final = external.get(f"http://127.0.0.2:{metrics_port}/metrics") assert response.status_code == 200, response.text - client_ip: Final = external.get( - "/v1/mcp/network/client-ip", - headers={"Authorization": f"Bearer {gateway.key}", "X-Forwarded-For": "1.2.3.4"}, - ) - assert client_ip.status_code == 200, client_ip.text - assert client_ip.json() == {"ip": "127.0.0.2"} + public: Final = f"http://127.0.0.1:{public_port}" + spoofed: Final = client_ip(public, "127.0.0.2", gateway.key, "1.2.3.4") + assert spoofed.status_code == 200, spoofed.text + assert spoofed.json() == {"ip": "127.0.0.2"} + behind_balancer: Final = client_ip(public, TRUSTED_BALANCER, gateway.key, "203.0.113.9") + assert behind_balancer.status_code == 200, behind_balancer.text + assert behind_balancer.json() == {"ip": "203.0.113.9"} + borrowed: Final = client_ip(public, "127.0.0.2", gateway.key, f"203.0.113.9, {TRUSTED_BALANCER}") + assert borrowed.status_code == 200, borrowed.text + assert borrowed.json() == {"ip": "127.0.0.2"} finally: assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text() assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text() diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index 27b2e9b54e9..278dc4621d3 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -1,193 +1,202 @@ -import asyncio +import functools import signal -import socket import sys from pathlib import Path from typing import Final -from unittest.mock import MagicMock, Mock, call, patch import pytest from click.testing import CliRunner +from pydantic import ValidationError from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server from litellm.proxy.rust_gateway import ( - child_argv, + Child, + Disabled, + Event, + ExitCode, + Exited, + Finished, + Running, + Signal, + State, + Stop, + Supervised, + Supervisor, cli_command, gateway_binary, - inherited_fd, - metrics_host, + python_spec, + rust_gateway_mode, + rust_spec, startup_error, - supervise, + step, + trusted_proxies, ) +TERM: Final = signal.SIGTERM +INT: Final = signal.SIGINT + @pytest.mark.parametrize( "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] ) -@pytest.mark.parametrize("enable", ["flag", "environment"]) -def test_unsupported_modes_fail_before_starting_children( - flag: str, enable: str, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("LITELLM_RUST_GATEWAY", "true" if enable == "environment" else "false") - arguments: Final = ([] if enable == "environment" else ["--rust_gateway"]) + [flag] +def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST_GATEWAY", "true") + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] - result: Final = CliRunner().invoke(run_server, arguments + values) + result: Final = CliRunner().invoke(run_server, [flag, *values]) assert result.exit_code != 0 - assert f"--rust_gateway does not support {flag}" in result.output + assert f"LITELLM_RUST_GATEWAY does not support {flag}" in result.output -def test_child_arguments_preserve_options_and_force_private_listener() -> None: - arguments: Final = ("--rust_gateway", "--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") - params: Final = run_server.make_context("litellm", list(child_argv(arguments, 45555))).params +@pytest.mark.parametrize( + ("environ", "expected"), + [ + ({}, Disabled()), + ({"LITELLM_RUST_GATEWAY": "false"}, Disabled()), + ({"LITELLM_RUST_GATEWAY": "off"}, Disabled()), + ({"LITELLM_RUST_GATEWAY": "0"}, Disabled()), + ({"LITELLM_RUST_GATEWAY": "true"}, Supervisor()), + ({"LITELLM_RUST_GATEWAY": "1"}, Supervisor()), + ({"LITELLM_RUST_GATEWAY": "yes"}, Supervisor()), + ({"LITELLM_RUST_GATEWAY": "on"}, Supervisor()), + ( + { + "LITELLM_RUST_GATEWAY": "true", + "_LITELLM_GATEWAY_FD": "17", + "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", + "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", + }, + Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5")), + ), + ], +) +def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None: + assert rust_gateway_mode(environ) == expected + + +def test_unrecognized_switch_value_is_rejected() -> None: + with pytest.raises(ValidationError): + rust_gateway_mode({"LITELLM_RUST_GATEWAY": "enabled"}) + + +@pytest.mark.parametrize( + ("operator_value", "expected"), + [ + (None, ("127.0.0.1",)), + ("", ("127.0.0.1",)), + ("10.0.0.5", ("127.0.0.1", "10.0.0.5")), + ("127.0.0.1, 10.0.0.5", ("127.0.0.1", "10.0.0.5")), + ("10.0.0.5,127.0.0.1", ("127.0.0.1", "10.0.0.5")), + ("*", ("*",)), + ("10.0.0.5,*", ("*",)), + ], +) +def test_operator_trust_is_kept_and_rust_hop_added(operator_value: str | None, expected: tuple[str, ...]) -> None: + assert trusted_proxies(operator_value) == expected + + +@pytest.mark.parametrize("operator_value", [None, "", "10.0.0.5", "10.0.0.5, 192.0.2.0/24", "127.0.0.1"]) +def test_python_child_always_trusts_the_rust_hop(operator_value: str | None) -> None: + environ: Final = {} if operator_value is None else {"FORWARDED_ALLOW_IPS": operator_value} + spec: Final = python_spec(("python",), 17, 45555, "0.0.0.0", environ) + assert spec.env["FORWARDED_ALLOW_IPS"].split(",")[0] == "127.0.0.1" + assert rust_gateway_mode(spec.env) == Supervised(17, "0.0.0.0", trusted_proxies(operator_value)) + + +def test_python_child_listens_privately_and_keeps_options() -> None: + arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") + spec: Final = python_spec(arguments, 17, 45555, "0.0.0.0", {}) + params: Final = run_server.make_context("litellm", list(spec.argv)).params assert params["host"] == "127.0.0.1" assert params["port"] == 45555 assert params["num_workers"] == 2 assert params["reload"] is True - assert params["rust_gateway"] is False + assert spec.pass_fds == (17,) assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli") +def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None: + spec: Final = rust_spec("gateway", 45555, "0.0.0.0", 4000, {"RUST_LOG": "info"}) + assert spec.argv == ("gateway",) + assert spec.pass_fds == () + assert dict(spec.env) == { + "RUST_LOG": "info", + "HOST": "0.0.0.0", + "PORT": "4000", + "LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555", + } + + def test_missing_binary_gives_install_instruction() -> None: assert startup_error(None, {}) == ( - "--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment" + "LITELLM_RUST_GATEWAY requires the litellm-gateway binary. Install litellm[proxy] in this environment" ) -@pytest.mark.parametrize("trusted", [None, "", "127.0.0.1", "192.0.2.1", "*", "127.0.0.1,192.0.2.1"]) -def test_configured_proxy_trust_is_not_silently_discarded(trusted: str | None, monkeypatch: pytest.MonkeyPatch) -> None: - if trusted is None: - monkeypatch.delenv("FORWARDED_ALLOW_IPS", raising=False) - else: - monkeypatch.setenv("FORWARDED_ALLOW_IPS", trusted) - expected: Final = ( - None - if trusted in (None, "", "127.0.0.1") - else "--rust_gateway does not support non-loopback FORWARDED_ALLOW_IPS; disable Rust forwarding for this deployment" - ) - assert startup_error("gateway", {}) == expected - - -@pytest.mark.parametrize("supervised", [False, True]) -@pytest.mark.parametrize("host", ["0.0.0.0", "192.0.2.1", "::"]) -def test_metrics_keep_configured_host_separate_from_private_listener( - supervised: bool, host: str, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", host) - if supervised: - monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17") - else: - monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) - assert metrics_host("127.0.0.1") == (host if supervised else "127.0.0.1") - - -def test_current_environment_binary_wins_over_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - binary: Final = tmp_path / "litellm-gateway" +def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + scripts: Final = tmp_path / "scripts" + elsewhere: Final = tmp_path / "elsewhere" + scripts.mkdir() + elsewhere.mkdir() + binary: Final = elsewhere / "litellm-gateway" binary.write_text("#!/bin/sh\nexit 0\n") binary.chmod(0o755) - monkeypatch.setattr("sysconfig.get_path", lambda _: str(tmp_path)) - monkeypatch.setenv("PATH", "") - assert gateway_binary() == str(binary) + monkeypatch.setenv("PATH", str(elsewhere)) + assert gateway_binary(str(scripts)) is None + assert gateway_binary(str(elsewhere)) == str(binary) -@pytest.mark.parametrize("fd", [None, "17"]) -def test_inherited_socket_reaches_uvicorn(fd: str | None, monkeypatch: pytest.MonkeyPatch) -> None: - if fd is None: - monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) - else: - monkeypatch.setenv("_LITELLM_GATEWAY_FD", fd) - expected: Final = int(fd) if fd is not None else None - options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555) - assert inherited_fd() == expected - assert options.get("fd") == expected +@pytest.mark.parametrize("listener_fd", [None, 17]) +def test_inherited_socket_reaches_uvicorn(listener_fd: int | None) -> None: + options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 45555, listener_fd=listener_fd + ) + assert options.get("fd") == listener_fd + + +def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]: + def advance(acc: tuple[State, tuple[Signal, ...]], event: Event) -> tuple[State, tuple[Signal, ...]]: + state, sent = acc + following, action = step(state, event) + return following, sent if action is None else (*sent, action) + + return functools.reduce(advance, events, (Running(), ())) + + +RUST: Final = Child.RUST +PYTHON: Final = Child.PYTHON @pytest.mark.parametrize( - ("event", "python_exit", "rust_exit", "expected"), + ("events", "signals", "code"), [ - ("signal", 0, 0, 0), - ("signal", 1, 0, 1), - ("signal", 0, 1, 1), - ("python-exit", 0, 0, 1), - ("rust-exit", 0, 0, 1), - ("python-during-drain", 1, 0, 1), - ("spawn-error", 0, 0, 1), + ( + (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 0)), + (Signal(RUST, TERM), Signal(PYTHON, TERM)), + ExitCode.CLEAN, + ), + ((Exited(PYTHON, 0),), (), ExitCode.CHILD_FAILED), + ((Exited(RUST, 1),), (), ExitCode.CHILD_FAILED), + ((Stop(TERM), Exited(PYTHON, 0)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), + ((Stop(TERM), Exited(RUST, 1)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), + ( + (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 1)), + (Signal(RUST, TERM), Signal(PYTHON, TERM)), + ExitCode.CHILD_FAILED, + ), + ( + (Stop(INT), Stop(INT), Exited(RUST, 0), Exited(PYTHON, 0)), + (Signal(RUST, INT), Signal(PYTHON, INT)), + ExitCode.CLEAN, + ), + ( + (Exited(RUST, 1), Stop(TERM), Exited(PYTHON, 0)), + (), + ExitCode.CHILD_FAILED, + ), ], ) -def test_supervisor_drains_before_stopping_python_and_reaps_both_children( - event: str, python_exit: int, rust_exit: int, expected: int, capsys: pytest.CaptureFixture[str] +def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit( + events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode ) -> None: - loop: Final = asyncio.new_event_loop() - handlers: Final = Mock() - listener: Final = MagicMock(spec=socket.socket) - listener.__enter__.return_value = listener - listener.getsockname.return_value = ("127.0.0.1", 45555) - listener.fileno.return_value = 17 - children: Final = {} - sent: Final = Mock() - - def current_loop(): - return Mock(wraps=loop, add_signal_handler=handlers) - - async def spawn(*argv, **kwargs): - done: Final = asyncio.events.get_running_loop().create_future() - pid: Final = 101 if argv[0] == "python" else 102 - - async def wait() -> int: - return await done - - def send_signal(signum: signal.Signals) -> None: - sent(pid, signum) - if pid == 102 and event == "python-during-drain": - children[101][1].set_result(python_exit) - return - if pid == 101: - assert children[102][1].done(), "Python must remain alive until Rust has drained" - done.set_result(python_exit if pid == 101 else rust_exit) - - process: Final = Mock(spec=asyncio.subprocess.Process, pid=pid, wait=wait, send_signal=send_signal) - if pid == 102 and event == "spawn-error": - raise OSError("gateway cannot start") - children[pid] = (process, done) - if pid == 102: - if event == "python-exit": - children[101][1].set_result(python_exit) - elif event == "rust-exit": - done.set_result(rust_exit) - else: - _, stop, signum = handlers.call_args_list[0].args - stop(signum) - stop(signum) - return process - - def kill_group(pid: int, signum: signal.Signals) -> None: - assert signum == signal.SIGTERM - _, done = children[pid] - if done.done(): - raise ProcessLookupError - done.set_result(1) - - with ( - patch("asyncio.events.new_event_loop", return_value=loop), - patch("asyncio.get_running_loop", side_effect=current_loop), - patch("asyncio.create_subprocess_exec", side_effect=spawn) as launch, - patch("socket.socket", return_value=listener), - patch("os.killpg", side_effect=kill_group) as kill, - patch.dict("os.environ", {"FORWARDED_ALLOW_IPS": "*"}), - ): - assert supervise("gateway", ("python", "--rust_gateway"), "0.0.0.0", 4000) == expected - - assert all(done.done() for _, done in children.values()) - assert call(101, signal.SIGTERM) in kill.call_args_list - python_call, rust_call = launch.call_args_list - assert python_call.args == ("python", "--host", "127.0.0.1", "--port", "45555") - assert python_call.kwargs["pass_fds"] == (17,) - assert python_call.kwargs["env"]["LITELLM_RUST_GATEWAY"] == "false" - assert python_call.kwargs["env"]["FORWARDED_ALLOW_IPS"] == "127.0.0.1" - assert rust_call.kwargs["env"]["LITELLM_FORWARD_UPSTREAM"] == "http://127.0.0.1:45555" - assert rust_call.kwargs["env"]["PORT"] == "4000" - if event == "spawn-error": - assert "Rust gateway startup failed: gateway cannot start" in capsys.readouterr().err - elif event == "signal" and rust_exit == 0: - assert sent.call_args_list == [call(102, signal.SIGTERM), call(101, signal.SIGTERM)] - else: - assert call(101, signal.SIGTERM) not in sent.call_args_list + assert _fold(events) == (Finished(code), signals) From 8b7cdef9903deb318368efbec0b76173336eab6a Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 06:32:23 +0900 Subject: [PATCH 08/15] refactor(proxy): pass the supervised children's environment into supervise Co-Authored-By: Claude Opus 5.5 --- litellm/proxy/proxy_cli.py | 2 +- litellm/proxy/rust_gateway.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 8a3d85c81b3..68db6d3f316 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -1309,7 +1309,7 @@ def run_server( public_port: Final = ( random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port ) - raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port)) + raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port, os.environ)) supervised: Final = gateway_mode if isinstance(gateway_mode, Supervised) else None args: Final = locals() diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 9a2d61f09f7..f2ebd870853 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -224,7 +224,7 @@ async def _drive( return await _drive(following, events, children) -async def _supervise(binary: str, command: Sequence[str], host: str, port: int) -> ExitCode: +async def _supervise(binary: str, command: Sequence[str], host: str, port: int, environ: Mapping[str, str]) -> ExitCode: loop: Final = asyncio.get_running_loop() events: Final[asyncio.Queue[Event]] = asyncio.Queue() for signum in (signal.SIGTERM, signal.SIGINT): @@ -233,9 +233,9 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int) listener.bind((RUST_HOP, 0)) listener.listen() _, upstream_port = _ADDRESS.validate_python(listener.getsockname()) - python: Final = await _spawn(python_spec(command, listener.fileno(), upstream_port, host, os.environ)) + python: Final = await _spawn(python_spec(command, listener.fileno(), upstream_port, host, environ)) try: - rust: Final = await _spawn(rust_spec(binary, upstream_port, host, port, os.environ)) + rust: Final = await _spawn(rust_spec(binary, upstream_port, host, port, environ)) except OSError: _terminate_group(python) await python.wait() @@ -252,9 +252,9 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int) await asyncio.gather(*waiters) -def supervise(binary: str, command: Sequence[str], host: str, port: int) -> int: +def supervise(binary: str, command: Sequence[str], host: str, port: int, environ: Mapping[str, str]) -> int: try: - return asyncio.run(_supervise(binary, command, host, port)) + return asyncio.run(_supervise(binary, command, host, port, environ)) except OSError as error: sys.stderr.write(f"Rust gateway startup failed: {error}\n") return ExitCode.CHILD_FAILED From fc2fce7102047bf1cc0070d7aafc26c924b93e79 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 06:35:19 +0900 Subject: [PATCH 09/15] refactor(proxy): declare rust gateway unions as explicit type aliases Co-Authored-By: Claude Opus 5.5 --- litellm/proxy/rust_gateway.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index f2ebd870853..65237eb00e0 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -9,7 +9,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from enum import Enum, IntEnum from types import MappingProxyType -from typing import Final +from typing import Final, TypeAlias from pydantic import TypeAdapter from typing_extensions import assert_never @@ -42,7 +42,7 @@ class Supervised: trusted_proxies: tuple[str, ...] -RustGatewayMode = Disabled | Supervisor | Supervised +RustGatewayMode: TypeAlias = Disabled | Supervisor | Supervised def trusted_proxies(operator_value: str | None) -> tuple[str, ...]: @@ -122,8 +122,8 @@ class Signal: signum: signal.Signals -Event = Stop | Exited -State = Running | Draining | Finished +Event: TypeAlias = Stop | Exited +State: TypeAlias = Running | Draining | Finished def _signal_next(signum: signal.Signals, order: tuple[Child, ...]) -> tuple[State, Signal | None]: From 4cb3cdbb039f33c1697edd04f3ba869b68761e3c Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 08:50:07 +0900 Subject: [PATCH 10/15] feat(proxy): supervise the Rust forwarding server from gateway.launch Co-Authored-By: Claude Opus 5.5 --- gateway/launch.py | 76 +++++++++++++++++-- .../configuration/test_rust_gateway.py | 71 +++++++++++++++++ tests/unit/gateway/test_launch.py | 69 +++++++++++++++-- 3 files changed, 206 insertions(+), 10 deletions(-) diff --git a/gateway/launch.py b/gateway/launch.py index 59f7121e513..d5d798f262b 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -19,9 +19,11 @@ Run with: import os import sys +import sysconfig from collections.abc import Callable, Mapping, Sequence from typing import TYPE_CHECKING, Final +from typing_extensions import assert_never from uvicorn.main import main as uvicorn_main from litellm_cli.main import without_gc_pauses @@ -31,17 +33,29 @@ if TYPE_CHECKING: from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings + from litellm.proxy.rust_gateway import Supervised GATEWAY_APP: Final = "gateway.launch:worker_app" KEEPALIVE_FLAG: Final = "--timeout-keep-alive" -def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str, ...]: +def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str], supervised: "Supervised | None") -> tuple[str, ...]: """Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly.""" keepalive: Final = environ.get("KEEPALIVE_TIMEOUT") + inherited_socket: Final = ( + ( + "--fd", + str(supervised.listener_fd), + "--proxy-headers", + "--forwarded-allow-ips", + ",".join(supervised.trusted_proxies), + ) + if supervised is not None + else () + ) if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv): - return (GATEWAY_APP, "--factory", *argv) - return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive) + return (GATEWAY_APP, "--factory", *argv, *inherited_socket) + return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive, *inherited_socket) def pool_database_url( @@ -75,7 +89,34 @@ def _serve(argv: Sequence[str]) -> None: uvicorn_main(tuple(argv), prog_name="uvicorn") -def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: +def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, int, Mapping[str, str]]: + """Validate the uvicorn flags for Rust forwarding and return the binary, public address and child environment.""" + from pydantic import TypeAdapter + + from litellm.proxy.rust_gateway import TRUSTED_PROXIES, startup_error + + params: Final[Mapping[str, object]] = uvicorn_main.make_context( + "uvicorn", list(uvicorn_argv(argv, os.environ, None)) + ).params + error: Final = startup_error( + binary, + { + "--ssl-keyfile": params["ssl_keyfile"], + "--ssl-certfile": params["ssl_certfile"], + "--ssl-ciphers": any(arg.split("=")[0] == "--ssl-ciphers" for arg in argv), + "--uds": params["uds"] is not None, + "--fd": params["fd"] is not None, + }, + ) + if error is not None: + sys.exit(error) + assert binary is not None + host, port = TypeAdapter(tuple[str, int]).validate_python((params["host"], params["port"])) + operator_trust: Final = TypeAdapter(str | None).validate_python(params["forwarded_allow_ips"]) + return binary, host, port, {} if operator_trust is None else {TRUSTED_PROXIES: operator_trust} + + +def configure_database() -> None: from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url @@ -86,7 +127,32 @@ def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) - sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}") if pooled_url is not None: export_pooled_database_url(pooled_url) - serve(uvicorn_argv(argv, os.environ)) + + +def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: + from litellm.proxy.rust_gateway import ( + Disabled, + Supervised, + Supervisor, + gateway_binary, + rust_gateway_mode, + supervise, + ) + + mode: Final = rust_gateway_mode(os.environ) + match mode: + case Disabled(): + configure_database() + serve(uvicorn_argv(argv, os.environ, None)) + case Supervisor(): + binary, host, port, trust = public_listener(argv, gateway_binary(sysconfig.get_path("scripts"))) + configure_database() + command: Final = (sys.executable, "-m", "gateway.launch", *argv) + sys.exit(supervise(binary, command, host, port, {**os.environ, **trust})) + case Supervised(): + serve(uvicorn_argv(argv, os.environ, mode)) + case _: + assert_never(mode) if __name__ == "__main__": diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index bbcb20cfa8c..729c0b847e3 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -107,3 +107,74 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( finally: assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text() assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text() + + +@pytest.mark.parametrize( + "server_arguments", + [("--no-proxy-headers",), ("--forwarded-allow-ips", "127.0.0.1")], +) +def test_gateway_launcher_private_hop_preserves_client_address( + tmp_path: Path, server_arguments: tuple[str, ...] +) -> None: + config: Final = tmp_path / "config.yaml" + config.write_text( + json.dumps({"model_list": [], "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY"}}) + ) + public_port: Final = allocate_port() + environment: Final = { + **{key: value for key, value in os.environ.items() if not key.startswith("DATABASE_") and key != "DIRECT_URL"}, + "LITELLM_MASTER_KEY": "sk-rust-forwarding-integration-key-not-a-secret", + "LITELLM_RUST_GATEWAY": "true", + "LITELLM_PGBOUNCER_ENABLED": "false", + "CONFIG_FILE_PATH": str(config), + "STORE_MODEL_IN_DB": "False", + "PYTHON_DOTENV_DISABLED": "1", + } + log: Final = tmp_path / "gateway.log" + with log.open("wb") as output: + with subprocess.Popen( + ( + sys.executable, + "-I", + "-m", + "gateway.launch", + "--host", + "0.0.0.0", + "--port", + str(public_port), + "--workers", + "2", + "--timeout-worker-healthcheck", + "120", + *server_arguments, + ), + env=environment, + stdout=output, + stderr=output, + ) as supervisor: + try: + with httpx.Client( + base_url=f"http://127.0.0.1:{public_port}", + transport=httpx.HTTPTransport(local_address="127.0.0.2"), + trust_env=False, + timeout=5, + ) as external: + assert eventually(lambda: ready(external), bool, seconds=180), log.read_text() + response: Final = external.get( + "/health/liveliness?client-ip-proof", headers={"X-Forwarded-For": "198.51.100.10"} + ) + assert response.status_code == 200, response.text + events: Final = eventually( + lambda: tuple( + line + for line in log.read_text().splitlines() + if "GET /health/liveliness?client-ip-proof " in line + ), + bool, + seconds=10, + ) + assert len(events) == 1, log.read_text() + assert "127.0.0.2:" in events[0], events + finally: + assert stop_root_process(supervisor), log.read_text() + assert supervisor.returncode == 0, log.read_text() diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index d682fcbad2d..606e760ab3f 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -15,10 +15,11 @@ import pytest from uvicorn.importer import import_from_string from uvicorn.main import main as uvicorn_main -from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv +from gateway.launch import GATEWAY_APP, main, pool_database_url, public_listener, uvicorn_argv from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings from litellm.proxy.proxy_server import app as proxy_app +from litellm.proxy.rust_gateway import Supervised DB_ENV: Final = { "DATABASE_HOST": "db.internal", @@ -114,17 +115,65 @@ def _uvicorn_params(argv: tuple[str, ...]) -> dict[str, object]: class TestUvicornArgv: + @pytest.mark.parametrize( + "arguments", [("--uds", "/tmp/gateway.sock"), ("--fd", "0"), ("--fd=17",), ("--ssl-keyfile", "unused")] + ) + def test_forwarding_rejects_unsupported_listeners_before_starting_processes( + self, arguments: tuple[str, ...] + ) -> None: + with pytest.raises(SystemExit, match=f"does not support {arguments[0].split('=')[0]}"): + public_listener(arguments, "gateway") + + @pytest.mark.parametrize( + ("argv", "trust"), + [ + ((), {}), + (("--forwarded-allow-ips", "10.0.0.5"), {"FORWARDED_ALLOW_IPS": "10.0.0.5"}), + (("--forwarded-allow-ips=*",), {"FORWARDED_ALLOW_IPS": "*"}), + ], + ) + def test_forwarding_carries_operator_proxy_trust_to_the_child( + self, argv: tuple[str, ...], trust: dict[str, str] + ) -> None: + assert public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") == ( + "gateway", + "0.0.0.0", + 4000, + trust, + ) + + def test_supervised_listener_uses_the_inherited_loopback_socket(self): + supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1",)) + params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised)) + assert params["fd"] == 17 + assert params["workers"] == 2 + + @pytest.mark.parametrize( + "argv", + [(), ("--no-proxy-headers",), ("--forwarded-allow-ips", "192.0.2.1"), ("--forwarded-allow-ips=*",)], + ) + def test_private_hop_always_processes_headers_with_the_resolved_trust(self, argv: tuple[str, ...]): + supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5")) + params: Final = _uvicorn_params(uvicorn_argv(argv, {}, supervised)) + assert params["proxy_headers"] is True + assert params["forwarded_allow_ips"] == "127.0.0.1,10.0.0.5" + + def test_direct_listener_keeps_explicit_proxy_header_options(self): + params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {}, None)) + assert params["proxy_headers"] is False + assert params["forwarded_allow_ips"] == "*" + def test_keepalive_env_reaches_uvicorn(self): - params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"})) + params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"}, None)) assert params["app"] == GATEWAY_APP assert params["workers"] == 4 assert params["timeout_keep_alive"] == 75 def test_unset_env_keeps_the_uvicorn_default(self): - assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}))["timeout_keep_alive"] == 5 + assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}, None))["timeout_keep_alive"] == 5 def test_an_explicit_flag_wins_over_the_env(self): - argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}) + argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}, None) assert _uvicorn_params(argv)["timeout_keep_alive"] == 30 def test_the_app_uvicorn_is_told_to_serve_is_the_trimmed_gateway(self, monkeypatch: pytest.MonkeyPatch) -> None: @@ -141,7 +190,7 @@ class TestUvicornArgv: ): monkeypatch.delenv(key, raising=False) - params: Final = _uvicorn_params(uvicorn_argv((), {})) + params: Final = _uvicorn_params(uvicorn_argv((), {}, None)) assert params["factory"] is True try: served: Final = import_from_string(cast(str, params["app"]))() @@ -229,6 +278,16 @@ class TestMain: DatabaseURLSettings.from_env().apply_to_env() assert os.environ["DATABASE_URL"] == pooled + def test_forwarding_disabled_keeps_uvicorn_socket_options( + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LITELLM_RUST_GATEWAY", "false") + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False) + served: Final[list[tuple[str, ...]]] = [] + main(("--uds", "/tmp/gateway.sock"), serve=lambda argv: served.append(tuple(argv))) + assert _uvicorn_params(served[0])["uds"] == "/tmp/gateway.sock" + def test_a_pooler_that_cannot_start_stops_the_gateway_before_uvicorn( self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path ): From a3e353c726e6975fc2a84dec15abdac632afe6b6 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 09:07:08 +0900 Subject: [PATCH 11/15] test(proxy): expect the listener fd argument in uvicorn init calls Co-Authored-By: Claude Opus 5.5 --- tests/unit/proxy/test_proxy_cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index 643c01e3e7f..db36f0d681f 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -1550,6 +1550,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=30, timeout_worker_healthcheck=None, + listener_fd=None, ) mock_uvicorn_run.assert_called_once() @@ -1626,6 +1627,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=None, timeout_worker_healthcheck=15, + listener_fd=None, ) @patch("uvicorn.run") From 1949baf3de29712b21b1c076a5e3db53789bef73 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 09:07:08 +0900 Subject: [PATCH 12/15] fix(proxy): restore the client's peer behind Rust instead of trusting the Rust hop Python now sees each request as if the client had connected to it directly. The Rust hop sends the real TCP peer in x-litellm-rust-peer, an app middleware puts it back into the ASGI scope, and uvicorn's ProxyHeadersMiddleware then runs with the same trust settings Python alone would use. Server-level proxy headers are off in supervised mode for uvicorn and gunicorn, since they would judge the loopback hop. The trust-list merge that always trusted 127.0.0.1 is gone, so operators keep the exact FORWARDED_ALLOW_IPS behavior they had. Co-Authored-By: Claude Opus 5.5 --- gateway/launch.py | 15 +- litellm/proxy/proxy_cli.py | 6 +- litellm/proxy/proxy_server.py | 5 + litellm/proxy/rust_gateway.py | 59 ++++++-- .../proxy/shutdown/graceful_uvicorn_worker.py | 10 +- .../configuration/test_rust_gateway.py | 2 +- tests/unit/gateway/test_launch.py | 27 ++-- tests/unit/proxy/test_rust_gateway.py | 135 +++++++++++++++--- 8 files changed, 206 insertions(+), 53 deletions(-) diff --git a/gateway/launch.py b/gateway/launch.py index d5d798f262b..5dec5de46fa 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -46,9 +46,7 @@ def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str], supervised: "S ( "--fd", str(supervised.listener_fd), - "--proxy-headers", - "--forwarded-allow-ips", - ",".join(supervised.trusted_proxies), + "--no-proxy-headers", ) if supervised is not None else () @@ -93,7 +91,7 @@ def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, """Validate the uvicorn flags for Rust forwarding and return the binary, public address and child environment.""" from pydantic import TypeAdapter - from litellm.proxy.rust_gateway import TRUSTED_PROXIES, startup_error + from litellm.proxy.rust_gateway import PROXY_HEADERS, TRUSTED_PROXIES, startup_error params: Final[Mapping[str, object]] = uvicorn_main.make_context( "uvicorn", list(uvicorn_argv(argv, os.environ, None)) @@ -113,7 +111,14 @@ def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, assert binary is not None host, port = TypeAdapter(tuple[str, int]).validate_python((params["host"], params["port"])) operator_trust: Final = TypeAdapter(str | None).validate_python(params["forwarded_allow_ips"]) - return binary, host, port, {} if operator_trust is None else {TRUSTED_PROXIES: operator_trust} + proxy_headers: Final = TypeAdapter(bool).validate_python(params["proxy_headers"]) + return ( + binary, + host, + port, + {PROXY_HEADERS: str(proxy_headers).lower()} + | ({} if operator_trust is None else {TRUSTED_PROXIES: operator_trust}), + ) def configure_database() -> None: diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 68db6d3f316..d1520fd56d9 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -429,7 +429,7 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), - **({"fd": listener_fd} if listener_fd is not None else {}), + **({"fd": listener_fd, "proxy_headers": False} if listener_fd is not None else {}), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -758,7 +758,9 @@ class ProxyInitializationHelpers: gunicorn_options: Final = { "bind": f"fd://{listener_fd}" if listener_fd is not None else f"{host}:{port}", "workers": num_workers, # default is 1 - "worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout), + "worker_class": build_graceful_shutdown_uvicorn_worker( + graceful_shutdown_timeout, proxy_headers=listener_fd is None + ), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, "preload": True, # Add the preload flag, "accesslog": "-", # Log to stdout diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c5fcbfb2564..8a50c116125 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -835,6 +835,7 @@ from litellm.proxy.plugin_routes import ( from litellm.proxy.plugin_routes import ( router as plugin_router, ) +from litellm.proxy.rust_gateway import RustPeerMiddleware, Supervised, rust_gateway_mode from litellm.proxy.spend_tracking.spend_event_producer import ( CollectorSettings, SpendEventProducer, @@ -21924,6 +21925,10 @@ if _server_root_paths: ) app.add_middleware(PerRequestRootPathMiddleware, root_paths=_server_root_paths) +_rust_gateway_mode: Final = rust_gateway_mode(os.environ) +if isinstance(_rust_gateway_mode, Supervised): + app.add_middleware(RustPeerMiddleware, mode=_rust_gateway_mode) + async def _stream_mcp_asgi_response(handle_fn, scope: dict, receive) -> "StreamingResponse": """ diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 65237eb00e0..0dbf3a4de3d 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -9,16 +9,22 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from enum import Enum, IntEnum from types import MappingProxyType -from typing import Final, TypeAlias +from typing import TYPE_CHECKING, Final, TypeAlias from pydantic import TypeAdapter from typing_extensions import assert_never +if TYPE_CHECKING: + from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope + ENABLE: Final = "LITELLM_RUST_GATEWAY" GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" +PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" +DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1" RUST_HOP: Final = "127.0.0.1" +PEER_HEADER: Final = b"x-litellm-rust-peer" _FD: Final = TypeAdapter(int) _ENABLED: Final = TypeAdapter(bool) @@ -39,26 +45,62 @@ class Supervisor: class Supervised: listener_fd: int metrics_host: str - trusted_proxies: tuple[str, ...] + proxy_headers: bool + trusted_proxies: str RustGatewayMode: TypeAlias = Disabled | Supervisor | Supervised -def trusted_proxies(operator_value: str | None) -> tuple[str, ...]: - configured: Final = tuple(host.strip() for host in (operator_value or "").split(",") if host.strip()) - return ("*",) if "*" in configured else (RUST_HOP, *(host for host in configured if host != RUST_HOP)) - - def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode: fd: Final = environ.get(GATEWAY_FD) if fd is not None: - return Supervised(_FD.validate_python(fd), environ[METRICS_HOST], trusted_proxies(environ.get(TRUSTED_PROXIES))) + return Supervised( + _FD.validate_python(fd), + environ[METRICS_HOST], + _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), + environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), + ) if _ENABLED.validate_python(environ.get(ENABLE, "false")): return Supervisor() return Disabled() +def _restore_peer(scope: "WWWScope") -> None: + headers: Final = tuple(scope["headers"]) + peer: Final = next((value for name, value in headers if name == PEER_HEADER), None) + if peer is None: + return + remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER) + scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop + client: Final = scope["client"] + if client is None or client[0] != RUST_HOP: + return + host, _, port = peer.decode("latin-1").rpartition(":") + scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope + + +class RustPeerMiddleware: + """Make Python see each request as if the client had connected to it directly. + + Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real + TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the + real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust + settings Python would have used on the public port. The server's proxy-header handling is off in + this mode, since it would judge Rust's loopback address instead of the real peer. + """ + + def __init__(self, app: "ASGI3Application", mode: Supervised) -> None: + from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + + self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app + + async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + if scope["type"] != "lifespan": + _restore_peer(scope) + await self.app(scope, receive, send) + + def gateway_binary(scripts_dir: str) -> str | None: name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway" return shutil.which(name, path=scripts_dir) @@ -175,7 +217,6 @@ def python_spec( **environ, GATEWAY_FD: str(listener_fd), METRICS_HOST: public_host, - TRUSTED_PROXIES: ",".join(trusted_proxies(environ.get(TRUSTED_PROXIES))), } ), pass_fds=(listener_fd,), diff --git a/litellm/proxy/shutdown/graceful_uvicorn_worker.py b/litellm/proxy/shutdown/graceful_uvicorn_worker.py index 6ae5ae60c3f..8bbd64190f8 100644 --- a/litellm/proxy/shutdown/graceful_uvicorn_worker.py +++ b/litellm/proxy/shutdown/graceful_uvicorn_worker.py @@ -1,10 +1,16 @@ from uvicorn.workers import UvicornWorker -def build_graceful_shutdown_uvicorn_worker(timeout_graceful_shutdown: float) -> type[UvicornWorker]: +def build_graceful_shutdown_uvicorn_worker( + timeout_graceful_shutdown: float, *, proxy_headers: bool = True +) -> type[UvicornWorker]: class GracefulShutdownUvicornWorker(UvicornWorker): def __init__(self, *args: object) -> None: - self.CONFIG_KWARGS = {**UvicornWorker.CONFIG_KWARGS, "timeout_graceful_shutdown": timeout_graceful_shutdown} + self.CONFIG_KWARGS = { + **UvicornWorker.CONFIG_KWARGS, + "timeout_graceful_shutdown": timeout_graceful_shutdown, + "proxy_headers": proxy_headers, + } super().__init__(*args) return GracefulShutdownUvicornWorker diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index 729c0b847e3..06d77d6f3d0 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -22,7 +22,6 @@ def ready(client: httpx.Client) -> bool: return False -@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) def client_ip(base_url: str, local_address: str, key: str, forwarded_for: str) -> httpx.Response: with httpx.Client( base_url=base_url, @@ -36,6 +35,7 @@ def client_ip(base_url: str, local_address: str, key: str, forwarded_for: str) - ) +@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) def test_gateway_preserves_metrics_bind_and_mcp_client_ip( gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...] ) -> None: diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index 606e760ab3f..789067653b7 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -127,12 +127,16 @@ class TestUvicornArgv: @pytest.mark.parametrize( ("argv", "trust"), [ - ((), {}), - (("--forwarded-allow-ips", "10.0.0.5"), {"FORWARDED_ALLOW_IPS": "10.0.0.5"}), - (("--forwarded-allow-ips=*",), {"FORWARDED_ALLOW_IPS": "*"}), + ((), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true"}), + ( + ("--forwarded-allow-ips", "10.0.0.5"), + {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "10.0.0.5"}, + ), + (("--forwarded-allow-ips=*",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "*"}), + (("--no-proxy-headers",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}), ], ) - def test_forwarding_carries_operator_proxy_trust_to_the_child( + def test_forwarding_carries_operator_proxy_header_options_to_the_child( self, argv: tuple[str, ...], trust: dict[str, str] ) -> None: assert public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") == ( @@ -143,20 +147,15 @@ class TestUvicornArgv: ) def test_supervised_listener_uses_the_inherited_loopback_socket(self): - supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1",)) + supervised: Final = Supervised(17, "0.0.0.0", True, "127.0.0.1") params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised)) assert params["fd"] == 17 assert params["workers"] == 2 - @pytest.mark.parametrize( - "argv", - [(), ("--no-proxy-headers",), ("--forwarded-allow-ips", "192.0.2.1"), ("--forwarded-allow-ips=*",)], - ) - def test_private_hop_always_processes_headers_with_the_resolved_trust(self, argv: tuple[str, ...]): - supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5")) - params: Final = _uvicorn_params(uvicorn_argv(argv, {}, supervised)) - assert params["proxy_headers"] is True - assert params["forwarded_allow_ips"] == "127.0.0.1,10.0.0.5" + @pytest.mark.parametrize("argv", [(), ("--proxy-headers",), ("--forwarded-allow-ips=*",)]) + def test_private_hop_leaves_proxy_headers_to_the_app(self, argv: tuple[str, ...]): + params: Final = _uvicorn_params(uvicorn_argv(argv, {}, Supervised(17, "0.0.0.0", True, "*"))) + assert params["proxy_headers"] is False def test_direct_listener_keeps_explicit_proxy_header_options(self): params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {}, None)) diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index 278dc4621d3..b71a098ddb3 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -1,12 +1,24 @@ +import asyncio import functools import signal import sys from pathlib import Path -from typing import Final +from typing import TYPE_CHECKING, Final import pytest from click.testing import CliRunner from pydantic import ValidationError +from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + +if TYPE_CHECKING: + from uvicorn._types import ( + ASGIReceiveCallable, + ASGISendCallable, + ASGISendEvent, + HTTPDisconnectEvent, + HTTPScope, + Scope, + ) from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server from litellm.proxy.rust_gateway import ( @@ -17,6 +29,7 @@ from litellm.proxy.rust_gateway import ( Exited, Finished, Running, + RustPeerMiddleware, Signal, State, Stop, @@ -29,7 +42,6 @@ from litellm.proxy.rust_gateway import ( rust_spec, startup_error, step, - trusted_proxies, ) TERM: Final = signal.SIGTERM @@ -66,7 +78,7 @@ def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", }, - Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5")), + Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"), ), ], ) @@ -80,27 +92,17 @@ def test_unrecognized_switch_value_is_rejected() -> None: @pytest.mark.parametrize( - ("operator_value", "expected"), + ("environ", "expected"), [ - (None, ("127.0.0.1",)), - ("", ("127.0.0.1",)), - ("10.0.0.5", ("127.0.0.1", "10.0.0.5")), - ("127.0.0.1, 10.0.0.5", ("127.0.0.1", "10.0.0.5")), - ("10.0.0.5,127.0.0.1", ("127.0.0.1", "10.0.0.5")), - ("*", ("*",)), - ("10.0.0.5,*", ("*",)), + ({}, Supervised(17, "0.0.0.0", True, "127.0.0.1")), + ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, Supervised(17, "0.0.0.0", True, "10.0.0.5")), + ({"FORWARDED_ALLOW_IPS": "*"}, Supervised(17, "0.0.0.0", True, "*")), + ({"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}, Supervised(17, "0.0.0.0", False, "127.0.0.1")), ], ) -def test_operator_trust_is_kept_and_rust_hop_added(operator_value: str | None, expected: tuple[str, ...]) -> None: - assert trusted_proxies(operator_value) == expected - - -@pytest.mark.parametrize("operator_value", [None, "", "10.0.0.5", "10.0.0.5, 192.0.2.0/24", "127.0.0.1"]) -def test_python_child_always_trusts_the_rust_hop(operator_value: str | None) -> None: - environ: Final = {} if operator_value is None else {"FORWARDED_ALLOW_IPS": operator_value} +def test_python_child_keeps_the_trust_python_alone_would_use(environ: dict[str, str], expected: Supervised) -> None: spec: Final = python_spec(("python",), 17, 45555, "0.0.0.0", environ) - assert spec.env["FORWARDED_ALLOW_IPS"].split(",")[0] == "127.0.0.1" - assert rust_gateway_mode(spec.env) == Supervised(17, "0.0.0.0", trusted_proxies(operator_value)) + assert rust_gateway_mode(spec.env) == expected def test_python_child_listens_privately_and_keeps_options() -> None: @@ -152,6 +154,7 @@ def test_inherited_socket_reaches_uvicorn(listener_fd: int | None) -> None: "127.0.0.1", 45555, listener_fd=listener_fd ) assert options.get("fd") == listener_fd + assert options.get("proxy_headers", True) is (listener_fd is None) def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]: @@ -200,3 +203,95 @@ def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit( events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode ) -> None: assert _fold(events) == (Finished(code), signals) + + +Headers = tuple[tuple[bytes, bytes], ...] + + +async def _receive() -> "HTTPDisconnectEvent": + return {"type": "http.disconnect"} + + +async def _send(message: "ASGISendEvent") -> None: + return None + + +async def _seen_by_app( + behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised +) -> tuple[tuple[str, int] | None, str, Headers]: + seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = ( + asyncio.get_running_loop().create_future() + ) + + async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + assert scope["type"] == "http" + seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"]))) + + direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe + app: Final = RustPeerMiddleware(probe, mode) if behind_rust else direct + scope: Final[HTTPScope] = { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": "/", + "raw_path": b"/", + "query_string": b"", + "root_path": "", + "headers": headers, + "client": client, + "server": ("127.0.0.1", 4000), + } + await app(scope, _receive, _send) + return seen.result() + + +PEER: Final = ("203.0.113.7", 51234) +TRUSTED_PEER: Final = ("10.0.12.34", 40000) +FORWARDED: Final[Headers] = ( + (b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"), + (b"x-forwarded-proto", b"https"), +) + + +@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)]) +@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))]) +@pytest.mark.parametrize( + "mode", + [ + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + Supervised(17, "0.0.0.0", True, "10.0.0.0/8"), + Supervised(17, "0.0.0.0", True, "*"), + Supervised(17, "0.0.0.0", False, "10.0.0.0/8"), + ], +) +async def test_python_behind_rust_sees_what_a_direct_connection_would( + peer: tuple[str, int], headers: Headers, mode: Supervised +) -> None: + rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}" + behind_rust: Final = await _seen_by_app( + True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode + ) + direct: Final = await _seen_by_app(False, peer, headers, mode) + assert behind_rust == direct + + +async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None: + client, scheme, _ = await _seen_by_app( + True, + ("127.0.0.1", 60000), + ((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")), + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + ) + assert (client, scheme) == (("203.0.113.7", 51234), "http") + + +async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None: + client, _, headers = await _seen_by_app( + True, + ("192.0.2.9", 1000), + ((b"x-litellm-rust-peer", b"10.0.12.34:1"),), + Supervised(17, "0.0.0.0", False, "127.0.0.1"), + ) + assert (client, headers) == (("192.0.2.9", 1000), ()) From a948afa6827dc84e76c3079582e2eb5a90850c28 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 10:24:24 +0900 Subject: [PATCH 13/15] feat(proxy): opt into Rust forwarding with LITELLM_RUST One switch turns on every optional Rust path, so forwarding reads the same variable through the same parser as the rust bridge instead of its own LITELLM_RUST_GATEWAY. An unrecognized value now leaves forwarding off, like every other Rust path, instead of failing startup. Co-Authored-By: Claude Opus 5.5 --- litellm/proxy/rust_gateway.py | 11 +++---- litellm/rust_bridge/configuration.py | 9 ++++-- .../configuration/test_rust_gateway.py | 4 +-- tests/unit/gateway/test_launch.py | 2 +- tests/unit/proxy/test_rust_gateway.py | 30 ++++++++----------- 5 files changed, 29 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 0dbf3a4de3d..8be060e362c 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -14,10 +14,11 @@ from typing import TYPE_CHECKING, Final, TypeAlias from pydantic import TypeAdapter from typing_extensions import assert_never +from litellm.rust_bridge.configuration import ENV_NAME, environment_switch + if TYPE_CHECKING: from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope -ENABLE: Final = "LITELLM_RUST_GATEWAY" GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" @@ -61,7 +62,7 @@ def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode: _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), ) - if _ENABLED.validate_python(environ.get(ENABLE, "false")): + if environment_switch(environ): return Supervisor() return Disabled() @@ -109,11 +110,11 @@ def gateway_binary(scripts_dir: str) -> str | None: def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: flag: Final = next((name for name, value in unsupported.items() if value), None) if flag is not None: - return f"{ENABLE} does not support {flag}" + return f"{ENV_NAME} does not support {flag}" if os.name != "posix": - return f"{ENABLE} requires POSIX inherited sockets and process signals" + return f"{ENV_NAME} requires POSIX inherited sockets and process signals" if binary is None: - return f"{ENABLE} requires the litellm-gateway binary. Install litellm[proxy] in this environment" + return f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment" return None diff --git a/litellm/rust_bridge/configuration.py b/litellm/rust_bridge/configuration.py index 152ba632996..e36527e9f4e 100644 --- a/litellm/rust_bridge/configuration.py +++ b/litellm/rust_bridge/configuration.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +from collections.abc import Mapping from enum import Enum, auto from functools import lru_cache from typing import Final @@ -8,7 +9,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never -_GLOBAL_ENV_NAME: Final = "LITELLM_RUST" +ENV_NAME: Final = "LITELLM_RUST" _ENV_BOOL: Final = TypeAdapter(bool) @@ -44,6 +45,10 @@ def _parse_env_bool(value: str | None) -> bool | None: return None +def environment_switch(environ: Mapping[str, str]) -> bool | None: + return _parse_env_bool(environ.get(ENV_NAME)) + + def decide( rollout: Rollout, *, @@ -72,7 +77,7 @@ def decision(rollout: Rollout) -> Decision: return decide( rollout, process_override=_CONFIGURATION.override, - environment_override=_parse_env_bool(os.getenv(_GLOBAL_ENV_NAME)), + environment_override=environment_switch(os.environ), ) diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index 06d77d6f3d0..b75da286bf7 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -55,7 +55,7 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( "LITELLM_MASTER_KEY": gateway.key, "STORE_MODEL_IN_DB": "False", "PYTHON_DOTENV_DISABLED": "1", - "LITELLM_RUST_GATEWAY": "true", + "LITELLM_RUST": "true", "FORWARDED_ALLOW_IPS": TRUSTED_BALANCER, } with (tmp_path / "proxy.log").open("wb") as output: @@ -124,7 +124,7 @@ def test_gateway_launcher_private_hop_preserves_client_address( environment: Final = { **{key: value for key, value in os.environ.items() if not key.startswith("DATABASE_") and key != "DIRECT_URL"}, "LITELLM_MASTER_KEY": "sk-rust-forwarding-integration-key-not-a-secret", - "LITELLM_RUST_GATEWAY": "true", + "LITELLM_RUST": "true", "LITELLM_PGBOUNCER_ENABLED": "false", "CONFIG_FILE_PATH": str(config), "STORE_MODEL_IN_DB": "False", diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index 789067653b7..418e0358b64 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -280,7 +280,7 @@ class TestMain: def test_forwarding_disabled_keeps_uvicorn_socket_options( self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch ) -> None: - monkeypatch.setenv("LITELLM_RUST_GATEWAY", "false") + monkeypatch.setenv("LITELLM_RUST", "false") monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False) served: Final[list[tuple[str, ...]]] = [] diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index b71a098ddb3..1072148be9f 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Final import pytest from click.testing import CliRunner -from pydantic import ValidationError from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware if TYPE_CHECKING: @@ -52,28 +51,30 @@ INT: Final = signal.SIGINT "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] ) def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("LITELLM_RUST_GATEWAY", "true") + monkeypatch.setenv("LITELLM_RUST", "true") monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] result: Final = CliRunner().invoke(run_server, [flag, *values]) assert result.exit_code != 0 - assert f"LITELLM_RUST_GATEWAY does not support {flag}" in result.output + assert f"LITELLM_RUST does not support {flag}" in result.output @pytest.mark.parametrize( ("environ", "expected"), [ ({}, Disabled()), - ({"LITELLM_RUST_GATEWAY": "false"}, Disabled()), - ({"LITELLM_RUST_GATEWAY": "off"}, Disabled()), - ({"LITELLM_RUST_GATEWAY": "0"}, Disabled()), - ({"LITELLM_RUST_GATEWAY": "true"}, Supervisor()), - ({"LITELLM_RUST_GATEWAY": "1"}, Supervisor()), - ({"LITELLM_RUST_GATEWAY": "yes"}, Supervisor()), - ({"LITELLM_RUST_GATEWAY": "on"}, Supervisor()), + ({"LITELLM_RUST": "false"}, Disabled()), + ({"LITELLM_RUST": "off"}, Disabled()), + ({"LITELLM_RUST": "0"}, Disabled()), + ({"LITELLM_RUST": "true"}, Supervisor()), + ({"LITELLM_RUST": "1"}, Supervisor()), + ({"LITELLM_RUST": "yes"}, Supervisor()), + ({"LITELLM_RUST": "on"}, Supervisor()), + ({"LITELLM_RUST": " 1 "}, Supervisor()), + ({"LITELLM_RUST": "enabled"}, Disabled()), ( { - "LITELLM_RUST_GATEWAY": "true", + "LITELLM_RUST": "true", "_LITELLM_GATEWAY_FD": "17", "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", @@ -86,11 +87,6 @@ def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: ob assert rust_gateway_mode(environ) == expected -def test_unrecognized_switch_value_is_rejected() -> None: - with pytest.raises(ValidationError): - rust_gateway_mode({"LITELLM_RUST_GATEWAY": "enabled"}) - - @pytest.mark.parametrize( ("environ", "expected"), [ @@ -131,7 +127,7 @@ def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> Non def test_missing_binary_gives_install_instruction() -> None: assert startup_error(None, {}) == ( - "LITELLM_RUST_GATEWAY requires the litellm-gateway binary. Install litellm[proxy] in this environment" + "LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment" ) From 20339c2eaa8d00cfc6f419c0de1851fa7fcb4943 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 13:21:05 +0900 Subject: [PATCH 14/15] fix(proxy): gather Rust forwarding into rust_forward and hand Rust Python's keepalive and drain Every Python line that exists only because Rust forwards to Python now lives in litellm/proxy/rust_forward (mode, peer, listener, supervisor, asgi), paired with the gateway-forward crate. proxy_server no longer knows about Rust. proxy_cli and gateway.launch wrap the app where they load it, apply server_overrides in the supervised child, and turn their own flags into one PublicListener The supervisor now resolves the keepalive and shutdown drain Python actually runs with and passes them to Rust as LITELLM_FORWARD_UPSTREAM_KEEPALIVE and LITELLM_FORWARD_SHUTDOWN_DRAIN, along with LITELLM_FORWARD_HOST and LITELLM_FORWARD_PORT. Before, `--keepalive_timeout 2` gave uvicorn 2s while Rust kept its default 5s pool and reused connections uvicorn was closing Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test-unit.yml | 1 + gateway/launch.py | 77 +++-- litellm/proxy/proxy_cli.py | 99 ++++-- litellm/proxy/proxy_server.py | 5 - litellm/proxy/rust_forward/AGENTS.md | 8 + litellm/proxy/rust_forward/__init__.py | 0 litellm/proxy/rust_forward/asgi.py | 10 + litellm/proxy/rust_forward/listener.py | 46 +++ litellm/proxy/rust_forward/mode.py | 68 ++++ litellm/proxy/rust_forward/peer.py | 57 ++++ .../supervisor.py} | 133 ++------ ...t_rust_gateway.py => test_rust_forward.py} | 0 tests/unit/gateway/test_launch.py | 63 +++- tests/unit/proxy/rust_forward/__init__.py | 0 .../unit/proxy/rust_forward/test_listener.py | 74 +++++ tests/unit/proxy/rust_forward/test_mode.py | 43 +++ tests/unit/proxy/rust_forward/test_peer.py | 144 +++++++++ .../proxy/rust_forward/test_supervisor.py | 139 +++++++++ tests/unit/proxy/test_proxy_cli.py | 4 +- tests/unit/proxy/test_rust_gateway.py | 293 ------------------ 20 files changed, 769 insertions(+), 495 deletions(-) create mode 100644 litellm/proxy/rust_forward/AGENTS.md create mode 100644 litellm/proxy/rust_forward/__init__.py create mode 100644 litellm/proxy/rust_forward/asgi.py create mode 100644 litellm/proxy/rust_forward/listener.py create mode 100644 litellm/proxy/rust_forward/mode.py create mode 100644 litellm/proxy/rust_forward/peer.py rename litellm/proxy/{rust_gateway.py => rust_forward/supervisor.py} (53%) rename tests/integration/configuration/{test_rust_gateway.py => test_rust_forward.py} (100%) create mode 100644 tests/unit/proxy/rust_forward/__init__.py create mode 100644 tests/unit/proxy/rust_forward/test_listener.py create mode 100644 tests/unit/proxy/rust_forward/test_mode.py create mode 100644 tests/unit/proxy/rust_forward/test_peer.py create mode 100644 tests/unit/proxy/rust_forward/test_supervisor.py delete mode 100644 tests/unit/proxy/test_rust_gateway.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 17f68cd7d32..5e61d6ebd4c 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -583,6 +583,7 @@ jobs: tests/unit/proxy/credential_endpoints tests/unit/proxy/discovery_endpoints tests/unit/proxy/shutdown + tests/unit/proxy/rust_forward tests/unit/proxy/public_endpoints tests/unit/proxy/prompts tests/unit/proxy/rag_endpoints diff --git a/gateway/launch.py b/gateway/launch.py index 5dec5de46fa..89341a4cb89 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -29,27 +29,29 @@ from uvicorn.main import main as uvicorn_main from litellm_cli.main import without_gc_pauses if TYPE_CHECKING: - from fastapi import FastAPI - from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings - from litellm.proxy.rust_gateway import Supervised + from litellm.proxy.rust_forward.listener import PublicListener + from litellm.proxy.rust_forward.mode import ServerOverrides + from litellm.proxy.rust_forward.peer import ASGIApp GATEWAY_APP: Final = "gateway.launch:worker_app" KEEPALIVE_FLAG: Final = "--timeout-keep-alive" -def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str], supervised: "Supervised | None") -> tuple[str, ...]: +def uvicorn_argv( + argv: Sequence[str], environ: Mapping[str, str], forwarded: "ServerOverrides | None" +) -> tuple[str, ...]: """Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly.""" keepalive: Final = environ.get("KEEPALIVE_TIMEOUT") inherited_socket: Final = ( - ( + () + if forwarded is None + else ( "--fd", - str(supervised.listener_fd), - "--no-proxy-headers", + str(forwarded.fd), + "--proxy-headers" if forwarded.proxy_headers else "--no-proxy-headers", ) - if supervised is not None - else () ) if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv): return (GATEWAY_APP, "--factory", *argv, *inherited_socket) @@ -77,26 +79,29 @@ def pool_database_url( return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth()) -def worker_app() -> "FastAPI": +def worker_app() -> "ASGIApp": with without_gc_pauses(): from gateway.main import app - return app + from litellm.proxy.rust_forward.mode import rust_forward_mode + from litellm.proxy.rust_forward.peer import wrap + return wrap(app, rust_forward_mode(os.environ)) def _serve(argv: Sequence[str]) -> None: uvicorn_main(tuple(argv), prog_name="uvicorn") -def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, int, Mapping[str, str]]: - """Validate the uvicorn flags for Rust forwarding and return the binary, public address and child environment.""" +def public_listener(argv: Sequence[str], binary: str | None) -> "tuple[str, PublicListener]": + """Validate the uvicorn flags for Rust forwarding and return the binary and the public listener.""" from pydantic import TypeAdapter - from litellm.proxy.rust_gateway import PROXY_HEADERS, TRUSTED_PROXIES, startup_error + from litellm.proxy.rust_forward.listener import PublicListener, StartupError, shutdown_drain_seconds, startup_check + from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager params: Final[Mapping[str, object]] = uvicorn_main.make_context( "uvicorn", list(uvicorn_argv(argv, os.environ, None)) ).params - error: Final = startup_error( + checked: Final = startup_check( binary, { "--ssl-keyfile": params["ssl_keyfile"], @@ -106,18 +111,18 @@ def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, "--fd": params["fd"] is not None, }, ) - if error is not None: - sys.exit(error) - assert binary is not None - host, port = TypeAdapter(tuple[str, int]).validate_python((params["host"], params["port"])) - operator_trust: Final = TypeAdapter(str | None).validate_python(params["forwarded_allow_ips"]) - proxy_headers: Final = TypeAdapter(bool).validate_python(params["proxy_headers"]) - return ( - binary, - host, - port, - {PROXY_HEADERS: str(proxy_headers).lower()} - | ({} if operator_trust is None else {TRUSTED_PROXIES: operator_trust}), + if isinstance(checked, StartupError): + sys.exit(checked.reason) + graceful_timeout: Final = TypeAdapter[int | None](int | None).validate_python(params["timeout_graceful_shutdown"]) + return checked, PublicListener( + host=TypeAdapter(str).validate_python(params["host"]), + port=TypeAdapter(int).validate_python(params["port"]), + keepalive_seconds=TypeAdapter(int).validate_python(params["timeout_keep_alive"]), + shutdown_drain_seconds=shutdown_drain_seconds( + GracefulShutdownManager.get_timeout() if graceful_timeout is None else graceful_timeout + ), + proxy_headers=TypeAdapter(bool).validate_python(params["proxy_headers"]), + trusted_proxies=TypeAdapter[str | None](str | None).validate_python(params["forwarded_allow_ips"]), ) @@ -135,27 +140,21 @@ def configure_database() -> None: def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: - from litellm.proxy.rust_gateway import ( - Disabled, - Supervised, - Supervisor, - gateway_binary, - rust_gateway_mode, - supervise, - ) + from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides + from litellm.proxy.rust_forward.supervisor import gateway_binary, supervise - mode: Final = rust_gateway_mode(os.environ) + mode: Final = rust_forward_mode(os.environ) match mode: case Disabled(): configure_database() serve(uvicorn_argv(argv, os.environ, None)) case Supervisor(): - binary, host, port, trust = public_listener(argv, gateway_binary(sysconfig.get_path("scripts"))) + binary, listener = public_listener(argv, gateway_binary(sysconfig.get_path("scripts"))) configure_database() command: Final = (sys.executable, "-m", "gateway.launch", *argv) - sys.exit(supervise(binary, command, host, port, {**os.environ, **trust})) + sys.exit(supervise(binary, command, listener, os.environ)) case Supervised(): - serve(uvicorn_argv(argv, os.environ, mode)) + serve(uvicorn_argv(argv, os.environ, server_overrides(mode))) case _: assert_never(mode) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index d1520fd56d9..cb8e936f3c5 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -36,6 +36,10 @@ from litellm.types.llms.base import LiteLLMBaseModel if TYPE_CHECKING: from fastapi import FastAPI + + from litellm.proxy.rust_forward.listener import PublicListener + from litellm.proxy.rust_forward.mode import ServerOverrides + from litellm.proxy.rust_forward.peer import ASGIApp else: FastAPI = Any @@ -262,6 +266,7 @@ def _apply_config_logging_settings(litellm_settings: StartupOutputSettings | Non WORKER_APP: Final = "litellm_cli.main:proxy_worker_app" +FORWARD_APP: Final = "litellm.proxy.rust_forward.asgi:app" WORKER_LOG_CONFIG_ENV: Final = "LITELLM_WORKER_LOG_CONFIG" _LOG_CONFIG: Final = TypeAdapter[str | Mapping[str, object]](str | Mapping[str, object]) _LOG_CONFIG_FILE: Final = TypeAdapter[Mapping[str, object]](Mapping[str, object]) @@ -390,12 +395,12 @@ class ProxyInitializationHelpers: logging.config.fileConfig(log_config, disable_existing_loggers=False) @staticmethod - def child_process_uvicorn_overrides(log_config: object) -> Mapping[str, object]: + def child_process_uvicorn_overrides(log_config: object, app: str = WORKER_APP) -> Mapping[str, object]: if log_config is None: - return MappingProxyType({"app": WORKER_APP, "factory": True}) + return MappingProxyType({"app": app, "factory": True}) ProxyInitializationHelpers.apply_log_config(_LOG_CONFIG.validate_python(log_config)) os.environ[WORKER_LOG_CONFIG_ENV] = json.dumps(log_config) - return MappingProxyType({"app": WORKER_APP, "factory": True, "log_config": None}) + return MappingProxyType({"app": app, "factory": True, "log_config": None}) @staticmethod def apply_worker_log_config(environ: Mapping[str, str]) -> None: @@ -410,7 +415,7 @@ class ProxyInitializationHelpers: log_config: str | None = None, keepalive_timeout: int | None = None, timeout_worker_healthcheck: int | None = None, - listener_fd: int | None = None, + forwarded: "ServerOverrides | None" = None, ) -> dict: """ Get the arguments for `uvicorn` worker @@ -429,7 +434,16 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), - **({"fd": listener_fd, "proxy_headers": False} if listener_fd is not None else {}), + **( + {} + if forwarded is None + else { + "app": FORWARD_APP, + "factory": True, + "fd": forwarded.fd, + "proxy_headers": forwarded.proxy_headers, + } + ), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -677,13 +691,13 @@ class ProxyInitializationHelpers: def run_gunicorn_server( host: str, port: int, - app: FastAPI, + app: "ASGIApp", num_workers: int, ssl_certfile_path: str, ssl_keyfile_path: str, max_requests_before_restart: int | None = None, max_requests_before_restart_jitter: int | None = None, - listener_fd: int | None = None, + forwarded: "ServerOverrides | None" = None, ): """ Run litellm with `gunicorn` @@ -756,10 +770,10 @@ class ProxyInitializationHelpers: ) graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout() gunicorn_options: Final = { - "bind": f"fd://{listener_fd}" if listener_fd is not None else f"{host}:{port}", + "bind": f"{host}:{port}" if forwarded is None else f"fd://{forwarded.fd}", "workers": num_workers, # default is 1 "worker_class": build_graceful_shutdown_uvicorn_worker( - graceful_shutdown_timeout, proxy_headers=listener_fd is None + graceful_shutdown_timeout, proxy_headers=True if forwarded is None else forwarded.proxy_headers ), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, "preload": True, # Add the preload flag, @@ -829,6 +843,34 @@ class ProxyInitializationHelpers: _is_port_in_use = is_port_in_use + @staticmethod + def public_listener(host: str, port: int, keepalive_timeout: int | None, run_gunicorn: bool) -> "PublicListener": + """``run_gunicorn_server`` never hands ``--keepalive_timeout`` to gunicorn, so its workers keep gunicorn's.""" + from gunicorn.config import Config as GunicornConfig + + from litellm.proxy.rust_forward.listener import ( + PublicListener, + shutdown_drain_seconds, + uvicorn_default_keepalive, + ) + from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager + + keepalive: Final = ( + TypeAdapter(int).validate_python(GunicornConfig().keepalive) + if run_gunicorn + else (uvicorn_default_keepalive() if keepalive_timeout is None else keepalive_timeout) + ) + return PublicListener( + host=host, + port=random.randint(1024, 49152) + if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) + else port, + keepalive_seconds=keepalive, + shutdown_drain_seconds=shutdown_drain_seconds(GracefulShutdownManager.get_timeout()), + proxy_headers=True, + trusted_proxies=None, + ) + @staticmethod def get_loop_type(): """Helper function to determine the event loop type based on platform""" @@ -1286,17 +1328,17 @@ def run_server( run_setup_wizard() return - from litellm.proxy.rust_gateway import Supervised, Supervisor, rust_gateway_mode + from litellm.proxy.rust_forward.mode import Supervised, Supervisor, rust_forward_mode, server_overrides - gateway_mode: Final = rust_gateway_mode(os.environ) - if isinstance(gateway_mode, Supervisor) and not any( + forward_mode: Final = rust_forward_mode(os.environ) + if isinstance(forward_mode, Supervisor) and not any( (version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data) ): - from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise + from litellm.proxy.rust_forward.listener import StartupError, startup_check + from litellm.proxy.rust_forward.supervisor import cli_command, gateway_binary, supervise - binary: Final = gateway_binary(sysconfig.get_path("scripts")) - gateway_error: Final = startup_error( - binary, + binary: Final = startup_check( + gateway_binary(sysconfig.get_path("scripts")), { "--ssl_keyfile_path": ssl_keyfile_path, "--ssl_certfile_path": ssl_certfile_path, @@ -1305,14 +1347,11 @@ def run_server( "--run_granian": run_granian, }, ) - if gateway_error is not None: - raise click.ClickException(gateway_error) - assert binary is not None - public_port: Final = ( - random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port - ) - raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port, os.environ)) - supervised: Final = gateway_mode if isinstance(gateway_mode, Supervised) else None + if isinstance(binary, StartupError): + raise click.ClickException(binary.reason) + listener: Final = ProxyInitializationHelpers.public_listener(host, port, keepalive_timeout, run_gunicorn) + raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), listener, os.environ)) + forwarded: Final = server_overrides(forward_mode) if isinstance(forward_mode, Supervised) else None args: Final = locals() try: @@ -1763,7 +1802,7 @@ def run_server( if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None: from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process - metrics_bind_host: Final = supervised.metrics_host if supervised is not None else host + metrics_bind_host: Final = host if forwarded is None else forwarded.metrics_host try: metrics_process: Final = start_metrics_server_process( host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir @@ -1782,7 +1821,7 @@ def run_server( log_config=log_config, keepalive_timeout=keepalive_timeout, timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None), - listener_fd=supervised.listener_fd if supervised is not None else None, + forwarded=forwarded, ) # Optional: recycle uvicorn workers after N requests if max_requests_before_restart is not None: @@ -1822,7 +1861,7 @@ def run_server( start_query_engine_reaper() if num_workers > 1 or reload: worker_overrides: Final = ProxyInitializationHelpers.child_process_uvicorn_overrides( - uvicorn_args.get("log_config") + uvicorn_args.get("log_config"), WORKER_APP if forwarded is None else FORWARD_APP ) if reload and num_workers > 1: from litellm.proxy.dev_reload import ( @@ -1848,16 +1887,18 @@ def run_server( workers=num_workers, ) elif run_gunicorn is True: + from litellm.proxy.rust_forward.peer import wrap + ProxyInitializationHelpers.run_gunicorn_server( host=host, port=port, - app=app, + app=wrap(app, forward_mode), num_workers=num_workers, ssl_certfile_path=ssl_certfile_path, ssl_keyfile_path=ssl_keyfile_path, max_requests_before_restart=max_requests_before_restart, max_requests_before_restart_jitter=max_requests_before_restart_jitter, - listener_fd=supervised.listener_fd if supervised is not None else None, + forwarded=forwarded, ) elif run_hypercorn is True: ProxyInitializationHelpers.init_hypercorn_server( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8a50c116125..c5fcbfb2564 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -835,7 +835,6 @@ from litellm.proxy.plugin_routes import ( from litellm.proxy.plugin_routes import ( router as plugin_router, ) -from litellm.proxy.rust_gateway import RustPeerMiddleware, Supervised, rust_gateway_mode from litellm.proxy.spend_tracking.spend_event_producer import ( CollectorSettings, SpendEventProducer, @@ -21925,10 +21924,6 @@ if _server_root_paths: ) app.add_middleware(PerRequestRootPathMiddleware, root_paths=_server_root_paths) -_rust_gateway_mode: Final = rust_gateway_mode(os.environ) -if isinstance(_rust_gateway_mode, Supervised): - app.add_middleware(RustPeerMiddleware, mode=_rust_gateway_mode) - async def _stream_mcp_asgi_response(handle_fn, scope: dict, receive) -> "StreamingResponse": """ diff --git a/litellm/proxy/rust_forward/AGENTS.md b/litellm/proxy/rust_forward/AGENTS.md new file mode 100644 index 00000000000..a2ab93cde5c --- /dev/null +++ b/litellm/proxy/rust_forward/AGENTS.md @@ -0,0 +1,8 @@ +# Rust forwarding, Python half + +With `LITELLM_RUST` on, the supervisor here runs the Rust gateway on the public port and Python on a private loopback listener. This package is the Python half of forwarding and pairs with the crate `litellm-rust/crates/gateway-forward`. Once Rust serves every route, both get deleted together, along with one dispatch branch in each entry point + +- Nothing outside this package imports Rust forwarding concepts, except the dispatch, the flag adapter and the app wrap in `litellm/proxy/proxy_cli.py` and `gateway/launch.py`. The proxy app itself never knows Rust is in front of it +- Python must see each request as if the client had connected directly. `x-litellm-rust-peer` is the only Rust-specific input, and `peer.py` honors it only from the loopback hop +- Python owns every setting Rust needs. The supervisor resolves the host, port, keepalive and shutdown drain Python actually runs with and hands them to Rust as the `LITELLM_FORWARD_*` variables in `rust_spec`. Rust never reads Python's own settings, so a new value Rust needs gets a new variable there +- Entry points turn their own flags into a `PublicListener`, apply `server_overrides` in the supervised child, and wrap the app with `peer.wrap` where they load it diff --git a/litellm/proxy/rust_forward/__init__.py b/litellm/proxy/rust_forward/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/proxy/rust_forward/asgi.py b/litellm/proxy/rust_forward/asgi.py new file mode 100644 index 00000000000..ff9bdc57cf8 --- /dev/null +++ b/litellm/proxy/rust_forward/asgi.py @@ -0,0 +1,10 @@ +import os + +from litellm.proxy.rust_forward.mode import rust_forward_mode +from litellm.proxy.rust_forward.peer import ASGIApp, wrap + + +def app() -> ASGIApp: + from litellm_cli.main import proxy_worker_app + + return wrap(proxy_worker_app(), rust_forward_mode(os.environ)) diff --git a/litellm/proxy/rust_forward/listener.py b/litellm/proxy/rust_forward/listener.py new file mode 100644 index 00000000000..28595a4099e --- /dev/null +++ b/litellm/proxy/rust_forward/listener.py @@ -0,0 +1,46 @@ +import os +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +from litellm.rust_bridge.configuration import ENV_NAME + + +@dataclass(frozen=True, slots=True) +class PublicListener: + host: str + port: int + keepalive_seconds: int + shutdown_drain_seconds: float + proxy_headers: bool + trusted_proxies: str | None + + +@dataclass(frozen=True, slots=True) +class StartupError: + reason: str + + +def startup_check(binary: str | None, unsupported: Mapping[str, object]) -> str | StartupError: + flag: Final = next((name for name, value in unsupported.items() if value), None) + if flag is not None: + return StartupError(f"{ENV_NAME} does not support {flag}") + if os.name != "posix": + return StartupError(f"{ENV_NAME} requires POSIX inherited sockets and process signals") + if binary is None: + return StartupError( + f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment" + ) + return binary + + +def uvicorn_default_keepalive() -> int: + import uvicorn + + return uvicorn.Config(app="", log_config=None).timeout_keep_alive + + +def shutdown_drain_seconds(graceful_timeout: float) -> float: + from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS + + return graceful_timeout + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS diff --git a/litellm/proxy/rust_forward/mode.py b/litellm/proxy/rust_forward/mode.py new file mode 100644 index 00000000000..a75287688ea --- /dev/null +++ b/litellm/proxy/rust_forward/mode.py @@ -0,0 +1,68 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final, TypeAlias + +from pydantic import TypeAdapter + +from litellm.rust_bridge.configuration import environment_switch + +GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" +METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" +PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" +TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" +DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1" +RUST_HOP: Final = "127.0.0.1" + +_FD: Final = TypeAdapter(int) +_ENABLED: Final = TypeAdapter(bool) + + +@dataclass(frozen=True, slots=True) +class Disabled: + pass + + +@dataclass(frozen=True, slots=True) +class Supervisor: + pass + + +@dataclass(frozen=True, slots=True) +class Supervised: + listener_fd: int + metrics_host: str + proxy_headers: bool + trusted_proxies: str + + +RustForwardMode: TypeAlias = Disabled | Supervisor | Supervised + + +def rust_forward_mode(environ: Mapping[str, str]) -> RustForwardMode: + fd: Final = environ.get(GATEWAY_FD) + if fd is not None: + return Supervised( + _FD.validate_python(fd), + environ[METRICS_HOST], + _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), + environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), + ) + if environment_switch(environ): + return Supervisor() + return Disabled() + + +@dataclass(frozen=True, slots=True) +class ServerOverrides: + fd: int + proxy_headers: bool + metrics_host: str + + +def server_overrides(mode: Supervised) -> ServerOverrides: + """The private listener the supervised server binds instead of its own flags. + + Proxy headers stay off at the server, since it would judge Rust's loopback address instead of the real peer. + ``peer.wrap`` applies the operator's trust settings to the restored peer. + """ + return ServerOverrides(fd=mode.listener_fd, proxy_headers=False, metrics_host=mode.metrics_host) diff --git a/litellm/proxy/rust_forward/peer.py b/litellm/proxy/rust_forward/peer.py new file mode 100644 index 00000000000..2d03d6ca35c --- /dev/null +++ b/litellm/proxy/rust_forward/peer.py @@ -0,0 +1,57 @@ +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Final, TypeAlias + +from typing_extensions import assert_never + +from litellm.proxy.rust_forward.mode import RUST_HOP, Disabled, RustForwardMode, Supervised, Supervisor + +if TYPE_CHECKING: + from uvicorn._types import ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope + +PEER_HEADER: Final = b"x-litellm-rust-peer" + +ASGIApp: TypeAlias = Callable[..., Awaitable[None]] + + +def _restore_peer(scope: "WWWScope") -> None: + headers: Final = tuple(scope["headers"]) + peer: Final = next((value for name, value in headers if name == PEER_HEADER), None) + if peer is None: + return + remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER) + scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop + client: Final = scope["client"] + if client is None or client[0] != RUST_HOP: + return + host, _, port = peer.decode("latin-1").rpartition(":") + scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope + + +class RustPeerMiddleware: + """Make Python see each request as if the client had connected to it directly. + + Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real + TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the + real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust + settings Python would have used on the public port. + """ + + def __init__(self, app: ASGIApp, mode: Supervised) -> None: + from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + + self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app + + async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + if scope["type"] != "lifespan": + _restore_peer(scope) + await self.app(scope, receive, send) + + +def wrap(app: ASGIApp, mode: RustForwardMode) -> ASGIApp: + match mode: + case Supervised(): + return RustPeerMiddleware(app, mode) + case Disabled() | Supervisor(): + return app + case _: + assert_never(mode) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_forward/supervisor.py similarity index 53% rename from litellm/proxy/rust_gateway.py rename to litellm/proxy/rust_forward/supervisor.py index 8be060e362c..6ab4b86dca3 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_forward/supervisor.py @@ -9,115 +9,22 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from enum import Enum, IntEnum from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeAlias +from typing import Final, TypeAlias from pydantic import TypeAdapter from typing_extensions import assert_never -from litellm.rust_bridge.configuration import ENV_NAME, environment_switch +from litellm.proxy.rust_forward.listener import PublicListener +from litellm.proxy.rust_forward.mode import GATEWAY_FD, METRICS_HOST, PROXY_HEADERS, RUST_HOP, TRUSTED_PROXIES -if TYPE_CHECKING: - from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope - -GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" -METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" -PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" -TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" -DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1" -RUST_HOP: Final = "127.0.0.1" -PEER_HEADER: Final = b"x-litellm-rust-peer" - -_FD: Final = TypeAdapter(int) -_ENABLED: Final = TypeAdapter(bool) _ADDRESS: Final = TypeAdapter(tuple[str, int]) -@dataclass(frozen=True, slots=True) -class Disabled: - pass - - -@dataclass(frozen=True, slots=True) -class Supervisor: - pass - - -@dataclass(frozen=True, slots=True) -class Supervised: - listener_fd: int - metrics_host: str - proxy_headers: bool - trusted_proxies: str - - -RustGatewayMode: TypeAlias = Disabled | Supervisor | Supervised - - -def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode: - fd: Final = environ.get(GATEWAY_FD) - if fd is not None: - return Supervised( - _FD.validate_python(fd), - environ[METRICS_HOST], - _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), - environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), - ) - if environment_switch(environ): - return Supervisor() - return Disabled() - - -def _restore_peer(scope: "WWWScope") -> None: - headers: Final = tuple(scope["headers"]) - peer: Final = next((value for name, value in headers if name == PEER_HEADER), None) - if peer is None: - return - remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER) - scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop - client: Final = scope["client"] - if client is None or client[0] != RUST_HOP: - return - host, _, port = peer.decode("latin-1").rpartition(":") - scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope - - -class RustPeerMiddleware: - """Make Python see each request as if the client had connected to it directly. - - Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real - TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the - real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust - settings Python would have used on the public port. The server's proxy-header handling is off in - this mode, since it would judge Rust's loopback address instead of the real peer. - """ - - def __init__(self, app: "ASGI3Application", mode: Supervised) -> None: - from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware - - self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app - - async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: - if scope["type"] != "lifespan": - _restore_peer(scope) - await self.app(scope, receive, send) - - def gateway_binary(scripts_dir: str) -> str | None: name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway" return shutil.which(name, path=scripts_dir) -def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: - flag: Final = next((name for name, value in unsupported.items() if value), None) - if flag is not None: - return f"{ENV_NAME} does not support {flag}" - if os.name != "posix": - return f"{ENV_NAME} requires POSIX inherited sockets and process signals" - if binary is None: - return f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment" - return None - - class Child(Enum): RUST = "rust" PYTHON = "python" @@ -209,7 +116,7 @@ class ChildSpec: def python_spec( - command: Sequence[str], listener_fd: int, upstream_port: int, public_host: str, environ: Mapping[str, str] + command: Sequence[str], listener_fd: int, upstream_port: int, listener: PublicListener, environ: Mapping[str, str] ) -> ChildSpec: return ChildSpec( argv=(*command, "--host", RUST_HOP, "--port", str(upstream_port)), @@ -217,22 +124,26 @@ def python_spec( { **environ, GATEWAY_FD: str(listener_fd), - METRICS_HOST: public_host, + METRICS_HOST: listener.host, + PROXY_HEADERS: str(listener.proxy_headers).lower(), + **({} if listener.trusted_proxies is None else {TRUSTED_PROXIES: listener.trusted_proxies}), } ), pass_fds=(listener_fd,), ) -def rust_spec(binary: str, upstream_port: int, host: str, port: int, environ: Mapping[str, str]) -> ChildSpec: +def rust_spec(binary: str, upstream_port: int, listener: PublicListener, environ: Mapping[str, str]) -> ChildSpec: return ChildSpec( argv=(binary,), env=MappingProxyType( { **environ, - "HOST": host, - "PORT": str(port), "LITELLM_FORWARD_UPSTREAM": f"http://{RUST_HOP}:{upstream_port}", + "LITELLM_FORWARD_HOST": listener.host, + "LITELLM_FORWARD_PORT": str(listener.port), + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE": str(listener.keepalive_seconds), + "LITELLM_FORWARD_SHUTDOWN_DRAIN": str(listener.shutdown_drain_seconds), } ), pass_fds=(), @@ -266,18 +177,20 @@ async def _drive( return await _drive(following, events, children) -async def _supervise(binary: str, command: Sequence[str], host: str, port: int, environ: Mapping[str, str]) -> ExitCode: +async def _supervise( + binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str] +) -> ExitCode: loop: Final = asyncio.get_running_loop() events: Final[asyncio.Queue[Event]] = asyncio.Queue() for signum in (signal.SIGTERM, signal.SIGINT): loop.add_signal_handler(signum, events.put_nowait, Stop(signum)) - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: - listener.bind((RUST_HOP, 0)) - listener.listen() - _, upstream_port = _ADDRESS.validate_python(listener.getsockname()) - python: Final = await _spawn(python_spec(command, listener.fileno(), upstream_port, host, environ)) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as private: + private.bind((RUST_HOP, 0)) + private.listen() + _, upstream_port = _ADDRESS.validate_python(private.getsockname()) + python: Final = await _spawn(python_spec(command, private.fileno(), upstream_port, listener, environ)) try: - rust: Final = await _spawn(rust_spec(binary, upstream_port, host, port, environ)) + rust: Final = await _spawn(rust_spec(binary, upstream_port, listener, environ)) except OSError: _terminate_group(python) await python.wait() @@ -294,9 +207,9 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int, await asyncio.gather(*waiters) -def supervise(binary: str, command: Sequence[str], host: str, port: int, environ: Mapping[str, str]) -> int: +def supervise(binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str]) -> int: try: - return asyncio.run(_supervise(binary, command, host, port, environ)) + return asyncio.run(_supervise(binary, command, listener, environ)) except OSError as error: sys.stderr.write(f"Rust gateway startup failed: {error}\n") return ExitCode.CHILD_FAILED diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_forward.py similarity index 100% rename from tests/integration/configuration/test_rust_gateway.py rename to tests/integration/configuration/test_rust_forward.py diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index 418e0358b64..36eaabfb029 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -12,14 +12,18 @@ from typing import Final, cast from unittest.mock import MagicMock, patch import pytest +import uvicorn from uvicorn.importer import import_from_string from uvicorn.main import main as uvicorn_main from gateway.launch import GATEWAY_APP, main, pool_database_url, public_listener, uvicorn_argv +from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings from litellm.proxy.proxy_server import app as proxy_app -from litellm.proxy.rust_gateway import Supervised +from litellm.proxy.rust_forward.mode import Supervised, server_overrides +from litellm.proxy.rust_forward.supervisor import rust_spec +from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager DB_ENV: Final = { "DATABASE_HOST": "db.internal", @@ -127,34 +131,59 @@ class TestUvicornArgv: @pytest.mark.parametrize( ("argv", "trust"), [ - ((), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true"}), - ( - ("--forwarded-allow-ips", "10.0.0.5"), - {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "10.0.0.5"}, - ), - (("--forwarded-allow-ips=*",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "*"}), - (("--no-proxy-headers",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}), + ((), (True, None)), + (("--forwarded-allow-ips", "10.0.0.5"), (True, "10.0.0.5")), + (("--forwarded-allow-ips=*",), (True, "*")), + (("--no-proxy-headers",), (False, None)), ], ) def test_forwarding_carries_operator_proxy_header_options_to_the_child( - self, argv: tuple[str, ...], trust: dict[str, str] + self, argv: tuple[str, ...], trust: tuple[bool, str | None], monkeypatch: pytest.MonkeyPatch ) -> None: - assert public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") == ( - "gateway", - "0.0.0.0", - 4000, - trust, - ) + monkeypatch.delenv("FORWARDED_ALLOW_IPS", raising=False) + binary, listener = public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") + assert (binary, listener.host, listener.port) == ("gateway", "0.0.0.0", 4000) + assert (listener.proxy_headers, listener.trusted_proxies) == trust + + @pytest.mark.parametrize( + ("argv", "environ"), + [(("--timeout-keep-alive", "2"), {}), ((), {"KEEPALIVE_TIMEOUT": "7"}), ((), {})], + ) + def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with( + self, argv: tuple[str, ...], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + for name, value in environ.items(): + monkeypatch.setenv(name, value) + _, listener = public_listener(argv, "gateway") + sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"] + assert sent == str(_uvicorn_params(uvicorn_argv(argv, os.environ, None))["timeout_keep_alive"]) + + def test_without_a_keepalive_rust_follows_uvicorns_own_default(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + _, listener = public_listener((), "gateway") + assert listener.keepalive_seconds == uvicorn.Config(app="", log_config=None).timeout_keep_alive + + @pytest.mark.parametrize("argv", [(), ("--timeout-graceful-shutdown", "12")]) + def test_rust_drains_for_pythons_graceful_timeout_plus_teardown( + self, argv: tuple[str, ...], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "7.5") + _, listener = public_listener(argv, "gateway") + flag: Final = _uvicorn_params(uvicorn_argv(argv, {}, None))["timeout_graceful_shutdown"] + graceful: Final = GracefulShutdownManager.get_timeout() if flag is None else float(str(flag)) + sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"] + assert float(sent) == graceful + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS def test_supervised_listener_uses_the_inherited_loopback_socket(self): - supervised: Final = Supervised(17, "0.0.0.0", True, "127.0.0.1") + supervised: Final = server_overrides(Supervised(17, "0.0.0.0", True, "127.0.0.1")) params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised)) assert params["fd"] == 17 assert params["workers"] == 2 @pytest.mark.parametrize("argv", [(), ("--proxy-headers",), ("--forwarded-allow-ips=*",)]) def test_private_hop_leaves_proxy_headers_to_the_app(self, argv: tuple[str, ...]): - params: Final = _uvicorn_params(uvicorn_argv(argv, {}, Supervised(17, "0.0.0.0", True, "*"))) + params: Final = _uvicorn_params(uvicorn_argv(argv, {}, server_overrides(Supervised(17, "0.0.0.0", True, "*")))) assert params["proxy_headers"] is False def test_direct_listener_keeps_explicit_proxy_header_options(self): diff --git a/tests/unit/proxy/rust_forward/__init__.py b/tests/unit/proxy/rust_forward/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/rust_forward/test_listener.py b/tests/unit/proxy/rust_forward/test_listener.py new file mode 100644 index 00000000000..d17b0e74e14 --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_listener.py @@ -0,0 +1,74 @@ +from collections.abc import Mapping +from typing import Final + +import pytest +import uvicorn +from click.testing import CliRunner +from gunicorn.config import Config as GunicornConfig + +from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS +from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server +from litellm.proxy.rust_forward.listener import PublicListener, StartupError, startup_check +from litellm.proxy.rust_forward.supervisor import rust_spec + + +@pytest.mark.parametrize( + "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] +) +def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "true") + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] + result: Final = CliRunner().invoke(run_server, [flag, *values]) + assert result.exit_code != 0 + assert f"LITELLM_RUST does not support {flag}" in result.output + + +def test_missing_binary_gives_install_instruction() -> None: + assert startup_check(None, {}) == StartupError( + "LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment" + ) + + +def _cli_listener(argv: list[str]) -> tuple[Mapping[str, object], PublicListener]: + params: Final = run_server.make_context("litellm", ["--port", "45999", *argv]).params + return params, ProxyInitializationHelpers.public_listener( + str(params["host"]), int(str(params["port"])), params["keepalive_timeout"], params["run_gunicorn"] is True + ) + + +def _uvicorn_keepalive(options: Mapping[str, object]) -> int: + configured: Final = {"timeout_keep_alive": options["timeout_keep_alive"]} if "timeout_keep_alive" in options else {} + return uvicorn.Config(app="", log_config=None, **configured).timeout_keep_alive + + +@pytest.mark.parametrize( + ("argv", "environ"), + [(["--keepalive_timeout", "2"], {}), ([], {"KEEPALIVE_TIMEOUT": "7"}), ([], {})], +) +def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with( + argv: list[str], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + for name, value in environ.items(): + monkeypatch.setenv(name, value) + params, listener = _cli_listener(argv) + uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 45555, keepalive_timeout=params["keepalive_timeout"] + ) + sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"] + assert sent == str(_uvicorn_keepalive(uvicorn_options)) + + +def test_gunicorn_workers_keep_gunicorns_keepalive_and_rust_follows(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + _, listener = _cli_listener(["--run_gunicorn", "--keepalive_timeout", "9"]) + assert listener.keepalive_seconds == GunicornConfig().keepalive + + +def test_rust_drains_for_pythons_graceful_timeout_plus_teardown(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "12.5") + _, listener = _cli_listener([]) + uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555) + sent: Final = float(rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"]) + assert sent == uvicorn_options["timeout_graceful_shutdown"] + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS diff --git a/tests/unit/proxy/rust_forward/test_mode.py b/tests/unit/proxy/rust_forward/test_mode.py new file mode 100644 index 00000000000..20005a21255 --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_mode.py @@ -0,0 +1,43 @@ +from typing import Final + +import pytest + +from litellm.proxy.proxy_cli import ProxyInitializationHelpers +from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides + + +@pytest.mark.parametrize( + ("environ", "expected"), + [ + ({}, Disabled()), + ({"LITELLM_RUST": "false"}, Disabled()), + ({"LITELLM_RUST": "off"}, Disabled()), + ({"LITELLM_RUST": "0"}, Disabled()), + ({"LITELLM_RUST": "true"}, Supervisor()), + ({"LITELLM_RUST": "1"}, Supervisor()), + ({"LITELLM_RUST": "yes"}, Supervisor()), + ({"LITELLM_RUST": "on"}, Supervisor()), + ({"LITELLM_RUST": " 1 "}, Supervisor()), + ({"LITELLM_RUST": "enabled"}, Disabled()), + ( + { + "LITELLM_RUST": "true", + "_LITELLM_GATEWAY_FD": "17", + "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", + "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", + }, + Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"), + ), + ], +) +def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None: + assert rust_forward_mode(environ) == expected + + +@pytest.mark.parametrize("mode", [None, Supervised(17, "0.0.0.0", True, "127.0.0.1")]) +def test_inherited_socket_reaches_uvicorn(mode: Supervised | None) -> None: + options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 45555, forwarded=None if mode is None else server_overrides(mode) + ) + assert options.get("fd") == (None if mode is None else mode.listener_fd) + assert options.get("proxy_headers", True) is (mode is None) diff --git a/tests/unit/proxy/rust_forward/test_peer.py b/tests/unit/proxy/rust_forward/test_peer.py new file mode 100644 index 00000000000..e96d9cf2b6a --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_peer.py @@ -0,0 +1,144 @@ +import asyncio +import gc +from typing import TYPE_CHECKING, Final + +import httpx +import pytest +from fastapi import Request +from fastapi.routing import APIRoute +from uvicorn.importer import import_from_string +from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + +if TYPE_CHECKING: + from uvicorn._types import ( + ASGIReceiveCallable, + ASGISendCallable, + ASGISendEvent, + HTTPDisconnectEvent, + HTTPScope, + Scope, + ) + +from litellm.proxy.proxy_server import app as proxy_app +from litellm.proxy.rust_forward.mode import Supervised +from litellm.proxy.rust_forward.peer import wrap + + +Headers = tuple[tuple[bytes, bytes], ...] + + +async def _receive() -> "HTTPDisconnectEvent": + return {"type": "http.disconnect"} + + +async def _send(message: "ASGISendEvent") -> None: + return None + + +async def _seen_by_app( + behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised +) -> tuple[tuple[str, int] | None, str, Headers]: + seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = ( + asyncio.get_running_loop().create_future() + ) + + async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + assert scope["type"] == "http" + seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"]))) + + direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe + app: Final = wrap(probe, mode) if behind_rust else direct + scope: Final[HTTPScope] = { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": "/", + "raw_path": b"/", + "query_string": b"", + "root_path": "", + "headers": headers, + "client": client, + "server": ("127.0.0.1", 4000), + } + await app(scope, _receive, _send) + return seen.result() + + +PEER: Final = ("203.0.113.7", 51234) +TRUSTED_PEER: Final = ("10.0.12.34", 40000) +FORWARDED: Final[Headers] = ( + (b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"), + (b"x-forwarded-proto", b"https"), +) + + +@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)]) +@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))]) +@pytest.mark.parametrize( + "mode", + [ + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + Supervised(17, "0.0.0.0", True, "10.0.0.0/8"), + Supervised(17, "0.0.0.0", True, "*"), + Supervised(17, "0.0.0.0", False, "10.0.0.0/8"), + ], +) +async def test_python_behind_rust_sees_what_a_direct_connection_would( + peer: tuple[str, int], headers: Headers, mode: Supervised +) -> None: + rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}" + behind_rust: Final = await _seen_by_app( + True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode + ) + direct: Final = await _seen_by_app(False, peer, headers, mode) + assert behind_rust == direct + + +async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None: + client, scheme, _ = await _seen_by_app( + True, + ("127.0.0.1", 60000), + ((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")), + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + ) + assert (client, scheme) == (("203.0.113.7", 51234), "http") + + +async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None: + client, _, headers = await _seen_by_app( + True, + ("192.0.2.9", 1000), + ((b"x-litellm-rust-peer", b"10.0.12.34:1"),), + Supervised(17, "0.0.0.0", False, "127.0.0.1"), + ) + assert (client, headers) == (("192.0.2.9", 1000), ()) + + +async def _peer_probe(request: Request) -> dict[str, object]: + assert request.client is not None + return {"client": [request.client.host, request.client.port], "peer": request.headers.get("x-litellm-rust-peer")} + + +@pytest.mark.parametrize("entry_point", ["litellm.proxy.rust_forward.asgi:app", "gateway.launch:worker_app"]) +async def test_every_entry_point_hands_routes_the_peer_rust_saw( + entry_point: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17") + monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", "0.0.0.0") + monkeypatch.setattr(proxy_app.router, "lifespan_context", proxy_app.router.lifespan_context) + monkeypatch.setattr( + proxy_app.router, "routes", [APIRoute("/__rust_forward_peer_probe", _peer_probe), *proxy_app.router.routes] + ) + try: + served: Final = import_from_string(entry_point)() + finally: + gc.unfreeze() + transport: Final = httpx.ASGITransport(app=served, client=("127.0.0.1", 60000)) + async with httpx.AsyncClient(transport=transport, base_url="http://proxy") as client: + response: Final = await client.get( + "/__rust_forward_peer_probe", headers={"x-litellm-rust-peer": "203.0.113.7:51234"} + ) + assert response.status_code == 200, response.text + assert response.json() == {"client": ["203.0.113.7", 51234], "peer": None} diff --git a/tests/unit/proxy/rust_forward/test_supervisor.py b/tests/unit/proxy/rust_forward/test_supervisor.py new file mode 100644 index 00000000000..224d97f349d --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_supervisor.py @@ -0,0 +1,139 @@ +import functools +import signal +import sys +from pathlib import Path +from typing import Final + +import pytest + +from litellm.proxy.proxy_cli import run_server +from litellm.proxy.rust_forward.listener import PublicListener +from litellm.proxy.rust_forward.mode import Supervised, rust_forward_mode +from litellm.proxy.rust_forward.supervisor import ( + Child, + Event, + ExitCode, + Exited, + Finished, + Running, + Signal, + State, + Stop, + cli_command, + gateway_binary, + python_spec, + rust_spec, + step, +) + +TERM: Final = signal.SIGTERM +INT: Final = signal.SIGINT + + +def _listener(proxy_headers: bool = True, trusted_proxies: str | None = None) -> PublicListener: + return PublicListener("0.0.0.0", 4000, 5, 40.0, proxy_headers, trusted_proxies) + + +@pytest.mark.parametrize( + ("environ", "listener", "expected"), + [ + ({}, _listener(), Supervised(17, "0.0.0.0", True, "127.0.0.1")), + ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(), Supervised(17, "0.0.0.0", True, "10.0.0.5")), + ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(True, "*"), Supervised(17, "0.0.0.0", True, "*")), + ({}, _listener(False), Supervised(17, "0.0.0.0", False, "127.0.0.1")), + ], +) +def test_python_child_keeps_the_trust_python_alone_would_use( + environ: dict[str, str], listener: PublicListener, expected: Supervised +) -> None: + spec: Final = python_spec(("python",), 17, 45555, listener, environ) + assert rust_forward_mode(spec.env) == expected + + +def test_python_child_listens_privately_and_keeps_options() -> None: + arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") + spec: Final = python_spec(arguments, 17, 45555, _listener(), {}) + params: Final = run_server.make_context("litellm", list(spec.argv)).params + assert params["host"] == "127.0.0.1" + assert params["port"] == 45555 + assert params["num_workers"] == 2 + assert params["reload"] is True + assert spec.pass_fds == (17,) + assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli") + + +def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None: + spec: Final = rust_spec( + "gateway", 45555, PublicListener("0.0.0.0", 4000, 2, 12.5, True, None), {"RUST_LOG": "info"} + ) + assert spec.argv == ("gateway",) + assert spec.pass_fds == () + assert dict(spec.env) == { + "RUST_LOG": "info", + "LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555", + "LITELLM_FORWARD_HOST": "0.0.0.0", + "LITELLM_FORWARD_PORT": "4000", + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE": "2", + "LITELLM_FORWARD_SHUTDOWN_DRAIN": "12.5", + } + + +def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + scripts: Final = tmp_path / "scripts" + elsewhere: Final = tmp_path / "elsewhere" + scripts.mkdir() + elsewhere.mkdir() + binary: Final = elsewhere / "litellm-gateway" + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o755) + monkeypatch.setenv("PATH", str(elsewhere)) + assert gateway_binary(str(scripts)) is None + assert gateway_binary(str(elsewhere)) == str(binary) + + +def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]: + def advance(acc: tuple[State, tuple[Signal, ...]], event: Event) -> tuple[State, tuple[Signal, ...]]: + state, sent = acc + following, action = step(state, event) + return following, sent if action is None else (*sent, action) + + return functools.reduce(advance, events, (Running(), ())) + + +RUST: Final = Child.RUST +PYTHON: Final = Child.PYTHON + + +@pytest.mark.parametrize( + ("events", "signals", "code"), + [ + ( + (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 0)), + (Signal(RUST, TERM), Signal(PYTHON, TERM)), + ExitCode.CLEAN, + ), + ((Exited(PYTHON, 0),), (), ExitCode.CHILD_FAILED), + ((Exited(RUST, 1),), (), ExitCode.CHILD_FAILED), + ((Stop(TERM), Exited(PYTHON, 0)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), + ((Stop(TERM), Exited(RUST, 1)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), + ( + (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 1)), + (Signal(RUST, TERM), Signal(PYTHON, TERM)), + ExitCode.CHILD_FAILED, + ), + ( + (Stop(INT), Stop(INT), Exited(RUST, 0), Exited(PYTHON, 0)), + (Signal(RUST, INT), Signal(PYTHON, INT)), + ExitCode.CLEAN, + ), + ( + (Exited(RUST, 1), Stop(TERM), Exited(PYTHON, 0)), + (), + ExitCode.CHILD_FAILED, + ), + ], +) +def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit( + events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode +) -> None: + assert _fold(events) == (Finished(code), signals) diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index db36f0d681f..0ce5cc58191 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -1550,7 +1550,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=30, timeout_worker_healthcheck=None, - listener_fd=None, + forwarded=None, ) mock_uvicorn_run.assert_called_once() @@ -1627,7 +1627,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=None, timeout_worker_healthcheck=15, - listener_fd=None, + forwarded=None, ) @patch("uvicorn.run") diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py deleted file mode 100644 index 1072148be9f..00000000000 --- a/tests/unit/proxy/test_rust_gateway.py +++ /dev/null @@ -1,293 +0,0 @@ -import asyncio -import functools -import signal -import sys -from pathlib import Path -from typing import TYPE_CHECKING, Final - -import pytest -from click.testing import CliRunner -from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware - -if TYPE_CHECKING: - from uvicorn._types import ( - ASGIReceiveCallable, - ASGISendCallable, - ASGISendEvent, - HTTPDisconnectEvent, - HTTPScope, - Scope, - ) - -from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server -from litellm.proxy.rust_gateway import ( - Child, - Disabled, - Event, - ExitCode, - Exited, - Finished, - Running, - RustPeerMiddleware, - Signal, - State, - Stop, - Supervised, - Supervisor, - cli_command, - gateway_binary, - python_spec, - rust_gateway_mode, - rust_spec, - startup_error, - step, -) - -TERM: Final = signal.SIGTERM -INT: Final = signal.SIGINT - - -@pytest.mark.parametrize( - "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] -) -def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("LITELLM_RUST", "true") - monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) - values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] - result: Final = CliRunner().invoke(run_server, [flag, *values]) - assert result.exit_code != 0 - assert f"LITELLM_RUST does not support {flag}" in result.output - - -@pytest.mark.parametrize( - ("environ", "expected"), - [ - ({}, Disabled()), - ({"LITELLM_RUST": "false"}, Disabled()), - ({"LITELLM_RUST": "off"}, Disabled()), - ({"LITELLM_RUST": "0"}, Disabled()), - ({"LITELLM_RUST": "true"}, Supervisor()), - ({"LITELLM_RUST": "1"}, Supervisor()), - ({"LITELLM_RUST": "yes"}, Supervisor()), - ({"LITELLM_RUST": "on"}, Supervisor()), - ({"LITELLM_RUST": " 1 "}, Supervisor()), - ({"LITELLM_RUST": "enabled"}, Disabled()), - ( - { - "LITELLM_RUST": "true", - "_LITELLM_GATEWAY_FD": "17", - "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", - "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", - }, - Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"), - ), - ], -) -def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None: - assert rust_gateway_mode(environ) == expected - - -@pytest.mark.parametrize( - ("environ", "expected"), - [ - ({}, Supervised(17, "0.0.0.0", True, "127.0.0.1")), - ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, Supervised(17, "0.0.0.0", True, "10.0.0.5")), - ({"FORWARDED_ALLOW_IPS": "*"}, Supervised(17, "0.0.0.0", True, "*")), - ({"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}, Supervised(17, "0.0.0.0", False, "127.0.0.1")), - ], -) -def test_python_child_keeps_the_trust_python_alone_would_use(environ: dict[str, str], expected: Supervised) -> None: - spec: Final = python_spec(("python",), 17, 45555, "0.0.0.0", environ) - assert rust_gateway_mode(spec.env) == expected - - -def test_python_child_listens_privately_and_keeps_options() -> None: - arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") - spec: Final = python_spec(arguments, 17, 45555, "0.0.0.0", {}) - params: Final = run_server.make_context("litellm", list(spec.argv)).params - assert params["host"] == "127.0.0.1" - assert params["port"] == 45555 - assert params["num_workers"] == 2 - assert params["reload"] is True - assert spec.pass_fds == (17,) - assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli") - - -def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None: - spec: Final = rust_spec("gateway", 45555, "0.0.0.0", 4000, {"RUST_LOG": "info"}) - assert spec.argv == ("gateway",) - assert spec.pass_fds == () - assert dict(spec.env) == { - "RUST_LOG": "info", - "HOST": "0.0.0.0", - "PORT": "4000", - "LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555", - } - - -def test_missing_binary_gives_install_instruction() -> None: - assert startup_error(None, {}) == ( - "LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment" - ) - - -def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - scripts: Final = tmp_path / "scripts" - elsewhere: Final = tmp_path / "elsewhere" - scripts.mkdir() - elsewhere.mkdir() - binary: Final = elsewhere / "litellm-gateway" - binary.write_text("#!/bin/sh\nexit 0\n") - binary.chmod(0o755) - monkeypatch.setenv("PATH", str(elsewhere)) - assert gateway_binary(str(scripts)) is None - assert gateway_binary(str(elsewhere)) == str(binary) - - -@pytest.mark.parametrize("listener_fd", [None, 17]) -def test_inherited_socket_reaches_uvicorn(listener_fd: int | None) -> None: - options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( - "127.0.0.1", 45555, listener_fd=listener_fd - ) - assert options.get("fd") == listener_fd - assert options.get("proxy_headers", True) is (listener_fd is None) - - -def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]: - def advance(acc: tuple[State, tuple[Signal, ...]], event: Event) -> tuple[State, tuple[Signal, ...]]: - state, sent = acc - following, action = step(state, event) - return following, sent if action is None else (*sent, action) - - return functools.reduce(advance, events, (Running(), ())) - - -RUST: Final = Child.RUST -PYTHON: Final = Child.PYTHON - - -@pytest.mark.parametrize( - ("events", "signals", "code"), - [ - ( - (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 0)), - (Signal(RUST, TERM), Signal(PYTHON, TERM)), - ExitCode.CLEAN, - ), - ((Exited(PYTHON, 0),), (), ExitCode.CHILD_FAILED), - ((Exited(RUST, 1),), (), ExitCode.CHILD_FAILED), - ((Stop(TERM), Exited(PYTHON, 0)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), - ((Stop(TERM), Exited(RUST, 1)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), - ( - (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 1)), - (Signal(RUST, TERM), Signal(PYTHON, TERM)), - ExitCode.CHILD_FAILED, - ), - ( - (Stop(INT), Stop(INT), Exited(RUST, 0), Exited(PYTHON, 0)), - (Signal(RUST, INT), Signal(PYTHON, INT)), - ExitCode.CLEAN, - ), - ( - (Exited(RUST, 1), Stop(TERM), Exited(PYTHON, 0)), - (), - ExitCode.CHILD_FAILED, - ), - ], -) -def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit( - events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode -) -> None: - assert _fold(events) == (Finished(code), signals) - - -Headers = tuple[tuple[bytes, bytes], ...] - - -async def _receive() -> "HTTPDisconnectEvent": - return {"type": "http.disconnect"} - - -async def _send(message: "ASGISendEvent") -> None: - return None - - -async def _seen_by_app( - behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised -) -> tuple[tuple[str, int] | None, str, Headers]: - seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = ( - asyncio.get_running_loop().create_future() - ) - - async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: - assert scope["type"] == "http" - seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"]))) - - direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe - app: Final = RustPeerMiddleware(probe, mode) if behind_rust else direct - scope: Final[HTTPScope] = { - "type": "http", - "asgi": {"version": "3.0", "spec_version": "2.3"}, - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": "/", - "raw_path": b"/", - "query_string": b"", - "root_path": "", - "headers": headers, - "client": client, - "server": ("127.0.0.1", 4000), - } - await app(scope, _receive, _send) - return seen.result() - - -PEER: Final = ("203.0.113.7", 51234) -TRUSTED_PEER: Final = ("10.0.12.34", 40000) -FORWARDED: Final[Headers] = ( - (b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"), - (b"x-forwarded-proto", b"https"), -) - - -@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)]) -@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))]) -@pytest.mark.parametrize( - "mode", - [ - Supervised(17, "0.0.0.0", True, "127.0.0.1"), - Supervised(17, "0.0.0.0", True, "10.0.0.0/8"), - Supervised(17, "0.0.0.0", True, "*"), - Supervised(17, "0.0.0.0", False, "10.0.0.0/8"), - ], -) -async def test_python_behind_rust_sees_what_a_direct_connection_would( - peer: tuple[str, int], headers: Headers, mode: Supervised -) -> None: - rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}" - behind_rust: Final = await _seen_by_app( - True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode - ) - direct: Final = await _seen_by_app(False, peer, headers, mode) - assert behind_rust == direct - - -async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None: - client, scheme, _ = await _seen_by_app( - True, - ("127.0.0.1", 60000), - ((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")), - Supervised(17, "0.0.0.0", True, "127.0.0.1"), - ) - assert (client, scheme) == (("203.0.113.7", 51234), "http") - - -async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None: - client, _, headers = await _seen_by_app( - True, - ("192.0.2.9", 1000), - ((b"x-litellm-rust-peer", b"10.0.12.34:1"),), - Supervised(17, "0.0.0.0", False, "127.0.0.1"), - ) - assert (client, headers) == (("192.0.2.9", 1000), ()) From 55dfc7208e0823be3a770b5f04b4c64541093935 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 13:39:56 +0900 Subject: [PATCH 15/15] fix(proxy): honor a factory app in the multi-worker reload options The supervised Python child serves rust_forward.asgi:app as a factory, so uvicorn's options carry factory=True. pool_config passed it to uvicorn.Config twice and `--reload --num_workers 2` crashed at startup under LITELLM_RUST Co-Authored-By: Claude Opus 5.5 --- litellm/proxy/dev_reload.py | 7 +++-- tests/unit/proxy/test_dev_reload.py | 41 +++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/dev_reload.py b/litellm/proxy/dev_reload.py index 500f581b34d..7b057271cb6 100644 --- a/litellm/proxy/dev_reload.py +++ b/litellm/proxy/dev_reload.py @@ -13,7 +13,7 @@ from socket import socket from typing import Final, Literal from pydantic import ConfigDict, ImportString, JsonValue, TypeAdapter, with_config -from typing_extensions import ReadOnly, Required, TypedDict +from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from uvicorn import Config, Server from uvicorn import config as uvicorn_config from uvicorn.supervisors import ChangeReload, Multiprocess @@ -42,6 +42,7 @@ class ServerOptions(TypedDict, total=False): @with_config(ConfigDict(extra="allow")) class ReloadOptions(ServerOptions): app: Required[ReadOnly[str]] + factory: NotRequired[ReadOnly[bool]] reload: ReadOnly[bool] reload_dirs: ReadOnly[tuple[str, ...]] reload_includes: ReadOnly[tuple[str, ...]] @@ -92,7 +93,9 @@ def load_worker_app(import_path: str, worker_app_factory: bool) -> Callable[..., def pool_config(options: ReloadOptions, workers: int, worker_overrides: Mapping[str, object] | None = None) -> Config: overrides: Final = worker_overrides or {} worker_app: Final = TypeAdapter(str).validate_python(overrides.get("app", options["app"])) - worker_app_factory: Final = TypeAdapter(bool).validate_python(overrides.get("factory", False)) + worker_app_factory: Final = TypeAdapter(bool).validate_python( + overrides.get("factory", options.get("factory", False)) + ) return Config( partial(load_worker_app, worker_app, worker_app_factory), factory=True, diff --git a/tests/unit/proxy/test_dev_reload.py b/tests/unit/proxy/test_dev_reload.py index 60a7c53820b..6650c1cd3e3 100644 --- a/tests/unit/proxy/test_dev_reload.py +++ b/tests/unit/proxy/test_dev_reload.py @@ -86,6 +86,39 @@ def test_a_broken_factory_exits_the_worker_as_a_startup_failure( assert exit_info.value.code == uvicorn_config.STARTUP_FAILURE +@pytest.mark.parametrize("worker_overrides", [None, {"factory": True}]) +def test_a_factory_flag_in_the_options_reaches_the_workers( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, worker_overrides: dict[str, object] | None +) -> None: + (tmp_path / "reload_factory_flag_app.py").write_text( + "async def served(scope, receive, send):\n" + " await send({'type': 'http.response.start', 'status': 204, 'headers': []})\n" + " await send({'type': 'http.response.body', 'body': b'', 'more_body': False})\n" + "\n" + "def factory():\n" + " return served\n" + ) + monkeypatch.syspath_prepend(str(tmp_path)) + config: Final = pool_config( + RELOAD_OPTIONS.validate_python({**_options("reload_factory_flag_app:factory"), "factory": True}), + workers=POOL_WORKERS, + worker_overrides=worker_overrides, + ) + config.load() + loaded_app: Final = config.loaded_app + assert loaded_app is not None + sent: Final[list[Message]] = [] + + async def receive() -> Message: + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(message: Message) -> None: + sent.append(message) + + asyncio.run(loaded_app({"type": "http", "method": "GET", "path": "/", "headers": []}, receive, send)) + assert sent[0] == {"type": "http.response.start", "status": 204, "headers": []} + + @REQUIRES_UVICORN_STARTUP_FAILURE @pytest.mark.parametrize( ("module", "source"), @@ -118,7 +151,9 @@ def test_the_reload_pool_defaults_the_worker_healthcheck_and_keeps_an_explicit_o explicit: Final = RELOAD_WORKER_HEALTHCHECK_SECONDS + 7 assert pool_config(_options("app:app"), workers=2).timeout_worker_healthcheck == RELOAD_WORKER_HEALTHCHECK_SECONDS assert ( - pool_config({**_options("app:app"), "timeout_worker_healthcheck": explicit}, workers=2).timeout_worker_healthcheck + pool_config( + {**_options("app:app"), "timeout_worker_healthcheck": explicit}, workers=2 + ).timeout_worker_healthcheck == explicit ) @@ -142,7 +177,9 @@ def test_the_reload_pool_accepts_and_keeps_every_uvicorn_arg_the_proxy_cli_build def test_the_reload_pool_passes_a_uvicorn_arg_it_does_not_list_through_to_the_workers() -> None: - config: Final = pool_config(RELOAD_OPTIONS.validate_python({**_options("app:app"), "backlog": 123}), workers=POOL_WORKERS) + config: Final = pool_config( + RELOAD_OPTIONS.validate_python({**_options("app:app"), "backlog": 123}), workers=POOL_WORKERS + ) assert (config.backlog, config.reload) == (123, False)