feat(proxy): supervise the opt-in Rust forwarding server

This commit is contained in:
Devin AI 2026-11-01 23:11:18 +00:00 • committed by Yujong Lee
parent cf48dc1769
commit 608a5c3e57
3 changed files with 211 additions and 1 deletions

View file

@ -419,6 +419,7 @@ class ProxyInitializationHelpers:
import litellm
from litellm._logging import _get_uvicorn_json_log_config, resolve_log_level
from litellm.proxy.rust_gateway import inherited_fd
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
uvicorn_args: Final = {
@ -427,6 +428,7 @@ class ProxyInitializationHelpers:
"port": port,
"server_header": False,
"timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(),
**({"fd": inherited_fd()} if inherited_fd() is not None else {}),
}
if log_config is not None:
print_startup_message(f"Using log_config: {log_config}")
@ -752,7 +754,9 @@ class ProxyInitializationHelpers:
)
graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout()
gunicorn_options: Final = {
"bind": f"{host}:{port}",
"bind": (
f"fd://{os.environ['_LITELLM_GATEWAY_FD']}" if "_LITELLM_GATEWAY_FD" in os.environ else f"{host}:{port}"
),
"workers": num_workers, # default is 1
"worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout),
"graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS,
@ -1039,6 +1043,13 @@ class ProxyInitializationHelpers:
is_flag=True,
help="Starts proxy via gunicorn, instead of uvicorn (better for managing multiple workers)",
)
@click.option(
"--rust_gateway",
envvar="LITELLM_RUST_GATEWAY",
default=False,
is_flag=True,
help="Run the Rust forwarding server on the public port, with Python on loopback",
)
@click.option(
"--run_hypercorn",
default=False,
@ -1241,6 +1252,7 @@ def run_server(
setup,
version,
run_gunicorn,
rust_gateway,
run_hypercorn,
run_granian,
ssl_keyfile_path,
@ -1280,6 +1292,27 @@ def run_server(
run_setup_wizard()
return
if rust_gateway and not any(
(version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data)
):
from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise
binary: Final = gateway_binary()
error: Final = startup_error(
binary,
{
"--ssl_keyfile_path": ssl_keyfile_path,
"--ssl_certfile_path": ssl_certfile_path,
"--ciphers": ciphers,
"--run_hypercorn": run_hypercorn,
"--run_granian": run_granian,
},
)
if error is not None:
raise click.ClickException(error)
assert binary is not None
raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, port))
args: Final = locals()
try:
from litellm.proxy.proxy_server import (

View file

@ -0,0 +1,115 @@
import asyncio
import os
import shutil
import signal
import socket
import sys
import sysconfig
from collections.abc import Mapping, Sequence
from typing import Final
from pydantic import TypeAdapter
GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD"
def gateway_binary() -> str | None:
name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway"
return shutil.which(name, path=sysconfig.get_path("scripts")) or shutil.which(name)
def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None:
flag: Final = next((name for name, value in unsupported.items() if value), None)
if flag is not None:
return f"--rust_gateway does not support {flag}"
if os.name != "posix":
return "--rust_gateway requires POSIX inherited sockets and process signals"
if binary is None:
return "--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment"
return None
def child_argv(argv: Sequence[str], port: int) -> tuple[str, ...]:
return (*(arg for arg in argv if arg != "--rust_gateway"), "--host", "127.0.0.1", "--port", str(port))
def inherited_fd() -> int | None:
value: Final = os.environ.get(GATEWAY_FD)
return int(value) if value is not None else None
def terminate_group(process: asyncio.subprocess.Process) -> None:
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
pass
async def _supervise(binary: str, command: Sequence[str], host: str, port: int) -> int:
loop: Final = asyncio.get_running_loop()
stopped: Final[asyncio.Future[signal.Signals]] = loop.create_future()
def stop(signum: signal.Signals) -> None:
if not stopped.done():
stopped.set_result(signum)
for signum in (signal.SIGTERM, signal.SIGINT):
loop.add_signal_handler(signum, stop, signum)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener:
listener.bind(("127.0.0.1", 0))
listener.listen()
_, upstream_port = TypeAdapter(tuple[str, int]).validate_python(listener.getsockname())
python: Final = await asyncio.create_subprocess_exec(
*child_argv(command, upstream_port),
env={
**os.environ,
"LITELLM_RUST_GATEWAY": "false",
GATEWAY_FD: str(listener.fileno()),
"FORWARDED_ALLOW_IPS": "127.0.0.1",
},
pass_fds=(listener.fileno(),),
start_new_session=True,
)
try:
rust: Final = await asyncio.create_subprocess_exec(
binary,
env={
**os.environ,
"HOST": host,
"PORT": str(port),
"LITELLM_FORWARD_UPSTREAM": f"http://127.0.0.1:{upstream_port}",
},
start_new_session=True,
)
except OSError:
terminate_group(python)
await python.wait()
raise
python_done: Final = asyncio.create_task(python.wait())
rust_done: Final = asyncio.create_task(rust.wait())
try:
done, _ = await asyncio.wait((stopped, python_done, rust_done), return_when=asyncio.FIRST_COMPLETED)
if python_done in done or rust_done in done:
return 1
rust.send_signal(stopped.result())
draining, _ = await asyncio.wait((python_done, rust_done), return_when=asyncio.FIRST_COMPLETED)
if python_done in draining or rust_done.result() != 0:
return 1
python.send_signal(stopped.result())
return 0 if await python_done == 0 else 1
finally:
terminate_group(rust)
terminate_group(python)
await asyncio.gather(python_done, rust_done)
def supervise(binary: str, command: Sequence[str], host: str, port: int) -> int:
try:
return asyncio.run(_supervise(binary, command, host, port))
except OSError as error:
sys.stderr.write(f"Rust gateway startup failed: {error}\n")
return 1
def cli_command(argv: Sequence[str]) -> tuple[str, ...]:
return (sys.executable, "-m", "litellm.proxy.proxy_cli", *argv)

View file

@ -0,0 +1,62 @@
import sys
from pathlib import Path
from typing import Final
import pytest
from click.testing import CliRunner
from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server
from litellm.proxy.rust_gateway import child_argv, cli_command, gateway_binary, inherited_fd, startup_error
@pytest.mark.parametrize(
"flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"]
)
@pytest.mark.parametrize("enable", ["flag", "environment"])
def test_unsupported_modes_fail_before_starting_children(
flag: str, enable: str, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("LITELLM_RUST_GATEWAY", "true" if enable == "environment" else "false")
arguments: Final = ([] if enable == "environment" else ["--rust_gateway"]) + [flag]
values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else []
result: Final = CliRunner().invoke(run_server, arguments + values)
assert result.exit_code != 0
assert f"--rust_gateway does not support {flag}" in result.output
def test_child_arguments_preserve_options_and_force_private_listener() -> None:
arguments: Final = ("--rust_gateway", "--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload")
params: Final = run_server.make_context("litellm", list(child_argv(arguments, 45555))).params
assert params["host"] == "127.0.0.1"
assert params["port"] == 45555
assert params["num_workers"] == 2
assert params["reload"] is True
assert params["rust_gateway"] is False
assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli")
def test_missing_binary_gives_install_instruction() -> None:
assert startup_error(None, {}) == (
"--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment"
)
def test_current_environment_binary_wins_over_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
binary: Final = tmp_path / "litellm-gateway"
binary.write_text("#!/bin/sh\nexit 0\n")
binary.chmod(0o755)
monkeypatch.setattr("sysconfig.get_path", lambda _: str(tmp_path))
monkeypatch.setenv("PATH", "")
assert gateway_binary() == str(binary)
@pytest.mark.parametrize("fd", [None, "17"])
def test_inherited_socket_reaches_uvicorn(fd: str | None, monkeypatch: pytest.MonkeyPatch) -> None:
if fd is None:
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
else:
monkeypatch.setenv("_LITELLM_GATEWAY_FD", fd)
expected: Final = int(fd) if fd is not None else None
options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555)
assert inherited_fd() == expected
assert options.get("fd") == expected