diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 0fbfa0b06a7..94878bd1af2 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -419,6 +419,7 @@ class ProxyInitializationHelpers: import litellm from litellm._logging import _get_uvicorn_json_log_config, resolve_log_level + from litellm.proxy.rust_gateway import inherited_fd from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager uvicorn_args: Final = { @@ -427,6 +428,7 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), + **({"fd": inherited_fd()} if inherited_fd() is not None else {}), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -752,7 +754,9 @@ class ProxyInitializationHelpers: ) graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout() gunicorn_options: Final = { - "bind": f"{host}:{port}", + "bind": ( + f"fd://{os.environ['_LITELLM_GATEWAY_FD']}" if "_LITELLM_GATEWAY_FD" in os.environ else f"{host}:{port}" + ), "workers": num_workers, # default is 1 "worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, @@ -1039,6 +1043,13 @@ class ProxyInitializationHelpers: is_flag=True, help="Starts proxy via gunicorn, instead of uvicorn (better for managing multiple workers)", ) +@click.option( + "--rust_gateway", + envvar="LITELLM_RUST_GATEWAY", + default=False, + is_flag=True, + help="Run the Rust forwarding server on the public port, with Python on loopback", +) @click.option( "--run_hypercorn", default=False, @@ -1241,6 +1252,7 @@ def run_server( setup, version, run_gunicorn, + rust_gateway, run_hypercorn, run_granian, ssl_keyfile_path, @@ -1280,6 +1292,27 @@ def run_server( run_setup_wizard() return + if rust_gateway and not any( + (version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data) + ): + from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise + + binary: Final = gateway_binary() + error: Final = startup_error( + binary, + { + "--ssl_keyfile_path": ssl_keyfile_path, + "--ssl_certfile_path": ssl_certfile_path, + "--ciphers": ciphers, + "--run_hypercorn": run_hypercorn, + "--run_granian": run_granian, + }, + ) + if error is not None: + raise click.ClickException(error) + assert binary is not None + raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, port)) + args: Final = locals() try: from litellm.proxy.proxy_server import ( diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py new file mode 100644 index 00000000000..830a4ab3d3d --- /dev/null +++ b/litellm/proxy/rust_gateway.py @@ -0,0 +1,115 @@ +import asyncio +import os +import shutil +import signal +import socket +import sys +import sysconfig +from collections.abc import Mapping, Sequence +from typing import Final + +from pydantic import TypeAdapter + +GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" + + +def gateway_binary() -> str | None: + name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway" + return shutil.which(name, path=sysconfig.get_path("scripts")) or shutil.which(name) + + +def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: + flag: Final = next((name for name, value in unsupported.items() if value), None) + if flag is not None: + return f"--rust_gateway does not support {flag}" + if os.name != "posix": + return "--rust_gateway requires POSIX inherited sockets and process signals" + if binary is None: + return "--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment" + return None + + +def child_argv(argv: Sequence[str], port: int) -> tuple[str, ...]: + return (*(arg for arg in argv if arg != "--rust_gateway"), "--host", "127.0.0.1", "--port", str(port)) + + +def inherited_fd() -> int | None: + value: Final = os.environ.get(GATEWAY_FD) + return int(value) if value is not None else None + + +def terminate_group(process: asyncio.subprocess.Process) -> None: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + + +async def _supervise(binary: str, command: Sequence[str], host: str, port: int) -> int: + loop: Final = asyncio.get_running_loop() + stopped: Final[asyncio.Future[signal.Signals]] = loop.create_future() + + def stop(signum: signal.Signals) -> None: + if not stopped.done(): + stopped.set_result(signum) + + for signum in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(signum, stop, signum) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen() + _, upstream_port = TypeAdapter(tuple[str, int]).validate_python(listener.getsockname()) + python: Final = await asyncio.create_subprocess_exec( + *child_argv(command, upstream_port), + env={ + **os.environ, + "LITELLM_RUST_GATEWAY": "false", + GATEWAY_FD: str(listener.fileno()), + "FORWARDED_ALLOW_IPS": "127.0.0.1", + }, + pass_fds=(listener.fileno(),), + start_new_session=True, + ) + try: + rust: Final = await asyncio.create_subprocess_exec( + binary, + env={ + **os.environ, + "HOST": host, + "PORT": str(port), + "LITELLM_FORWARD_UPSTREAM": f"http://127.0.0.1:{upstream_port}", + }, + start_new_session=True, + ) + except OSError: + terminate_group(python) + await python.wait() + raise + python_done: Final = asyncio.create_task(python.wait()) + rust_done: Final = asyncio.create_task(rust.wait()) + try: + done, _ = await asyncio.wait((stopped, python_done, rust_done), return_when=asyncio.FIRST_COMPLETED) + if python_done in done or rust_done in done: + return 1 + rust.send_signal(stopped.result()) + draining, _ = await asyncio.wait((python_done, rust_done), return_when=asyncio.FIRST_COMPLETED) + if python_done in draining or rust_done.result() != 0: + return 1 + python.send_signal(stopped.result()) + return 0 if await python_done == 0 else 1 + finally: + terminate_group(rust) + terminate_group(python) + await asyncio.gather(python_done, rust_done) + + +def supervise(binary: str, command: Sequence[str], host: str, port: int) -> int: + try: + return asyncio.run(_supervise(binary, command, host, port)) + except OSError as error: + sys.stderr.write(f"Rust gateway startup failed: {error}\n") + return 1 + + +def cli_command(argv: Sequence[str]) -> tuple[str, ...]: + return (sys.executable, "-m", "litellm.proxy.proxy_cli", *argv) diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py new file mode 100644 index 00000000000..c9992d07d14 --- /dev/null +++ b/tests/unit/proxy/test_rust_gateway.py @@ -0,0 +1,62 @@ +import sys +from pathlib import Path +from typing import Final + +import pytest +from click.testing import CliRunner + +from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server +from litellm.proxy.rust_gateway import child_argv, cli_command, gateway_binary, inherited_fd, startup_error + + +@pytest.mark.parametrize( + "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] +) +@pytest.mark.parametrize("enable", ["flag", "environment"]) +def test_unsupported_modes_fail_before_starting_children( + flag: str, enable: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("LITELLM_RUST_GATEWAY", "true" if enable == "environment" else "false") + arguments: Final = ([] if enable == "environment" else ["--rust_gateway"]) + [flag] + values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] + result: Final = CliRunner().invoke(run_server, arguments + values) + assert result.exit_code != 0 + assert f"--rust_gateway does not support {flag}" in result.output + + +def test_child_arguments_preserve_options_and_force_private_listener() -> None: + arguments: Final = ("--rust_gateway", "--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") + params: Final = run_server.make_context("litellm", list(child_argv(arguments, 45555))).params + assert params["host"] == "127.0.0.1" + assert params["port"] == 45555 + assert params["num_workers"] == 2 + assert params["reload"] is True + assert params["rust_gateway"] is False + assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli") + + +def test_missing_binary_gives_install_instruction() -> None: + assert startup_error(None, {}) == ( + "--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment" + ) + + +def test_current_environment_binary_wins_over_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + binary: Final = tmp_path / "litellm-gateway" + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o755) + monkeypatch.setattr("sysconfig.get_path", lambda _: str(tmp_path)) + monkeypatch.setenv("PATH", "") + assert gateway_binary() == str(binary) + + +@pytest.mark.parametrize("fd", [None, "17"]) +def test_inherited_socket_reaches_uvicorn(fd: str | None, monkeypatch: pytest.MonkeyPatch) -> None: + if fd is None: + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + else: + monkeypatch.setenv("_LITELLM_GATEWAY_FD", fd) + expected: Final = int(fd) if fd is not None else None + options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555) + assert inherited_fd() == expected + assert options.get("fd") == expected