mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
feat(proxy): supervise the opt-in Rust forwarding server
This commit is contained in:
parent
cf48dc1769
commit
608a5c3e57
3 changed files with 211 additions and 1 deletions
|
|
@ -419,6 +419,7 @@ class ProxyInitializationHelpers:
|
|||
|
||||
import litellm
|
||||
from litellm._logging import _get_uvicorn_json_log_config, resolve_log_level
|
||||
from litellm.proxy.rust_gateway import inherited_fd
|
||||
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
|
||||
|
||||
uvicorn_args: Final = {
|
||||
|
|
@ -427,6 +428,7 @@ class ProxyInitializationHelpers:
|
|||
"port": port,
|
||||
"server_header": False,
|
||||
"timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(),
|
||||
**({"fd": inherited_fd()} if inherited_fd() is not None else {}),
|
||||
}
|
||||
if log_config is not None:
|
||||
print_startup_message(f"Using log_config: {log_config}")
|
||||
|
|
@ -752,7 +754,9 @@ class ProxyInitializationHelpers:
|
|||
)
|
||||
graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout()
|
||||
gunicorn_options: Final = {
|
||||
"bind": f"{host}:{port}",
|
||||
"bind": (
|
||||
f"fd://{os.environ['_LITELLM_GATEWAY_FD']}" if "_LITELLM_GATEWAY_FD" in os.environ else f"{host}:{port}"
|
||||
),
|
||||
"workers": num_workers, # default is 1
|
||||
"worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout),
|
||||
"graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS,
|
||||
|
|
@ -1039,6 +1043,13 @@ class ProxyInitializationHelpers:
|
|||
is_flag=True,
|
||||
help="Starts proxy via gunicorn, instead of uvicorn (better for managing multiple workers)",
|
||||
)
|
||||
@click.option(
|
||||
"--rust_gateway",
|
||||
envvar="LITELLM_RUST_GATEWAY",
|
||||
default=False,
|
||||
is_flag=True,
|
||||
help="Run the Rust forwarding server on the public port, with Python on loopback",
|
||||
)
|
||||
@click.option(
|
||||
"--run_hypercorn",
|
||||
default=False,
|
||||
|
|
@ -1241,6 +1252,7 @@ def run_server(
|
|||
setup,
|
||||
version,
|
||||
run_gunicorn,
|
||||
rust_gateway,
|
||||
run_hypercorn,
|
||||
run_granian,
|
||||
ssl_keyfile_path,
|
||||
|
|
@ -1280,6 +1292,27 @@ def run_server(
|
|||
run_setup_wizard()
|
||||
return
|
||||
|
||||
if rust_gateway and not any(
|
||||
(version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data)
|
||||
):
|
||||
from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise
|
||||
|
||||
binary: Final = gateway_binary()
|
||||
error: Final = startup_error(
|
||||
binary,
|
||||
{
|
||||
"--ssl_keyfile_path": ssl_keyfile_path,
|
||||
"--ssl_certfile_path": ssl_certfile_path,
|
||||
"--ciphers": ciphers,
|
||||
"--run_hypercorn": run_hypercorn,
|
||||
"--run_granian": run_granian,
|
||||
},
|
||||
)
|
||||
if error is not None:
|
||||
raise click.ClickException(error)
|
||||
assert binary is not None
|
||||
raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, port))
|
||||
|
||||
args: Final = locals()
|
||||
try:
|
||||
from litellm.proxy.proxy_server import (
|
||||
|
|
|
|||
115
litellm/proxy/rust_gateway.py
Normal file
115
litellm/proxy/rust_gateway.py
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import sys
|
||||
import sysconfig
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD"
|
||||
|
||||
|
||||
def gateway_binary() -> str | None:
|
||||
name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway"
|
||||
return shutil.which(name, path=sysconfig.get_path("scripts")) or shutil.which(name)
|
||||
|
||||
|
||||
def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None:
|
||||
flag: Final = next((name for name, value in unsupported.items() if value), None)
|
||||
if flag is not None:
|
||||
return f"--rust_gateway does not support {flag}"
|
||||
if os.name != "posix":
|
||||
return "--rust_gateway requires POSIX inherited sockets and process signals"
|
||||
if binary is None:
|
||||
return "--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment"
|
||||
return None
|
||||
|
||||
|
||||
def child_argv(argv: Sequence[str], port: int) -> tuple[str, ...]:
|
||||
return (*(arg for arg in argv if arg != "--rust_gateway"), "--host", "127.0.0.1", "--port", str(port))
|
||||
|
||||
|
||||
def inherited_fd() -> int | None:
|
||||
value: Final = os.environ.get(GATEWAY_FD)
|
||||
return int(value) if value is not None else None
|
||||
|
||||
|
||||
def terminate_group(process: asyncio.subprocess.Process) -> None:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
async def _supervise(binary: str, command: Sequence[str], host: str, port: int) -> int:
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
stopped: Final[asyncio.Future[signal.Signals]] = loop.create_future()
|
||||
|
||||
def stop(signum: signal.Signals) -> None:
|
||||
if not stopped.done():
|
||||
stopped.set_result(signum)
|
||||
|
||||
for signum in (signal.SIGTERM, signal.SIGINT):
|
||||
loop.add_signal_handler(signum, stop, signum)
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener:
|
||||
listener.bind(("127.0.0.1", 0))
|
||||
listener.listen()
|
||||
_, upstream_port = TypeAdapter(tuple[str, int]).validate_python(listener.getsockname())
|
||||
python: Final = await asyncio.create_subprocess_exec(
|
||||
*child_argv(command, upstream_port),
|
||||
env={
|
||||
**os.environ,
|
||||
"LITELLM_RUST_GATEWAY": "false",
|
||||
GATEWAY_FD: str(listener.fileno()),
|
||||
"FORWARDED_ALLOW_IPS": "127.0.0.1",
|
||||
},
|
||||
pass_fds=(listener.fileno(),),
|
||||
start_new_session=True,
|
||||
)
|
||||
try:
|
||||
rust: Final = await asyncio.create_subprocess_exec(
|
||||
binary,
|
||||
env={
|
||||
**os.environ,
|
||||
"HOST": host,
|
||||
"PORT": str(port),
|
||||
"LITELLM_FORWARD_UPSTREAM": f"http://127.0.0.1:{upstream_port}",
|
||||
},
|
||||
start_new_session=True,
|
||||
)
|
||||
except OSError:
|
||||
terminate_group(python)
|
||||
await python.wait()
|
||||
raise
|
||||
python_done: Final = asyncio.create_task(python.wait())
|
||||
rust_done: Final = asyncio.create_task(rust.wait())
|
||||
try:
|
||||
done, _ = await asyncio.wait((stopped, python_done, rust_done), return_when=asyncio.FIRST_COMPLETED)
|
||||
if python_done in done or rust_done in done:
|
||||
return 1
|
||||
rust.send_signal(stopped.result())
|
||||
draining, _ = await asyncio.wait((python_done, rust_done), return_when=asyncio.FIRST_COMPLETED)
|
||||
if python_done in draining or rust_done.result() != 0:
|
||||
return 1
|
||||
python.send_signal(stopped.result())
|
||||
return 0 if await python_done == 0 else 1
|
||||
finally:
|
||||
terminate_group(rust)
|
||||
terminate_group(python)
|
||||
await asyncio.gather(python_done, rust_done)
|
||||
|
||||
|
||||
def supervise(binary: str, command: Sequence[str], host: str, port: int) -> int:
|
||||
try:
|
||||
return asyncio.run(_supervise(binary, command, host, port))
|
||||
except OSError as error:
|
||||
sys.stderr.write(f"Rust gateway startup failed: {error}\n")
|
||||
return 1
|
||||
|
||||
|
||||
def cli_command(argv: Sequence[str]) -> tuple[str, ...]:
|
||||
return (sys.executable, "-m", "litellm.proxy.proxy_cli", *argv)
|
||||
62
tests/unit/proxy/test_rust_gateway.py
Normal file
62
tests/unit/proxy/test_rust_gateway.py
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server
|
||||
from litellm.proxy.rust_gateway import child_argv, cli_command, gateway_binary, inherited_fd, startup_error
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"]
|
||||
)
|
||||
@pytest.mark.parametrize("enable", ["flag", "environment"])
|
||||
def test_unsupported_modes_fail_before_starting_children(
|
||||
flag: str, enable: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("LITELLM_RUST_GATEWAY", "true" if enable == "environment" else "false")
|
||||
arguments: Final = ([] if enable == "environment" else ["--rust_gateway"]) + [flag]
|
||||
values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else []
|
||||
result: Final = CliRunner().invoke(run_server, arguments + values)
|
||||
assert result.exit_code != 0
|
||||
assert f"--rust_gateway does not support {flag}" in result.output
|
||||
|
||||
|
||||
def test_child_arguments_preserve_options_and_force_private_listener() -> None:
|
||||
arguments: Final = ("--rust_gateway", "--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload")
|
||||
params: Final = run_server.make_context("litellm", list(child_argv(arguments, 45555))).params
|
||||
assert params["host"] == "127.0.0.1"
|
||||
assert params["port"] == 45555
|
||||
assert params["num_workers"] == 2
|
||||
assert params["reload"] is True
|
||||
assert params["rust_gateway"] is False
|
||||
assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli")
|
||||
|
||||
|
||||
def test_missing_binary_gives_install_instruction() -> None:
|
||||
assert startup_error(None, {}) == (
|
||||
"--rust_gateway requires the litellm-gateway binary. Install litellm[proxy] in this environment"
|
||||
)
|
||||
|
||||
|
||||
def test_current_environment_binary_wins_over_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
binary: Final = tmp_path / "litellm-gateway"
|
||||
binary.write_text("#!/bin/sh\nexit 0\n")
|
||||
binary.chmod(0o755)
|
||||
monkeypatch.setattr("sysconfig.get_path", lambda _: str(tmp_path))
|
||||
monkeypatch.setenv("PATH", "")
|
||||
assert gateway_binary() == str(binary)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fd", [None, "17"])
|
||||
def test_inherited_socket_reaches_uvicorn(fd: str | None, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
if fd is None:
|
||||
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("_LITELLM_GATEWAY_FD", fd)
|
||||
expected: Final = int(fd) if fd is not None else None
|
||||
options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555)
|
||||
assert inherited_fd() == expected
|
||||
assert options.get("fd") == expected
|
||||
Loading…
Add table
Reference in a new issue