This commit is contained in:
yujonglee 2026-11-03 05:03:47 +00:00 • committed by GitHub
commit afef065f7a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 1312 additions and 35 deletions

View file

@ -583,6 +583,7 @@ jobs:
tests/unit/proxy/credential_endpoints
tests/unit/proxy/discovery_endpoints
tests/unit/proxy/shutdown
tests/unit/proxy/rust_forward
tests/unit/proxy/public_endpoints
tests/unit/proxy/prompts
tests/unit/proxy/rag_endpoints

View file

@ -19,29 +19,43 @@ Run with:
import os
import sys
import sysconfig
from collections.abc import Callable, Mapping, Sequence
from typing import TYPE_CHECKING, Final
from typing_extensions import assert_never
from uvicorn.main import main as uvicorn_main
from litellm_cli.main import without_gc_pauses
if TYPE_CHECKING:
from fastapi import FastAPI
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings
from litellm.proxy.rust_forward.listener import PublicListener
from litellm.proxy.rust_forward.mode import ServerOverrides
from litellm.proxy.rust_forward.peer import ASGIApp
GATEWAY_APP: Final = "gateway.launch:worker_app"
KEEPALIVE_FLAG: Final = "--timeout-keep-alive"
def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str, ...]:
def uvicorn_argv(
argv: Sequence[str], environ: Mapping[str, str], forwarded: "ServerOverrides | None"
) -> tuple[str, ...]:
"""Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly."""
keepalive: Final = environ.get("KEEPALIVE_TIMEOUT")
inherited_socket: Final = (
()
if forwarded is None
else (
"--fd",
str(forwarded.fd),
"--proxy-headers" if forwarded.proxy_headers else "--no-proxy-headers",
)
)
if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv):
return (GATEWAY_APP, "--factory", *argv)
return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive)
return (GATEWAY_APP, "--factory", *argv, *inherited_socket)
return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive, *inherited_socket)
def pool_database_url(
@ -65,17 +79,54 @@ def pool_database_url(
return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth())
def worker_app() -> "FastAPI":
def worker_app() -> "ASGIApp":
with without_gc_pauses():
from gateway.main import app
return app
from litellm.proxy.rust_forward.mode import rust_forward_mode
from litellm.proxy.rust_forward.peer import wrap
return wrap(app, rust_forward_mode(os.environ))
def _serve(argv: Sequence[str]) -> None:
uvicorn_main(tuple(argv), prog_name="uvicorn")
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
def public_listener(argv: Sequence[str], binary: str | None) -> "tuple[str, PublicListener]":
"""Validate the uvicorn flags for Rust forwarding and return the binary and the public listener."""
from pydantic import TypeAdapter
from litellm.proxy.rust_forward.listener import PublicListener, StartupError, shutdown_drain_seconds, startup_check
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
params: Final[Mapping[str, object]] = uvicorn_main.make_context(
"uvicorn", list(uvicorn_argv(argv, os.environ, None))
).params
checked: Final = startup_check(
binary,
{
"--ssl-keyfile": params["ssl_keyfile"],
"--ssl-certfile": params["ssl_certfile"],
"--ssl-ciphers": any(arg.split("=")[0] == "--ssl-ciphers" for arg in argv),
"--uds": params["uds"] is not None,
"--fd": params["fd"] is not None,
},
)
if isinstance(checked, StartupError):
sys.exit(checked.reason)
graceful_timeout: Final = TypeAdapter[int | None](int | None).validate_python(params["timeout_graceful_shutdown"])
return checked, PublicListener(
host=TypeAdapter(str).validate_python(params["host"]),
port=TypeAdapter(int).validate_python(params["port"]),
keepalive_seconds=TypeAdapter(int).validate_python(params["timeout_keep_alive"]),
shutdown_drain_seconds=shutdown_drain_seconds(
GracefulShutdownManager.get_timeout() if graceful_timeout is None else graceful_timeout
),
proxy_headers=TypeAdapter(bool).validate_python(params["proxy_headers"]),
trusted_proxies=TypeAdapter[str | None](str | None).validate_python(params["forwarded_allow_ips"]),
)
def configure_database() -> None:
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url
@ -86,7 +137,26 @@ def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -
sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}")
if pooled_url is not None:
export_pooled_database_url(pooled_url)
serve(uvicorn_argv(argv, os.environ))
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides
from litellm.proxy.rust_forward.supervisor import gateway_binary, supervise
mode: Final = rust_forward_mode(os.environ)
match mode:
case Disabled():
configure_database()
serve(uvicorn_argv(argv, os.environ, None))
case Supervisor():
binary, listener = public_listener(argv, gateway_binary(sysconfig.get_path("scripts")))
configure_database()
command: Final = (sys.executable, "-m", "gateway.launch", *argv)
sys.exit(supervise(binary, command, listener, os.environ))
case Supervised():
serve(uvicorn_argv(argv, os.environ, server_overrides(mode)))
case _:
assert_never(mode)
if __name__ == "__main__":

View file

@ -13,7 +13,7 @@ from socket import socket
from typing import Final, Literal
from pydantic import ConfigDict, ImportString, JsonValue, TypeAdapter, with_config
from typing_extensions import ReadOnly, Required, TypedDict
from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
from uvicorn import Config, Server
from uvicorn import config as uvicorn_config
from uvicorn.supervisors import ChangeReload, Multiprocess
@ -42,6 +42,7 @@ class ServerOptions(TypedDict, total=False):
@with_config(ConfigDict(extra="allow"))
class ReloadOptions(ServerOptions):
app: Required[ReadOnly[str]]
factory: NotRequired[ReadOnly[bool]]
reload: ReadOnly[bool]
reload_dirs: ReadOnly[tuple[str, ...]]
reload_includes: ReadOnly[tuple[str, ...]]
@ -92,7 +93,9 @@ def load_worker_app(import_path: str, worker_app_factory: bool) -> Callable[...,
def pool_config(options: ReloadOptions, workers: int, worker_overrides: Mapping[str, object] | None = None) -> Config:
overrides: Final = worker_overrides or {}
worker_app: Final = TypeAdapter(str).validate_python(overrides.get("app", options["app"]))
worker_app_factory: Final = TypeAdapter(bool).validate_python(overrides.get("factory", False))
worker_app_factory: Final = TypeAdapter(bool).validate_python(
overrides.get("factory", options.get("factory", False))
)
return Config(
partial(load_worker_app, worker_app, worker_app_factory),
factory=True,

View file

@ -8,6 +8,7 @@ import random
import re
import subprocess
import sys
import sysconfig
import urllib.parse as urlparse
from collections.abc import Callable, Iterable, Mapping, Sequence
from pathlib import Path
@ -35,6 +36,10 @@ from litellm.types.llms.base import LiteLLMBaseModel
if TYPE_CHECKING:
from fastapi import FastAPI
from litellm.proxy.rust_forward.listener import PublicListener
from litellm.proxy.rust_forward.mode import ServerOverrides
from litellm.proxy.rust_forward.peer import ASGIApp
else:
FastAPI = Any
@ -261,6 +266,7 @@ def _apply_config_logging_settings(litellm_settings: StartupOutputSettings | Non
WORKER_APP: Final = "litellm_cli.main:proxy_worker_app"
FORWARD_APP: Final = "litellm.proxy.rust_forward.asgi:app"
WORKER_LOG_CONFIG_ENV: Final = "LITELLM_WORKER_LOG_CONFIG"
_LOG_CONFIG: Final = TypeAdapter[str | Mapping[str, object]](str | Mapping[str, object])
_LOG_CONFIG_FILE: Final = TypeAdapter[Mapping[str, object]](Mapping[str, object])
@ -389,12 +395,12 @@ class ProxyInitializationHelpers:
logging.config.fileConfig(log_config, disable_existing_loggers=False)
@staticmethod
def child_process_uvicorn_overrides(log_config: object) -> Mapping[str, object]:
def child_process_uvicorn_overrides(log_config: object, app: str = WORKER_APP) -> Mapping[str, object]:
if log_config is None:
return MappingProxyType({"app": WORKER_APP, "factory": True})
return MappingProxyType({"app": app, "factory": True})
ProxyInitializationHelpers.apply_log_config(_LOG_CONFIG.validate_python(log_config))
os.environ[WORKER_LOG_CONFIG_ENV] = json.dumps(log_config)
return MappingProxyType({"app": WORKER_APP, "factory": True, "log_config": None})
return MappingProxyType({"app": app, "factory": True, "log_config": None})
@staticmethod
def apply_worker_log_config(environ: Mapping[str, str]) -> None:
@ -409,6 +415,7 @@ class ProxyInitializationHelpers:
log_config: str | None = None,
keepalive_timeout: int | None = None,
timeout_worker_healthcheck: int | None = None,
forwarded: "ServerOverrides | None" = None,
) -> dict:
"""
Get the arguments for `uvicorn` worker
@ -427,6 +434,16 @@ class ProxyInitializationHelpers:
"port": port,
"server_header": False,
"timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(),
**(
{}
if forwarded is None
else {
"app": FORWARD_APP,
"factory": True,
"fd": forwarded.fd,
"proxy_headers": forwarded.proxy_headers,
}
),
}
if log_config is not None:
print_startup_message(f"Using log_config: {log_config}")
@ -674,12 +691,13 @@ class ProxyInitializationHelpers:
def run_gunicorn_server(
host: str,
port: int,
app: FastAPI,
app: "ASGIApp",
num_workers: int,
ssl_certfile_path: str,
ssl_keyfile_path: str,
max_requests_before_restart: int | None = None,
max_requests_before_restart_jitter: int | None = None,
forwarded: "ServerOverrides | None" = None,
):
"""
Run litellm with `gunicorn`
@ -752,9 +770,11 @@ class ProxyInitializationHelpers:
)
graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout()
gunicorn_options: Final = {
"bind": f"{host}:{port}",
"bind": f"{host}:{port}" if forwarded is None else f"fd://{forwarded.fd}",
"workers": num_workers, # default is 1
"worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout),
"worker_class": build_graceful_shutdown_uvicorn_worker(
graceful_shutdown_timeout, proxy_headers=True if forwarded is None else forwarded.proxy_headers
),
"graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS,
"preload": True, # Add the preload flag,
"accesslog": "-", # Log to stdout
@ -815,7 +835,7 @@ class ProxyInitializationHelpers:
_run_ollama_serve = run_ollama_serve
@staticmethod
def is_port_in_use(port):
def is_port_in_use(port: int):
import socket
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
@ -823,6 +843,34 @@ class ProxyInitializationHelpers:
_is_port_in_use = is_port_in_use
@staticmethod
def public_listener(host: str, port: int, keepalive_timeout: int | None, run_gunicorn: bool) -> "PublicListener":
"""``run_gunicorn_server`` never hands ``--keepalive_timeout`` to gunicorn, so its workers keep gunicorn's."""
from gunicorn.config import Config as GunicornConfig
from litellm.proxy.rust_forward.listener import (
PublicListener,
shutdown_drain_seconds,
uvicorn_default_keepalive,
)
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
keepalive: Final = (
TypeAdapter(int).validate_python(GunicornConfig().keepalive)
if run_gunicorn
else (uvicorn_default_keepalive() if keepalive_timeout is None else keepalive_timeout)
)
return PublicListener(
host=host,
port=random.randint(1024, 49152)
if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port)
else port,
keepalive_seconds=keepalive,
shutdown_drain_seconds=shutdown_drain_seconds(GracefulShutdownManager.get_timeout()),
proxy_headers=True,
trusted_proxies=None,
)
@staticmethod
def get_loop_type():
"""Helper function to determine the event loop type based on platform"""
@ -1210,8 +1258,8 @@ class ProxyInitializationHelpers:
)
def run_server(
cli_args,
host,
port,
host: str,
port: int,
api_base,
api_version,
model,
@ -1280,6 +1328,31 @@ def run_server(
run_setup_wizard()
return
from litellm.proxy.rust_forward.mode import Supervised, Supervisor, rust_forward_mode, server_overrides
forward_mode: Final = rust_forward_mode(os.environ)
if isinstance(forward_mode, Supervisor) and not any(
(version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data)
):
from litellm.proxy.rust_forward.listener import StartupError, startup_check
from litellm.proxy.rust_forward.supervisor import cli_command, gateway_binary, supervise
binary: Final = startup_check(
gateway_binary(sysconfig.get_path("scripts")),
{
"--ssl_keyfile_path": ssl_keyfile_path,
"--ssl_certfile_path": ssl_certfile_path,
"--ciphers": ciphers,
"--run_hypercorn": run_hypercorn,
"--run_granian": run_granian,
},
)
if isinstance(binary, StartupError):
raise click.ClickException(binary.reason)
listener: Final = ProxyInitializationHelpers.public_listener(host, port, keepalive_timeout, run_gunicorn)
raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), listener, os.environ))
forwarded: Final = server_overrides(forward_mode) if isinstance(forward_mode, Supervised) else None
args: Final = locals()
try:
from litellm.proxy.proxy_server import (
@ -1729,14 +1802,15 @@ def run_server(
if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None:
from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process
metrics_bind_host: Final = host if forwarded is None else forwarded.metrics_host
try:
metrics_process: Final = start_metrics_server_process(
host=host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir
host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir
)
except MetricsServerStartupError as error:
raise click.ClickException(str(error)) from error
print_startup_message(
f"\033[1;32mLiteLLM: Serving Prometheus metrics on {host}:{prometheus_metrics_port}/metrics "
f"\033[1;32mLiteLLM: Serving Prometheus metrics on {metrics_bind_host}:{prometheus_metrics_port}/metrics "
f"(pid {metrics_process.pid})\033[0m"
)
@ -1747,6 +1821,7 @@ def run_server(
log_config=log_config,
keepalive_timeout=keepalive_timeout,
timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None),
forwarded=forwarded,
)
# Optional: recycle uvicorn workers after N requests
if max_requests_before_restart is not None:
@ -1786,7 +1861,7 @@ def run_server(
start_query_engine_reaper()
if num_workers > 1 or reload:
worker_overrides: Final = ProxyInitializationHelpers.child_process_uvicorn_overrides(
uvicorn_args.get("log_config")
uvicorn_args.get("log_config"), WORKER_APP if forwarded is None else FORWARD_APP
)
if reload and num_workers > 1:
from litellm.proxy.dev_reload import (
@ -1812,15 +1887,18 @@ def run_server(
workers=num_workers,
)
elif run_gunicorn is True:
from litellm.proxy.rust_forward.peer import wrap
ProxyInitializationHelpers.run_gunicorn_server(
host=host,
port=port,
app=app,
app=wrap(app, forward_mode),
num_workers=num_workers,
ssl_certfile_path=ssl_certfile_path,
ssl_keyfile_path=ssl_keyfile_path,
max_requests_before_restart=max_requests_before_restart,
max_requests_before_restart_jitter=max_requests_before_restart_jitter,
forwarded=forwarded,
)
elif run_hypercorn is True:
ProxyInitializationHelpers.init_hypercorn_server(

View file

@ -0,0 +1,8 @@
# Rust forwarding, Python half
With `LITELLM_RUST` on, the supervisor here runs the Rust gateway on the public port and Python on a private loopback listener. This package is the Python half of forwarding and pairs with the crate `litellm-rust/crates/gateway-forward`. Once Rust serves every route, both get deleted together, along with one dispatch branch in each entry point
- Nothing outside this package imports Rust forwarding concepts, except the dispatch, the flag adapter and the app wrap in `litellm/proxy/proxy_cli.py` and `gateway/launch.py`. The proxy app itself never knows Rust is in front of it
- Python must see each request as if the client had connected directly. `x-litellm-rust-peer` is the only Rust-specific input, and `peer.py` honors it only from the loopback hop
- Python owns every setting Rust needs. The supervisor resolves the host, port, keepalive and shutdown drain Python actually runs with and hands them to Rust as the `LITELLM_FORWARD_*` variables in `rust_spec`. Rust never reads Python's own settings, so a new value Rust needs gets a new variable there
- Entry points turn their own flags into a `PublicListener`, apply `server_overrides` in the supervised child, and wrap the app with `peer.wrap` where they load it

View file

View file

@ -0,0 +1,10 @@
import os
from litellm.proxy.rust_forward.mode import rust_forward_mode
from litellm.proxy.rust_forward.peer import ASGIApp, wrap
def app() -> ASGIApp:
from litellm_cli.main import proxy_worker_app
return wrap(proxy_worker_app(), rust_forward_mode(os.environ))

View file

@ -0,0 +1,46 @@
import os
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final
from litellm.rust_bridge.configuration import ENV_NAME
@dataclass(frozen=True, slots=True)
class PublicListener:
host: str
port: int
keepalive_seconds: int
shutdown_drain_seconds: float
proxy_headers: bool
trusted_proxies: str | None
@dataclass(frozen=True, slots=True)
class StartupError:
reason: str
def startup_check(binary: str | None, unsupported: Mapping[str, object]) -> str | StartupError:
flag: Final = next((name for name, value in unsupported.items() if value), None)
if flag is not None:
return StartupError(f"{ENV_NAME} does not support {flag}")
if os.name != "posix":
return StartupError(f"{ENV_NAME} requires POSIX inherited sockets and process signals")
if binary is None:
return StartupError(
f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment"
)
return binary
def uvicorn_default_keepalive() -> int:
import uvicorn
return uvicorn.Config(app="", log_config=None).timeout_keep_alive
def shutdown_drain_seconds(graceful_timeout: float) -> float:
from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
return graceful_timeout + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS

View file

@ -0,0 +1,68 @@
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final, TypeAlias
from pydantic import TypeAdapter
from litellm.rust_bridge.configuration import environment_switch
GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD"
METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST"
PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS"
TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS"
DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1"
RUST_HOP: Final = "127.0.0.1"
_FD: Final = TypeAdapter(int)
_ENABLED: Final = TypeAdapter(bool)
@dataclass(frozen=True, slots=True)
class Disabled:
pass
@dataclass(frozen=True, slots=True)
class Supervisor:
pass
@dataclass(frozen=True, slots=True)
class Supervised:
listener_fd: int
metrics_host: str
proxy_headers: bool
trusted_proxies: str
RustForwardMode: TypeAlias = Disabled | Supervisor | Supervised
def rust_forward_mode(environ: Mapping[str, str]) -> RustForwardMode:
fd: Final = environ.get(GATEWAY_FD)
if fd is not None:
return Supervised(
_FD.validate_python(fd),
environ[METRICS_HOST],
_ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")),
environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES),
)
if environment_switch(environ):
return Supervisor()
return Disabled()
@dataclass(frozen=True, slots=True)
class ServerOverrides:
fd: int
proxy_headers: bool
metrics_host: str
def server_overrides(mode: Supervised) -> ServerOverrides:
"""The private listener the supervised server binds instead of its own flags.
Proxy headers stay off at the server, since it would judge Rust's loopback address instead of the real peer.
``peer.wrap`` applies the operator's trust settings to the restored peer.
"""
return ServerOverrides(fd=mode.listener_fd, proxy_headers=False, metrics_host=mode.metrics_host)

View file

@ -0,0 +1,57 @@
from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Final, TypeAlias
from typing_extensions import assert_never
from litellm.proxy.rust_forward.mode import RUST_HOP, Disabled, RustForwardMode, Supervised, Supervisor
if TYPE_CHECKING:
from uvicorn._types import ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope
PEER_HEADER: Final = b"x-litellm-rust-peer"
ASGIApp: TypeAlias = Callable[..., Awaitable[None]]
def _restore_peer(scope: "WWWScope") -> None:
headers: Final = tuple(scope["headers"])
peer: Final = next((value for name, value in headers if name == PEER_HEADER), None)
if peer is None:
return
remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER)
scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop
client: Final = scope["client"]
if client is None or client[0] != RUST_HOP:
return
host, _, port = peer.decode("latin-1").rpartition(":")
scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope
class RustPeerMiddleware:
"""Make Python see each request as if the client had connected to it directly.
Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real
TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the
real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust
settings Python would have used on the public port.
"""
def __init__(self, app: ASGIApp, mode: Supervised) -> None:
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app
async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None:
if scope["type"] != "lifespan":
_restore_peer(scope)
await self.app(scope, receive, send)
def wrap(app: ASGIApp, mode: RustForwardMode) -> ASGIApp:
match mode:
case Supervised():
return RustPeerMiddleware(app, mode)
case Disabled() | Supervisor():
return app
case _:
assert_never(mode)

View file

@ -0,0 +1,219 @@
import asyncio
import contextlib
import os
import shutil
import signal
import socket
import sys
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from enum import Enum, IntEnum
from types import MappingProxyType
from typing import Final, TypeAlias
from pydantic import TypeAdapter
from typing_extensions import assert_never
from litellm.proxy.rust_forward.listener import PublicListener
from litellm.proxy.rust_forward.mode import GATEWAY_FD, METRICS_HOST, PROXY_HEADERS, RUST_HOP, TRUSTED_PROXIES
_ADDRESS: Final = TypeAdapter(tuple[str, int])
def gateway_binary(scripts_dir: str) -> str | None:
name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway"
return shutil.which(name, path=scripts_dir)
class Child(Enum):
RUST = "rust"
PYTHON = "python"
DRAIN_ORDER: Final = (Child.RUST, Child.PYTHON)
class ExitCode(IntEnum):
CLEAN = 0
CHILD_FAILED = 1
@dataclass(frozen=True, slots=True)
class Stop:
signum: signal.Signals
@dataclass(frozen=True, slots=True)
class Exited:
child: Child
code: int
@dataclass(frozen=True, slots=True)
class Running:
pass
@dataclass(frozen=True, slots=True)
class Draining:
signum: signal.Signals
signalled: Child
remaining: tuple[Child, ...]
@dataclass(frozen=True, slots=True)
class Finished:
code: ExitCode
@dataclass(frozen=True, slots=True)
class Signal:
child: Child
signum: signal.Signals
Event: TypeAlias = Stop | Exited
State: TypeAlias = Running | Draining | Finished
def _signal_next(signum: signal.Signals, order: tuple[Child, ...]) -> tuple[State, Signal | None]:
if not order:
return Finished(ExitCode.CLEAN), None
return Draining(signum, order[0], order[1:]), Signal(order[0], signum)
def step(state: State, event: Event) -> tuple[State, Signal | None]:
match state:
case Running():
match event:
case Stop(signum):
return _signal_next(signum, DRAIN_ORDER)
case Exited():
return Finished(ExitCode.CHILD_FAILED), None
case _:
assert_never(event)
case Draining(signum, signalled, remaining):
match event:
case Stop():
return state, None
case Exited(child, code) if child is signalled and code == 0:
return _signal_next(signum, remaining)
case Exited():
return Finished(ExitCode.CHILD_FAILED), None
case _:
assert_never(event)
case Finished():
return state, None
case _:
assert_never(state)
@dataclass(frozen=True, slots=True)
class ChildSpec:
argv: tuple[str, ...]
env: Mapping[str, str]
pass_fds: tuple[int, ...]
def python_spec(
command: Sequence[str], listener_fd: int, upstream_port: int, listener: PublicListener, environ: Mapping[str, str]
) -> ChildSpec:
return ChildSpec(
argv=(*command, "--host", RUST_HOP, "--port", str(upstream_port)),
env=MappingProxyType(
{
**environ,
GATEWAY_FD: str(listener_fd),
METRICS_HOST: listener.host,
PROXY_HEADERS: str(listener.proxy_headers).lower(),
**({} if listener.trusted_proxies is None else {TRUSTED_PROXIES: listener.trusted_proxies}),
}
),
pass_fds=(listener_fd,),
)
def rust_spec(binary: str, upstream_port: int, listener: PublicListener, environ: Mapping[str, str]) -> ChildSpec:
return ChildSpec(
argv=(binary,),
env=MappingProxyType(
{
**environ,
"LITELLM_FORWARD_UPSTREAM": f"http://{RUST_HOP}:{upstream_port}",
"LITELLM_FORWARD_HOST": listener.host,
"LITELLM_FORWARD_PORT": str(listener.port),
"LITELLM_FORWARD_UPSTREAM_KEEPALIVE": str(listener.keepalive_seconds),
"LITELLM_FORWARD_SHUTDOWN_DRAIN": str(listener.shutdown_drain_seconds),
}
),
pass_fds=(),
)
async def _spawn(spec: ChildSpec) -> asyncio.subprocess.Process:
return await asyncio.create_subprocess_exec(
*spec.argv, env=dict(spec.env), pass_fds=spec.pass_fds, start_new_session=True
)
def _terminate_group(process: asyncio.subprocess.Process) -> None:
with contextlib.suppress(ProcessLookupError):
os.killpg(process.pid, signal.SIGTERM)
async def _report_exit(child: Child, process: asyncio.subprocess.Process, events: asyncio.Queue[Event]) -> None:
events.put_nowait(Exited(child, await process.wait()))
async def _drive(
state: State, events: asyncio.Queue[Event], children: Mapping[Child, asyncio.subprocess.Process]
) -> ExitCode:
if isinstance(state, Finished):
return state.code
following, action = step(state, await events.get())
if action is not None:
with contextlib.suppress(ProcessLookupError):
children[action.child].send_signal(action.signum)
return await _drive(following, events, children)
async def _supervise(
binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str]
) -> ExitCode:
loop: Final = asyncio.get_running_loop()
events: Final[asyncio.Queue[Event]] = asyncio.Queue()
for signum in (signal.SIGTERM, signal.SIGINT):
loop.add_signal_handler(signum, events.put_nowait, Stop(signum))
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as private:
private.bind((RUST_HOP, 0))
private.listen()
_, upstream_port = _ADDRESS.validate_python(private.getsockname())
python: Final = await _spawn(python_spec(command, private.fileno(), upstream_port, listener, environ))
try:
rust: Final = await _spawn(rust_spec(binary, upstream_port, listener, environ))
except OSError:
_terminate_group(python)
await python.wait()
raise
children: Final = MappingProxyType({Child.RUST: rust, Child.PYTHON: python})
waiters: Final = tuple(
asyncio.create_task(_report_exit(child, process, events)) for child, process in children.items()
)
try:
return await _drive(Running(), events, children)
finally:
for process in children.values():
_terminate_group(process)
await asyncio.gather(*waiters)
def supervise(binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str]) -> int:
try:
return asyncio.run(_supervise(binary, command, listener, environ))
except OSError as error:
sys.stderr.write(f"Rust gateway startup failed: {error}\n")
return ExitCode.CHILD_FAILED
def cli_command(argv: Sequence[str]) -> tuple[str, ...]:
return (sys.executable, "-m", "litellm.proxy.proxy_cli", *argv)

View file

@ -1,10 +1,16 @@
from uvicorn.workers import UvicornWorker
def build_graceful_shutdown_uvicorn_worker(timeout_graceful_shutdown: float) -> type[UvicornWorker]:
def build_graceful_shutdown_uvicorn_worker(
timeout_graceful_shutdown: float, *, proxy_headers: bool = True
) -> type[UvicornWorker]:
class GracefulShutdownUvicornWorker(UvicornWorker):
def __init__(self, *args: object) -> None:
self.CONFIG_KWARGS = {**UvicornWorker.CONFIG_KWARGS, "timeout_graceful_shutdown": timeout_graceful_shutdown}
self.CONFIG_KWARGS = {
**UvicornWorker.CONFIG_KWARGS,
"timeout_graceful_shutdown": timeout_graceful_shutdown,
"proxy_headers": proxy_headers,
}
super().__init__(*args)
return GracefulShutdownUvicornWorker

View file

@ -1,6 +1,7 @@
from __future__ import annotations
import os
from collections.abc import Mapping
from enum import Enum, auto
from functools import lru_cache
from typing import Final
@ -8,7 +9,7 @@ from typing import Final
from pydantic import TypeAdapter, ValidationError
from typing_extensions import assert_never
_GLOBAL_ENV_NAME: Final = "LITELLM_RUST"
ENV_NAME: Final = "LITELLM_RUST"
_ENV_BOOL: Final = TypeAdapter(bool)
@ -44,6 +45,10 @@ def _parse_env_bool(value: str | None) -> bool | None:
return None
def environment_switch(environ: Mapping[str, str]) -> bool | None:
return _parse_env_bool(environ.get(ENV_NAME))
def decide(
rollout: Rollout,
*,
@ -72,7 +77,7 @@ def decision(rollout: Rollout) -> Decision:
return decide(
rollout,
process_override=_CONFIGURATION.override,
environment_override=_parse_env_bool(os.getenv(_GLOBAL_ENV_NAME)),
environment_override=environment_switch(os.environ),
)

View file

@ -0,0 +1,180 @@
import json
import os
import subprocess
import sys
from pathlib import Path
from typing import Final
import httpx
import pytest
from tests.integration._support.client import Gateway, eventually
from tests.integration._support.ports import allocate_port
from tests.integration._support.process import stop_root_process
TRUSTED_BALANCER: Final = "127.0.0.3"
def ready(client: httpx.Client) -> bool:
try:
return client.get("/health/liveliness").status_code == 200
except httpx.TransportError:
return False
def client_ip(base_url: str, local_address: str, key: str, forwarded_for: str) -> httpx.Response:
with httpx.Client(
base_url=base_url,
transport=httpx.HTTPTransport(local_address=local_address),
trust_env=False,
timeout=5,
) as client:
return client.get(
"/v1/mcp/network/client-ip",
headers={"Authorization": f"Bearer {key}", "X-Forwarded-For": forwarded_for},
)
@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)])
def test_gateway_preserves_metrics_bind_and_mcp_client_ip(
gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...]
) -> None:
config: Final = tmp_path / "config.yaml"
config.write_text(
json.dumps(
{
"model_list": [],
"general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY", "use_x_forwarded_for": False},
}
)
)
metrics_port: Final = allocate_port()
public_port: Final = allocate_port()
environment: Final = {
**{key: value for key, value in os.environ.items() if key not in ("DATABASE_URL", "DATABASE_URL_READ_REPLICA")},
"LITELLM_MASTER_KEY": gateway.key,
"STORE_MODEL_IN_DB": "False",
"PYTHON_DOTENV_DISABLED": "1",
"LITELLM_RUST": "true",
"FORWARDED_ALLOW_IPS": TRUSTED_BALANCER,
}
with (tmp_path / "proxy.log").open("wb") as output:
with subprocess.Popen(
(
sys.executable,
"-I",
"-m",
"litellm.proxy.proxy_cli",
"--config",
str(config),
"--host",
"0.0.0.0",
"--port",
str(public_port),
"--num_workers",
"2",
"--timeout_worker_healthcheck",
"120",
"--prometheus_metrics_port",
str(metrics_port),
*server_arguments,
),
env=environment,
stdout=output,
stderr=output,
) as supervisor:
try:
with httpx.Client(
base_url=f"http://127.0.0.1:{public_port}",
transport=httpx.HTTPTransport(local_address="127.0.0.2"),
trust_env=False,
follow_redirects=True,
timeout=5,
) as external:
assert eventually(lambda: ready(external), bool, seconds=180), (tmp_path / "proxy.log").read_text()
response: Final = external.get(f"http://127.0.0.2:{metrics_port}/metrics")
assert response.status_code == 200, response.text
public: Final = f"http://127.0.0.1:{public_port}"
spoofed: Final = client_ip(public, "127.0.0.2", gateway.key, "1.2.3.4")
assert spoofed.status_code == 200, spoofed.text
assert spoofed.json() == {"ip": "127.0.0.2"}
behind_balancer: Final = client_ip(public, TRUSTED_BALANCER, gateway.key, "203.0.113.9")
assert behind_balancer.status_code == 200, behind_balancer.text
assert behind_balancer.json() == {"ip": "203.0.113.9"}
borrowed: Final = client_ip(public, "127.0.0.2", gateway.key, f"203.0.113.9, {TRUSTED_BALANCER}")
assert borrowed.status_code == 200, borrowed.text
assert borrowed.json() == {"ip": "127.0.0.2"}
finally:
assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text()
assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text()
@pytest.mark.parametrize(
"server_arguments",
[("--no-proxy-headers",), ("--forwarded-allow-ips", "127.0.0.1")],
)
def test_gateway_launcher_private_hop_preserves_client_address(
tmp_path: Path, server_arguments: tuple[str, ...]
) -> None:
config: Final = tmp_path / "config.yaml"
config.write_text(
json.dumps({"model_list": [], "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY"}})
)
public_port: Final = allocate_port()
environment: Final = {
**{key: value for key, value in os.environ.items() if not key.startswith("DATABASE_") and key != "DIRECT_URL"},
"LITELLM_MASTER_KEY": "sk-rust-forwarding-integration-key-not-a-secret",
"LITELLM_RUST": "true",
"LITELLM_PGBOUNCER_ENABLED": "false",
"CONFIG_FILE_PATH": str(config),
"STORE_MODEL_IN_DB": "False",
"PYTHON_DOTENV_DISABLED": "1",
}
log: Final = tmp_path / "gateway.log"
with log.open("wb") as output:
with subprocess.Popen(
(
sys.executable,
"-I",
"-m",
"gateway.launch",
"--host",
"0.0.0.0",
"--port",
str(public_port),
"--workers",
"2",
"--timeout-worker-healthcheck",
"120",
*server_arguments,
),
env=environment,
stdout=output,
stderr=output,
) as supervisor:
try:
with httpx.Client(
base_url=f"http://127.0.0.1:{public_port}",
transport=httpx.HTTPTransport(local_address="127.0.0.2"),
trust_env=False,
timeout=5,
) as external:
assert eventually(lambda: ready(external), bool, seconds=180), log.read_text()
response: Final = external.get(
"/health/liveliness?client-ip-proof", headers={"X-Forwarded-For": "198.51.100.10"}
)
assert response.status_code == 200, response.text
events: Final = eventually(
lambda: tuple(
line
for line in log.read_text().splitlines()
if "GET /health/liveliness?client-ip-proof " in line
),
bool,
seconds=10,
)
assert len(events) == 1, log.read_text()
assert "127.0.0.2:" in events[0], events
finally:
assert stop_root_process(supervisor), log.read_text()
assert supervisor.returncode == 0, log.read_text()

View file

@ -12,13 +12,18 @@ from typing import Final, cast
from unittest.mock import MagicMock, patch
import pytest
import uvicorn
from uvicorn.importer import import_from_string
from uvicorn.main import main as uvicorn_main
from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv
from gateway.launch import GATEWAY_APP, main, pool_database_url, public_listener, uvicorn_argv
from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings
from litellm.proxy.proxy_server import app as proxy_app
from litellm.proxy.rust_forward.mode import Supervised, server_overrides
from litellm.proxy.rust_forward.supervisor import rust_spec
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
DB_ENV: Final = {
"DATABASE_HOST": "db.internal",
@ -114,17 +119,89 @@ def _uvicorn_params(argv: tuple[str, ...]) -> dict[str, object]:
class TestUvicornArgv:
@pytest.mark.parametrize(
"arguments", [("--uds", "/tmp/gateway.sock"), ("--fd", "0"), ("--fd=17",), ("--ssl-keyfile", "unused")]
)
def test_forwarding_rejects_unsupported_listeners_before_starting_processes(
self, arguments: tuple[str, ...]
) -> None:
with pytest.raises(SystemExit, match=f"does not support {arguments[0].split('=')[0]}"):
public_listener(arguments, "gateway")
@pytest.mark.parametrize(
("argv", "trust"),
[
((), (True, None)),
(("--forwarded-allow-ips", "10.0.0.5"), (True, "10.0.0.5")),
(("--forwarded-allow-ips=*",), (True, "*")),
(("--no-proxy-headers",), (False, None)),
],
)
def test_forwarding_carries_operator_proxy_header_options_to_the_child(
self, argv: tuple[str, ...], trust: tuple[bool, str | None], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("FORWARDED_ALLOW_IPS", raising=False)
binary, listener = public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway")
assert (binary, listener.host, listener.port) == ("gateway", "0.0.0.0", 4000)
assert (listener.proxy_headers, listener.trusted_proxies) == trust
@pytest.mark.parametrize(
("argv", "environ"),
[(("--timeout-keep-alive", "2"), {}), ((), {"KEEPALIVE_TIMEOUT": "7"}), ((), {})],
)
def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with(
self, argv: tuple[str, ...], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False)
for name, value in environ.items():
monkeypatch.setenv(name, value)
_, listener = public_listener(argv, "gateway")
sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"]
assert sent == str(_uvicorn_params(uvicorn_argv(argv, os.environ, None))["timeout_keep_alive"])
def test_without_a_keepalive_rust_follows_uvicorns_own_default(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False)
_, listener = public_listener((), "gateway")
assert listener.keepalive_seconds == uvicorn.Config(app="", log_config=None).timeout_keep_alive
@pytest.mark.parametrize("argv", [(), ("--timeout-graceful-shutdown", "12")])
def test_rust_drains_for_pythons_graceful_timeout_plus_teardown(
self, argv: tuple[str, ...], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "7.5")
_, listener = public_listener(argv, "gateway")
flag: Final = _uvicorn_params(uvicorn_argv(argv, {}, None))["timeout_graceful_shutdown"]
graceful: Final = GracefulShutdownManager.get_timeout() if flag is None else float(str(flag))
sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"]
assert float(sent) == graceful + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
def test_supervised_listener_uses_the_inherited_loopback_socket(self):
supervised: Final = server_overrides(Supervised(17, "0.0.0.0", True, "127.0.0.1"))
params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised))
assert params["fd"] == 17
assert params["workers"] == 2
@pytest.mark.parametrize("argv", [(), ("--proxy-headers",), ("--forwarded-allow-ips=*",)])
def test_private_hop_leaves_proxy_headers_to_the_app(self, argv: tuple[str, ...]):
params: Final = _uvicorn_params(uvicorn_argv(argv, {}, server_overrides(Supervised(17, "0.0.0.0", True, "*"))))
assert params["proxy_headers"] is False
def test_direct_listener_keeps_explicit_proxy_header_options(self):
params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {}, None))
assert params["proxy_headers"] is False
assert params["forwarded_allow_ips"] == "*"
def test_keepalive_env_reaches_uvicorn(self):
params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"}))
params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"}, None))
assert params["app"] == GATEWAY_APP
assert params["workers"] == 4
assert params["timeout_keep_alive"] == 75
def test_unset_env_keeps_the_uvicorn_default(self):
assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}))["timeout_keep_alive"] == 5
assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}, None))["timeout_keep_alive"] == 5
def test_an_explicit_flag_wins_over_the_env(self):
argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"})
argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}, None)
assert _uvicorn_params(argv)["timeout_keep_alive"] == 30
def test_the_app_uvicorn_is_told_to_serve_is_the_trimmed_gateway(self, monkeypatch: pytest.MonkeyPatch) -> None:
@ -141,7 +218,7 @@ class TestUvicornArgv:
):
monkeypatch.delenv(key, raising=False)
params: Final = _uvicorn_params(uvicorn_argv((), {}))
params: Final = _uvicorn_params(uvicorn_argv((), {}, None))
assert params["factory"] is True
try:
served: Final = import_from_string(cast(str, params["app"]))()
@ -229,6 +306,16 @@ class TestMain:
DatabaseURLSettings.from_env().apply_to_env()
assert os.environ["DATABASE_URL"] == pooled
def test_forwarding_disabled_keeps_uvicorn_socket_options(
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("LITELLM_RUST", "false")
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False)
served: Final[list[tuple[str, ...]]] = []
main(("--uds", "/tmp/gateway.sock"), serve=lambda argv: served.append(tuple(argv)))
assert _uvicorn_params(served[0])["uds"] == "/tmp/gateway.sock"
def test_a_pooler_that_cannot_start_stops_the_gateway_before_uvicorn(
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):

View file

@ -0,0 +1,74 @@
from collections.abc import Mapping
from typing import Final
import pytest
import uvicorn
from click.testing import CliRunner
from gunicorn.config import Config as GunicornConfig
from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server
from litellm.proxy.rust_forward.listener import PublicListener, StartupError, startup_check
from litellm.proxy.rust_forward.supervisor import rust_spec
@pytest.mark.parametrize(
"flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"]
)
def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_RUST", "true")
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else []
result: Final = CliRunner().invoke(run_server, [flag, *values])
assert result.exit_code != 0
assert f"LITELLM_RUST does not support {flag}" in result.output
def test_missing_binary_gives_install_instruction() -> None:
assert startup_check(None, {}) == StartupError(
"LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment"
)
def _cli_listener(argv: list[str]) -> tuple[Mapping[str, object], PublicListener]:
params: Final = run_server.make_context("litellm", ["--port", "45999", *argv]).params
return params, ProxyInitializationHelpers.public_listener(
str(params["host"]), int(str(params["port"])), params["keepalive_timeout"], params["run_gunicorn"] is True
)
def _uvicorn_keepalive(options: Mapping[str, object]) -> int:
configured: Final = {"timeout_keep_alive": options["timeout_keep_alive"]} if "timeout_keep_alive" in options else {}
return uvicorn.Config(app="", log_config=None, **configured).timeout_keep_alive
@pytest.mark.parametrize(
("argv", "environ"),
[(["--keepalive_timeout", "2"], {}), ([], {"KEEPALIVE_TIMEOUT": "7"}), ([], {})],
)
def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with(
argv: list[str], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False)
for name, value in environ.items():
monkeypatch.setenv(name, value)
params, listener = _cli_listener(argv)
uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args(
"127.0.0.1", 45555, keepalive_timeout=params["keepalive_timeout"]
)
sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"]
assert sent == str(_uvicorn_keepalive(uvicorn_options))
def test_gunicorn_workers_keep_gunicorns_keepalive_and_rust_follows(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False)
_, listener = _cli_listener(["--run_gunicorn", "--keepalive_timeout", "9"])
assert listener.keepalive_seconds == GunicornConfig().keepalive
def test_rust_drains_for_pythons_graceful_timeout_plus_teardown(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "12.5")
_, listener = _cli_listener([])
uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555)
sent: Final = float(rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"])
assert sent == uvicorn_options["timeout_graceful_shutdown"] + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS

View file

@ -0,0 +1,43 @@
from typing import Final
import pytest
from litellm.proxy.proxy_cli import ProxyInitializationHelpers
from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides
@pytest.mark.parametrize(
("environ", "expected"),
[
({}, Disabled()),
({"LITELLM_RUST": "false"}, Disabled()),
({"LITELLM_RUST": "off"}, Disabled()),
({"LITELLM_RUST": "0"}, Disabled()),
({"LITELLM_RUST": "true"}, Supervisor()),
({"LITELLM_RUST": "1"}, Supervisor()),
({"LITELLM_RUST": "yes"}, Supervisor()),
({"LITELLM_RUST": "on"}, Supervisor()),
({"LITELLM_RUST": " 1 "}, Supervisor()),
({"LITELLM_RUST": "enabled"}, Disabled()),
(
{
"LITELLM_RUST": "true",
"_LITELLM_GATEWAY_FD": "17",
"_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0",
"FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5",
},
Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"),
),
],
)
def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None:
assert rust_forward_mode(environ) == expected
@pytest.mark.parametrize("mode", [None, Supervised(17, "0.0.0.0", True, "127.0.0.1")])
def test_inherited_socket_reaches_uvicorn(mode: Supervised | None) -> None:
options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args(
"127.0.0.1", 45555, forwarded=None if mode is None else server_overrides(mode)
)
assert options.get("fd") == (None if mode is None else mode.listener_fd)
assert options.get("proxy_headers", True) is (mode is None)

View file

@ -0,0 +1,144 @@
import asyncio
import gc
from typing import TYPE_CHECKING, Final
import httpx
import pytest
from fastapi import Request
from fastapi.routing import APIRoute
from uvicorn.importer import import_from_string
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
if TYPE_CHECKING:
from uvicorn._types import (
ASGIReceiveCallable,
ASGISendCallable,
ASGISendEvent,
HTTPDisconnectEvent,
HTTPScope,
Scope,
)
from litellm.proxy.proxy_server import app as proxy_app
from litellm.proxy.rust_forward.mode import Supervised
from litellm.proxy.rust_forward.peer import wrap
Headers = tuple[tuple[bytes, bytes], ...]
async def _receive() -> "HTTPDisconnectEvent":
return {"type": "http.disconnect"}
async def _send(message: "ASGISendEvent") -> None:
return None
async def _seen_by_app(
behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised
) -> tuple[tuple[str, int] | None, str, Headers]:
seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = (
asyncio.get_running_loop().create_future()
)
async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None:
assert scope["type"] == "http"
seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"])))
direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe
app: Final = wrap(probe, mode) if behind_rust else direct
scope: Final[HTTPScope] = {
"type": "http",
"asgi": {"version": "3.0", "spec_version": "2.3"},
"http_version": "1.1",
"method": "GET",
"scheme": "http",
"path": "/",
"raw_path": b"/",
"query_string": b"",
"root_path": "",
"headers": headers,
"client": client,
"server": ("127.0.0.1", 4000),
}
await app(scope, _receive, _send)
return seen.result()
PEER: Final = ("203.0.113.7", 51234)
TRUSTED_PEER: Final = ("10.0.12.34", 40000)
FORWARDED: Final[Headers] = (
(b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"),
(b"x-forwarded-proto", b"https"),
)
@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)])
@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))])
@pytest.mark.parametrize(
"mode",
[
Supervised(17, "0.0.0.0", True, "127.0.0.1"),
Supervised(17, "0.0.0.0", True, "10.0.0.0/8"),
Supervised(17, "0.0.0.0", True, "*"),
Supervised(17, "0.0.0.0", False, "10.0.0.0/8"),
],
)
async def test_python_behind_rust_sees_what_a_direct_connection_would(
peer: tuple[str, int], headers: Headers, mode: Supervised
) -> None:
rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}"
behind_rust: Final = await _seen_by_app(
True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode
)
direct: Final = await _seen_by_app(False, peer, headers, mode)
assert behind_rust == direct
async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None:
client, scheme, _ = await _seen_by_app(
True,
("127.0.0.1", 60000),
((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")),
Supervised(17, "0.0.0.0", True, "127.0.0.1"),
)
assert (client, scheme) == (("203.0.113.7", 51234), "http")
async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None:
client, _, headers = await _seen_by_app(
True,
("192.0.2.9", 1000),
((b"x-litellm-rust-peer", b"10.0.12.34:1"),),
Supervised(17, "0.0.0.0", False, "127.0.0.1"),
)
assert (client, headers) == (("192.0.2.9", 1000), ())
async def _peer_probe(request: Request) -> dict[str, object]:
assert request.client is not None
return {"client": [request.client.host, request.client.port], "peer": request.headers.get("x-litellm-rust-peer")}
@pytest.mark.parametrize("entry_point", ["litellm.proxy.rust_forward.asgi:app", "gateway.launch:worker_app"])
async def test_every_entry_point_hands_routes_the_peer_rust_saw(
entry_point: str, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17")
monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", "0.0.0.0")
monkeypatch.setattr(proxy_app.router, "lifespan_context", proxy_app.router.lifespan_context)
monkeypatch.setattr(
proxy_app.router, "routes", [APIRoute("/__rust_forward_peer_probe", _peer_probe), *proxy_app.router.routes]
)
try:
served: Final = import_from_string(entry_point)()
finally:
gc.unfreeze()
transport: Final = httpx.ASGITransport(app=served, client=("127.0.0.1", 60000))
async with httpx.AsyncClient(transport=transport, base_url="http://proxy") as client:
response: Final = await client.get(
"/__rust_forward_peer_probe", headers={"x-litellm-rust-peer": "203.0.113.7:51234"}
)
assert response.status_code == 200, response.text
assert response.json() == {"client": ["203.0.113.7", 51234], "peer": None}

View file

@ -0,0 +1,139 @@
import functools
import signal
import sys
from pathlib import Path
from typing import Final
import pytest
from litellm.proxy.proxy_cli import run_server
from litellm.proxy.rust_forward.listener import PublicListener
from litellm.proxy.rust_forward.mode import Supervised, rust_forward_mode
from litellm.proxy.rust_forward.supervisor import (
Child,
Event,
ExitCode,
Exited,
Finished,
Running,
Signal,
State,
Stop,
cli_command,
gateway_binary,
python_spec,
rust_spec,
step,
)
TERM: Final = signal.SIGTERM
INT: Final = signal.SIGINT
def _listener(proxy_headers: bool = True, trusted_proxies: str | None = None) -> PublicListener:
return PublicListener("0.0.0.0", 4000, 5, 40.0, proxy_headers, trusted_proxies)
@pytest.mark.parametrize(
("environ", "listener", "expected"),
[
({}, _listener(), Supervised(17, "0.0.0.0", True, "127.0.0.1")),
({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(), Supervised(17, "0.0.0.0", True, "10.0.0.5")),
({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(True, "*"), Supervised(17, "0.0.0.0", True, "*")),
({}, _listener(False), Supervised(17, "0.0.0.0", False, "127.0.0.1")),
],
)
def test_python_child_keeps_the_trust_python_alone_would_use(
environ: dict[str, str], listener: PublicListener, expected: Supervised
) -> None:
spec: Final = python_spec(("python",), 17, 45555, listener, environ)
assert rust_forward_mode(spec.env) == expected
def test_python_child_listens_privately_and_keeps_options() -> None:
arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload")
spec: Final = python_spec(arguments, 17, 45555, _listener(), {})
params: Final = run_server.make_context("litellm", list(spec.argv)).params
assert params["host"] == "127.0.0.1"
assert params["port"] == 45555
assert params["num_workers"] == 2
assert params["reload"] is True
assert spec.pass_fds == (17,)
assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli")
def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None:
spec: Final = rust_spec(
"gateway", 45555, PublicListener("0.0.0.0", 4000, 2, 12.5, True, None), {"RUST_LOG": "info"}
)
assert spec.argv == ("gateway",)
assert spec.pass_fds == ()
assert dict(spec.env) == {
"RUST_LOG": "info",
"LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555",
"LITELLM_FORWARD_HOST": "0.0.0.0",
"LITELLM_FORWARD_PORT": "4000",
"LITELLM_FORWARD_UPSTREAM_KEEPALIVE": "2",
"LITELLM_FORWARD_SHUTDOWN_DRAIN": "12.5",
}
def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
scripts: Final = tmp_path / "scripts"
elsewhere: Final = tmp_path / "elsewhere"
scripts.mkdir()
elsewhere.mkdir()
binary: Final = elsewhere / "litellm-gateway"
binary.write_text("#!/bin/sh\nexit 0\n")
binary.chmod(0o755)
monkeypatch.setenv("PATH", str(elsewhere))
assert gateway_binary(str(scripts)) is None
assert gateway_binary(str(elsewhere)) == str(binary)
def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]:
def advance(acc: tuple[State, tuple[Signal, ...]], event: Event) -> tuple[State, tuple[Signal, ...]]:
state, sent = acc
following, action = step(state, event)
return following, sent if action is None else (*sent, action)
return functools.reduce(advance, events, (Running(), ()))
RUST: Final = Child.RUST
PYTHON: Final = Child.PYTHON
@pytest.mark.parametrize(
("events", "signals", "code"),
[
(
(Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 0)),
(Signal(RUST, TERM), Signal(PYTHON, TERM)),
ExitCode.CLEAN,
),
((Exited(PYTHON, 0),), (), ExitCode.CHILD_FAILED),
((Exited(RUST, 1),), (), ExitCode.CHILD_FAILED),
((Stop(TERM), Exited(PYTHON, 0)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED),
((Stop(TERM), Exited(RUST, 1)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED),
(
(Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 1)),
(Signal(RUST, TERM), Signal(PYTHON, TERM)),
ExitCode.CHILD_FAILED,
),
(
(Stop(INT), Stop(INT), Exited(RUST, 0), Exited(PYTHON, 0)),
(Signal(RUST, INT), Signal(PYTHON, INT)),
ExitCode.CLEAN,
),
(
(Exited(RUST, 1), Stop(TERM), Exited(PYTHON, 0)),
(),
ExitCode.CHILD_FAILED,
),
],
)
def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit(
events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode
) -> None:
assert _fold(events) == (Finished(code), signals)

View file

@ -86,6 +86,39 @@ def test_a_broken_factory_exits_the_worker_as_a_startup_failure(
assert exit_info.value.code == uvicorn_config.STARTUP_FAILURE
@pytest.mark.parametrize("worker_overrides", [None, {"factory": True}])
def test_a_factory_flag_in_the_options_reaches_the_workers(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, worker_overrides: dict[str, object] | None
) -> None:
(tmp_path / "reload_factory_flag_app.py").write_text(
"async def served(scope, receive, send):\n"
" await send({'type': 'http.response.start', 'status': 204, 'headers': []})\n"
" await send({'type': 'http.response.body', 'body': b'', 'more_body': False})\n"
"\n"
"def factory():\n"
" return served\n"
)
monkeypatch.syspath_prepend(str(tmp_path))
config: Final = pool_config(
RELOAD_OPTIONS.validate_python({**_options("reload_factory_flag_app:factory"), "factory": True}),
workers=POOL_WORKERS,
worker_overrides=worker_overrides,
)
config.load()
loaded_app: Final = config.loaded_app
assert loaded_app is not None
sent: Final[list[Message]] = []
async def receive() -> Message:
return {"type": "http.request", "body": b"", "more_body": False}
async def send(message: Message) -> None:
sent.append(message)
asyncio.run(loaded_app({"type": "http", "method": "GET", "path": "/", "headers": []}, receive, send))
assert sent[0] == {"type": "http.response.start", "status": 204, "headers": []}
@REQUIRES_UVICORN_STARTUP_FAILURE
@pytest.mark.parametrize(
("module", "source"),
@ -118,7 +151,9 @@ def test_the_reload_pool_defaults_the_worker_healthcheck_and_keeps_an_explicit_o
explicit: Final = RELOAD_WORKER_HEALTHCHECK_SECONDS + 7
assert pool_config(_options("app:app"), workers=2).timeout_worker_healthcheck == RELOAD_WORKER_HEALTHCHECK_SECONDS
assert (
pool_config({**_options("app:app"), "timeout_worker_healthcheck": explicit}, workers=2).timeout_worker_healthcheck
pool_config(
{**_options("app:app"), "timeout_worker_healthcheck": explicit}, workers=2
).timeout_worker_healthcheck
== explicit
)
@ -142,7 +177,9 @@ def test_the_reload_pool_accepts_and_keeps_every_uvicorn_arg_the_proxy_cli_build
def test_the_reload_pool_passes_a_uvicorn_arg_it_does_not_list_through_to_the_workers() -> None:
config: Final = pool_config(RELOAD_OPTIONS.validate_python({**_options("app:app"), "backlog": 123}), workers=POOL_WORKERS)
config: Final = pool_config(
RELOAD_OPTIONS.validate_python({**_options("app:app"), "backlog": 123}), workers=POOL_WORKERS
)
assert (config.backlog, config.reload) == (123, False)

View file

@ -1550,6 +1550,7 @@ class TestProxyInitializationHelpers:
log_config=None,
keepalive_timeout=30,
timeout_worker_healthcheck=None,
forwarded=None,
)
mock_uvicorn_run.assert_called_once()
@ -1626,6 +1627,7 @@ class TestProxyInitializationHelpers:
log_config=None,
keepalive_timeout=None,
timeout_worker_healthcheck=15,
forwarded=None,
)
@patch("uvicorn.run")