fix(proxy): gather Rust forwarding into rust_forward and hand Rust Python's keepalive and drain

Every Python line that exists only because Rust forwards to Python now lives in
litellm/proxy/rust_forward (mode, peer, listener, supervisor, asgi), paired with the
gateway-forward crate. proxy_server no longer knows about Rust. proxy_cli and
gateway.launch wrap the app where they load it, apply server_overrides in the
supervised child, and turn their own flags into one PublicListener

The supervisor now resolves the keepalive and shutdown drain Python actually runs
with and passes them to Rust as LITELLM_FORWARD_UPSTREAM_KEEPALIVE and
LITELLM_FORWARD_SHUTDOWN_DRAIN, along with LITELLM_FORWARD_HOST and
LITELLM_FORWARD_PORT. Before, `--keepalive_timeout 2` gave uvicorn 2s while Rust
kept its default 5s pool and reused connections uvicorn was closing

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-11-03 13:21:05 +09:00
parent 4a8563b01b
commit b2e18b93a3
20 changed files with 769 additions and 495 deletions

View file

@ -583,6 +583,7 @@ jobs:
tests/unit/proxy/credential_endpoints
tests/unit/proxy/discovery_endpoints
tests/unit/proxy/shutdown
tests/unit/proxy/rust_forward
tests/unit/proxy/public_endpoints
tests/unit/proxy/prompts
tests/unit/proxy/rag_endpoints

View file

@ -29,27 +29,29 @@ from uvicorn.main import main as uvicorn_main
from litellm_cli.main import without_gc_pauses
if TYPE_CHECKING:
from fastapi import FastAPI
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings
from litellm.proxy.rust_gateway import Supervised
from litellm.proxy.rust_forward.listener import PublicListener
from litellm.proxy.rust_forward.mode import ServerOverrides
from litellm.proxy.rust_forward.peer import ASGIApp
GATEWAY_APP: Final = "gateway.launch:worker_app"
KEEPALIVE_FLAG: Final = "--timeout-keep-alive"
def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str], supervised: "Supervised | None") -> tuple[str, ...]:
def uvicorn_argv(
argv: Sequence[str], environ: Mapping[str, str], forwarded: "ServerOverrides | None"
) -> tuple[str, ...]:
"""Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly."""
keepalive: Final = environ.get("KEEPALIVE_TIMEOUT")
inherited_socket: Final = (
(
()
if forwarded is None
else (
"--fd",
str(supervised.listener_fd),
"--no-proxy-headers",
str(forwarded.fd),
"--proxy-headers" if forwarded.proxy_headers else "--no-proxy-headers",
)
if supervised is not None
else ()
)
if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv):
return (GATEWAY_APP, "--factory", *argv, *inherited_socket)
@ -77,26 +79,29 @@ def pool_database_url(
return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth())
def worker_app() -> "FastAPI":
def worker_app() -> "ASGIApp":
with without_gc_pauses():
from gateway.main import app
return app
from litellm.proxy.rust_forward.mode import rust_forward_mode
from litellm.proxy.rust_forward.peer import wrap
return wrap(app, rust_forward_mode(os.environ))
def _serve(argv: Sequence[str]) -> None:
uvicorn_main(tuple(argv), prog_name="uvicorn")
def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, int, Mapping[str, str]]:
"""Validate the uvicorn flags for Rust forwarding and return the binary, public address and child environment."""
def public_listener(argv: Sequence[str], binary: str | None) -> "tuple[str, PublicListener]":
"""Validate the uvicorn flags for Rust forwarding and return the binary and the public listener."""
from pydantic import TypeAdapter
from litellm.proxy.rust_gateway import PROXY_HEADERS, TRUSTED_PROXIES, startup_error
from litellm.proxy.rust_forward.listener import PublicListener, StartupError, shutdown_drain_seconds, startup_check
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
params: Final[Mapping[str, object]] = uvicorn_main.make_context(
"uvicorn", list(uvicorn_argv(argv, os.environ, None))
).params
error: Final = startup_error(
checked: Final = startup_check(
binary,
{
"--ssl-keyfile": params["ssl_keyfile"],
@ -106,18 +111,18 @@ def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str,
"--fd": params["fd"] is not None,
},
)
if error is not None:
sys.exit(error)
assert binary is not None
host, port = TypeAdapter(tuple[str, int]).validate_python((params["host"], params["port"]))
operator_trust: Final = TypeAdapter(str | None).validate_python(params["forwarded_allow_ips"])
proxy_headers: Final = TypeAdapter(bool).validate_python(params["proxy_headers"])
return (
binary,
host,
port,
{PROXY_HEADERS: str(proxy_headers).lower()}
| ({} if operator_trust is None else {TRUSTED_PROXIES: operator_trust}),
if isinstance(checked, StartupError):
sys.exit(checked.reason)
graceful_timeout: Final = TypeAdapter[int | None](int | None).validate_python(params["timeout_graceful_shutdown"])
return checked, PublicListener(
host=TypeAdapter(str).validate_python(params["host"]),
port=TypeAdapter(int).validate_python(params["port"]),
keepalive_seconds=TypeAdapter(int).validate_python(params["timeout_keep_alive"]),
shutdown_drain_seconds=shutdown_drain_seconds(
GracefulShutdownManager.get_timeout() if graceful_timeout is None else graceful_timeout
),
proxy_headers=TypeAdapter(bool).validate_python(params["proxy_headers"]),
trusted_proxies=TypeAdapter[str | None](str | None).validate_python(params["forwarded_allow_ips"]),
)
@ -135,27 +140,21 @@ def configure_database() -> None:
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
from litellm.proxy.rust_gateway import (
Disabled,
Supervised,
Supervisor,
gateway_binary,
rust_gateway_mode,
supervise,
)
from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides
from litellm.proxy.rust_forward.supervisor import gateway_binary, supervise
mode: Final = rust_gateway_mode(os.environ)
mode: Final = rust_forward_mode(os.environ)
match mode:
case Disabled():
configure_database()
serve(uvicorn_argv(argv, os.environ, None))
case Supervisor():
binary, host, port, trust = public_listener(argv, gateway_binary(sysconfig.get_path("scripts")))
binary, listener = public_listener(argv, gateway_binary(sysconfig.get_path("scripts")))
configure_database()
command: Final = (sys.executable, "-m", "gateway.launch", *argv)
sys.exit(supervise(binary, command, host, port, {**os.environ, **trust}))
sys.exit(supervise(binary, command, listener, os.environ))
case Supervised():
serve(uvicorn_argv(argv, os.environ, mode))
serve(uvicorn_argv(argv, os.environ, server_overrides(mode)))
case _:
assert_never(mode)

View file

@ -36,6 +36,10 @@ from litellm.types.llms.base import LiteLLMBaseModel
if TYPE_CHECKING:
from fastapi import FastAPI
from litellm.proxy.rust_forward.listener import PublicListener
from litellm.proxy.rust_forward.mode import ServerOverrides
from litellm.proxy.rust_forward.peer import ASGIApp
else:
FastAPI = Any
@ -262,6 +266,7 @@ def _apply_config_logging_settings(litellm_settings: StartupOutputSettings | Non
WORKER_APP: Final = "litellm_cli.main:proxy_worker_app"
FORWARD_APP: Final = "litellm.proxy.rust_forward.asgi:app"
WORKER_LOG_CONFIG_ENV: Final = "LITELLM_WORKER_LOG_CONFIG"
_LOG_CONFIG: Final = TypeAdapter[str | Mapping[str, object]](str | Mapping[str, object])
_LOG_CONFIG_FILE: Final = TypeAdapter[Mapping[str, object]](Mapping[str, object])
@ -390,12 +395,12 @@ class ProxyInitializationHelpers:
logging.config.fileConfig(log_config, disable_existing_loggers=False)
@staticmethod
def child_process_uvicorn_overrides(log_config: object) -> Mapping[str, object]:
def child_process_uvicorn_overrides(log_config: object, app: str = WORKER_APP) -> Mapping[str, object]:
if log_config is None:
return MappingProxyType({"app": WORKER_APP, "factory": True})
return MappingProxyType({"app": app, "factory": True})
ProxyInitializationHelpers.apply_log_config(_LOG_CONFIG.validate_python(log_config))
os.environ[WORKER_LOG_CONFIG_ENV] = json.dumps(log_config)
return MappingProxyType({"app": WORKER_APP, "factory": True, "log_config": None})
return MappingProxyType({"app": app, "factory": True, "log_config": None})
@staticmethod
def apply_worker_log_config(environ: Mapping[str, str]) -> None:
@ -410,7 +415,7 @@ class ProxyInitializationHelpers:
log_config: str | None = None,
keepalive_timeout: int | None = None,
timeout_worker_healthcheck: int | None = None,
listener_fd: int | None = None,
forwarded: "ServerOverrides | None" = None,
) -> dict:
"""
Get the arguments for `uvicorn` worker
@ -429,7 +434,16 @@ class ProxyInitializationHelpers:
"port": port,
"server_header": False,
"timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(),
**({"fd": listener_fd, "proxy_headers": False} if listener_fd is not None else {}),
**(
{}
if forwarded is None
else {
"app": FORWARD_APP,
"factory": True,
"fd": forwarded.fd,
"proxy_headers": forwarded.proxy_headers,
}
),
}
if log_config is not None:
print_startup_message(f"Using log_config: {log_config}")
@ -677,13 +691,13 @@ class ProxyInitializationHelpers:
def run_gunicorn_server(
host: str,
port: int,
app: FastAPI,
app: "ASGIApp",
num_workers: int,
ssl_certfile_path: str,
ssl_keyfile_path: str,
max_requests_before_restart: int | None = None,
max_requests_before_restart_jitter: int | None = None,
listener_fd: int | None = None,
forwarded: "ServerOverrides | None" = None,
):
"""
Run litellm with `gunicorn`
@ -756,10 +770,10 @@ class ProxyInitializationHelpers:
)
graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout()
gunicorn_options: Final = {
"bind": f"fd://{listener_fd}" if listener_fd is not None else f"{host}:{port}",
"bind": f"{host}:{port}" if forwarded is None else f"fd://{forwarded.fd}",
"workers": num_workers, # default is 1
"worker_class": build_graceful_shutdown_uvicorn_worker(
graceful_shutdown_timeout, proxy_headers=listener_fd is None
graceful_shutdown_timeout, proxy_headers=True if forwarded is None else forwarded.proxy_headers
),
"graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS,
"preload": True, # Add the preload flag,
@ -829,6 +843,34 @@ class ProxyInitializationHelpers:
_is_port_in_use = is_port_in_use
@staticmethod
def public_listener(host: str, port: int, keepalive_timeout: int | None, run_gunicorn: bool) -> "PublicListener":
"""``run_gunicorn_server`` never hands ``--keepalive_timeout`` to gunicorn, so its workers keep gunicorn's."""
from gunicorn.config import Config as GunicornConfig
from litellm.proxy.rust_forward.listener import (
PublicListener,
shutdown_drain_seconds,
uvicorn_default_keepalive,
)
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
keepalive: Final = (
TypeAdapter(int).validate_python(GunicornConfig().keepalive)
if run_gunicorn
else (uvicorn_default_keepalive() if keepalive_timeout is None else keepalive_timeout)
)
return PublicListener(
host=host,
port=random.randint(1024, 49152)
if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port)
else port,
keepalive_seconds=keepalive,
shutdown_drain_seconds=shutdown_drain_seconds(GracefulShutdownManager.get_timeout()),
proxy_headers=True,
trusted_proxies=None,
)
@staticmethod
def get_loop_type():
"""Helper function to determine the event loop type based on platform"""
@ -1286,17 +1328,17 @@ def run_server(
run_setup_wizard()
return
from litellm.proxy.rust_gateway import Supervised, Supervisor, rust_gateway_mode
from litellm.proxy.rust_forward.mode import Supervised, Supervisor, rust_forward_mode, server_overrides
gateway_mode: Final = rust_gateway_mode(os.environ)
if isinstance(gateway_mode, Supervisor) and not any(
forward_mode: Final = rust_forward_mode(os.environ)
if isinstance(forward_mode, Supervisor) and not any(
(version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data)
):
from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise
from litellm.proxy.rust_forward.listener import StartupError, startup_check
from litellm.proxy.rust_forward.supervisor import cli_command, gateway_binary, supervise
binary: Final = gateway_binary(sysconfig.get_path("scripts"))
gateway_error: Final = startup_error(
binary,
binary: Final = startup_check(
gateway_binary(sysconfig.get_path("scripts")),
{
"--ssl_keyfile_path": ssl_keyfile_path,
"--ssl_certfile_path": ssl_certfile_path,
@ -1305,14 +1347,11 @@ def run_server(
"--run_granian": run_granian,
},
)
if gateway_error is not None:
raise click.ClickException(gateway_error)
assert binary is not None
public_port: Final = (
random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port
)
raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port, os.environ))
supervised: Final = gateway_mode if isinstance(gateway_mode, Supervised) else None
if isinstance(binary, StartupError):
raise click.ClickException(binary.reason)
listener: Final = ProxyInitializationHelpers.public_listener(host, port, keepalive_timeout, run_gunicorn)
raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), listener, os.environ))
forwarded: Final = server_overrides(forward_mode) if isinstance(forward_mode, Supervised) else None
args: Final = locals()
try:
@ -1763,7 +1802,7 @@ def run_server(
if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None:
from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process
metrics_bind_host: Final = supervised.metrics_host if supervised is not None else host
metrics_bind_host: Final = host if forwarded is None else forwarded.metrics_host
try:
metrics_process: Final = start_metrics_server_process(
host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir
@ -1782,7 +1821,7 @@ def run_server(
log_config=log_config,
keepalive_timeout=keepalive_timeout,
timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None),
listener_fd=supervised.listener_fd if supervised is not None else None,
forwarded=forwarded,
)
# Optional: recycle uvicorn workers after N requests
if max_requests_before_restart is not None:
@ -1822,7 +1861,7 @@ def run_server(
start_query_engine_reaper()
if num_workers > 1 or reload:
worker_overrides: Final = ProxyInitializationHelpers.child_process_uvicorn_overrides(
uvicorn_args.get("log_config")
uvicorn_args.get("log_config"), WORKER_APP if forwarded is None else FORWARD_APP
)
if reload and num_workers > 1:
from litellm.proxy.dev_reload import (
@ -1848,16 +1887,18 @@ def run_server(
workers=num_workers,
)
elif run_gunicorn is True:
from litellm.proxy.rust_forward.peer import wrap
ProxyInitializationHelpers.run_gunicorn_server(
host=host,
port=port,
app=app,
app=wrap(app, forward_mode),
num_workers=num_workers,
ssl_certfile_path=ssl_certfile_path,
ssl_keyfile_path=ssl_keyfile_path,
max_requests_before_restart=max_requests_before_restart,
max_requests_before_restart_jitter=max_requests_before_restart_jitter,
listener_fd=supervised.listener_fd if supervised is not None else None,
forwarded=forwarded,
)
elif run_hypercorn is True:
ProxyInitializationHelpers.init_hypercorn_server(

View file

@ -835,7 +835,6 @@ from litellm.proxy.plugin_routes import (
from litellm.proxy.plugin_routes import (
router as plugin_router,
)
from litellm.proxy.rust_gateway import RustPeerMiddleware, Supervised, rust_gateway_mode
from litellm.proxy.spend_tracking.spend_event_producer import (
CollectorSettings,
SpendEventProducer,
@ -21925,10 +21924,6 @@ if _server_root_paths:
)
app.add_middleware(PerRequestRootPathMiddleware, root_paths=_server_root_paths)
_rust_gateway_mode: Final = rust_gateway_mode(os.environ)
if isinstance(_rust_gateway_mode, Supervised):
app.add_middleware(RustPeerMiddleware, mode=_rust_gateway_mode)
async def _stream_mcp_asgi_response(handle_fn, scope: dict, receive) -> "StreamingResponse":
"""

View file

@ -0,0 +1,8 @@
# Rust forwarding, Python half
With `LITELLM_RUST` on, the supervisor here runs the Rust gateway on the public port and Python on a private loopback listener. This package is the Python half of forwarding and pairs with the crate `litellm-rust/crates/gateway-forward`. Once Rust serves every route, both get deleted together, along with one dispatch branch in each entry point
- Nothing outside this package imports Rust forwarding concepts, except the dispatch, the flag adapter and the app wrap in `litellm/proxy/proxy_cli.py` and `gateway/launch.py`. The proxy app itself never knows Rust is in front of it
- Python must see each request as if the client had connected directly. `x-litellm-rust-peer` is the only Rust-specific input, and `peer.py` honors it only from the loopback hop
- Python owns every setting Rust needs. The supervisor resolves the host, port, keepalive and shutdown drain Python actually runs with and hands them to Rust as the `LITELLM_FORWARD_*` variables in `rust_spec`. Rust never reads Python's own settings, so a new value Rust needs gets a new variable there
- Entry points turn their own flags into a `PublicListener`, apply `server_overrides` in the supervised child, and wrap the app with `peer.wrap` where they load it

View file

View file

@ -0,0 +1,10 @@
import os
from litellm.proxy.rust_forward.mode import rust_forward_mode
from litellm.proxy.rust_forward.peer import ASGIApp, wrap
def app() -> ASGIApp:
from litellm_cli.main import proxy_worker_app
return wrap(proxy_worker_app(), rust_forward_mode(os.environ))

View file

@ -0,0 +1,46 @@
import os
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final
from litellm.rust_bridge.configuration import ENV_NAME
@dataclass(frozen=True, slots=True)
class PublicListener:
host: str
port: int
keepalive_seconds: int
shutdown_drain_seconds: float
proxy_headers: bool
trusted_proxies: str | None
@dataclass(frozen=True, slots=True)
class StartupError:
reason: str
def startup_check(binary: str | None, unsupported: Mapping[str, object]) -> str | StartupError:
flag: Final = next((name for name, value in unsupported.items() if value), None)
if flag is not None:
return StartupError(f"{ENV_NAME} does not support {flag}")
if os.name != "posix":
return StartupError(f"{ENV_NAME} requires POSIX inherited sockets and process signals")
if binary is None:
return StartupError(
f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment"
)
return binary
def uvicorn_default_keepalive() -> int:
import uvicorn
return uvicorn.Config(app="", log_config=None).timeout_keep_alive
def shutdown_drain_seconds(graceful_timeout: float) -> float:
from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
return graceful_timeout + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS

View file

@ -0,0 +1,68 @@
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final, TypeAlias
from pydantic import TypeAdapter
from litellm.rust_bridge.configuration import environment_switch
GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD"
METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST"
PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS"
TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS"
DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1"
RUST_HOP: Final = "127.0.0.1"
_FD: Final = TypeAdapter(int)
_ENABLED: Final = TypeAdapter(bool)
@dataclass(frozen=True, slots=True)
class Disabled:
pass
@dataclass(frozen=True, slots=True)
class Supervisor:
pass
@dataclass(frozen=True, slots=True)
class Supervised:
listener_fd: int
metrics_host: str
proxy_headers: bool
trusted_proxies: str
RustForwardMode: TypeAlias = Disabled | Supervisor | Supervised
def rust_forward_mode(environ: Mapping[str, str]) -> RustForwardMode:
fd: Final = environ.get(GATEWAY_FD)
if fd is not None:
return Supervised(
_FD.validate_python(fd),
environ[METRICS_HOST],
_ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")),
environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES),
)
if environment_switch(environ):
return Supervisor()
return Disabled()
@dataclass(frozen=True, slots=True)
class ServerOverrides:
fd: int
proxy_headers: bool
metrics_host: str
def server_overrides(mode: Supervised) -> ServerOverrides:
"""The private listener the supervised server binds instead of its own flags.
Proxy headers stay off at the server, since it would judge Rust's loopback address instead of the real peer.
``peer.wrap`` applies the operator's trust settings to the restored peer.
"""
return ServerOverrides(fd=mode.listener_fd, proxy_headers=False, metrics_host=mode.metrics_host)

View file

@ -0,0 +1,57 @@
from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Final, TypeAlias
from typing_extensions import assert_never
from litellm.proxy.rust_forward.mode import RUST_HOP, Disabled, RustForwardMode, Supervised, Supervisor
if TYPE_CHECKING:
from uvicorn._types import ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope
PEER_HEADER: Final = b"x-litellm-rust-peer"
ASGIApp: TypeAlias = Callable[..., Awaitable[None]]
def _restore_peer(scope: "WWWScope") -> None:
headers: Final = tuple(scope["headers"])
peer: Final = next((value for name, value in headers if name == PEER_HEADER), None)
if peer is None:
return
remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER)
scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop
client: Final = scope["client"]
if client is None or client[0] != RUST_HOP:
return
host, _, port = peer.decode("latin-1").rpartition(":")
scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope
class RustPeerMiddleware:
"""Make Python see each request as if the client had connected to it directly.
Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real
TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the
real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust
settings Python would have used on the public port.
"""
def __init__(self, app: ASGIApp, mode: Supervised) -> None:
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app
async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None:
if scope["type"] != "lifespan":
_restore_peer(scope)
await self.app(scope, receive, send)
def wrap(app: ASGIApp, mode: RustForwardMode) -> ASGIApp:
match mode:
case Supervised():
return RustPeerMiddleware(app, mode)
case Disabled() | Supervisor():
return app
case _:
assert_never(mode)

View file

@ -9,115 +9,22 @@ from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from enum import Enum, IntEnum
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, TypeAlias
from typing import Final, TypeAlias
from pydantic import TypeAdapter
from typing_extensions import assert_never
from litellm.rust_bridge.configuration import ENV_NAME, environment_switch
from litellm.proxy.rust_forward.listener import PublicListener
from litellm.proxy.rust_forward.mode import GATEWAY_FD, METRICS_HOST, PROXY_HEADERS, RUST_HOP, TRUSTED_PROXIES
if TYPE_CHECKING:
from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope
GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD"
METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST"
PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS"
TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS"
DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1"
RUST_HOP: Final = "127.0.0.1"
PEER_HEADER: Final = b"x-litellm-rust-peer"
_FD: Final = TypeAdapter(int)
_ENABLED: Final = TypeAdapter(bool)
_ADDRESS: Final = TypeAdapter(tuple[str, int])
@dataclass(frozen=True, slots=True)
class Disabled:
pass
@dataclass(frozen=True, slots=True)
class Supervisor:
pass
@dataclass(frozen=True, slots=True)
class Supervised:
listener_fd: int
metrics_host: str
proxy_headers: bool
trusted_proxies: str
RustGatewayMode: TypeAlias = Disabled | Supervisor | Supervised
def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode:
fd: Final = environ.get(GATEWAY_FD)
if fd is not None:
return Supervised(
_FD.validate_python(fd),
environ[METRICS_HOST],
_ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")),
environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES),
)
if environment_switch(environ):
return Supervisor()
return Disabled()
def _restore_peer(scope: "WWWScope") -> None:
headers: Final = tuple(scope["headers"])
peer: Final = next((value for name, value in headers if name == PEER_HEADER), None)
if peer is None:
return
remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER)
scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop
client: Final = scope["client"]
if client is None or client[0] != RUST_HOP:
return
host, _, port = peer.decode("latin-1").rpartition(":")
scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope
class RustPeerMiddleware:
"""Make Python see each request as if the client had connected to it directly.
Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real
TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the
real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust
settings Python would have used on the public port. The server's proxy-header handling is off in
this mode, since it would judge Rust's loopback address instead of the real peer.
"""
def __init__(self, app: "ASGI3Application", mode: Supervised) -> None:
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app
async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None:
if scope["type"] != "lifespan":
_restore_peer(scope)
await self.app(scope, receive, send)
def gateway_binary(scripts_dir: str) -> str | None:
name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway"
return shutil.which(name, path=scripts_dir)
def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None:
flag: Final = next((name for name, value in unsupported.items() if value), None)
if flag is not None:
return f"{ENV_NAME} does not support {flag}"
if os.name != "posix":
return f"{ENV_NAME} requires POSIX inherited sockets and process signals"
if binary is None:
return f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment"
return None
class Child(Enum):
RUST = "rust"
PYTHON = "python"
@ -209,7 +116,7 @@ class ChildSpec:
def python_spec(
command: Sequence[str], listener_fd: int, upstream_port: int, public_host: str, environ: Mapping[str, str]
command: Sequence[str], listener_fd: int, upstream_port: int, listener: PublicListener, environ: Mapping[str, str]
) -> ChildSpec:
return ChildSpec(
argv=(*command, "--host", RUST_HOP, "--port", str(upstream_port)),
@ -217,22 +124,26 @@ def python_spec(
{
**environ,
GATEWAY_FD: str(listener_fd),
METRICS_HOST: public_host,
METRICS_HOST: listener.host,
PROXY_HEADERS: str(listener.proxy_headers).lower(),
**({} if listener.trusted_proxies is None else {TRUSTED_PROXIES: listener.trusted_proxies}),
}
),
pass_fds=(listener_fd,),
)
def rust_spec(binary: str, upstream_port: int, host: str, port: int, environ: Mapping[str, str]) -> ChildSpec:
def rust_spec(binary: str, upstream_port: int, listener: PublicListener, environ: Mapping[str, str]) -> ChildSpec:
return ChildSpec(
argv=(binary,),
env=MappingProxyType(
{
**environ,
"HOST": host,
"PORT": str(port),
"LITELLM_FORWARD_UPSTREAM": f"http://{RUST_HOP}:{upstream_port}",
"LITELLM_FORWARD_HOST": listener.host,
"LITELLM_FORWARD_PORT": str(listener.port),
"LITELLM_FORWARD_UPSTREAM_KEEPALIVE": str(listener.keepalive_seconds),
"LITELLM_FORWARD_SHUTDOWN_DRAIN": str(listener.shutdown_drain_seconds),
}
),
pass_fds=(),
@ -266,18 +177,20 @@ async def _drive(
return await _drive(following, events, children)
async def _supervise(binary: str, command: Sequence[str], host: str, port: int, environ: Mapping[str, str]) -> ExitCode:
async def _supervise(
binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str]
) -> ExitCode:
loop: Final = asyncio.get_running_loop()
events: Final[asyncio.Queue[Event]] = asyncio.Queue()
for signum in (signal.SIGTERM, signal.SIGINT):
loop.add_signal_handler(signum, events.put_nowait, Stop(signum))
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener:
listener.bind((RUST_HOP, 0))
listener.listen()
_, upstream_port = _ADDRESS.validate_python(listener.getsockname())
python: Final = await _spawn(python_spec(command, listener.fileno(), upstream_port, host, environ))
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as private:
private.bind((RUST_HOP, 0))
private.listen()
_, upstream_port = _ADDRESS.validate_python(private.getsockname())
python: Final = await _spawn(python_spec(command, private.fileno(), upstream_port, listener, environ))
try:
rust: Final = await _spawn(rust_spec(binary, upstream_port, host, port, environ))
rust: Final = await _spawn(rust_spec(binary, upstream_port, listener, environ))
except OSError:
_terminate_group(python)
await python.wait()
@ -294,9 +207,9 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int,
await asyncio.gather(*waiters)
def supervise(binary: str, command: Sequence[str], host: str, port: int, environ: Mapping[str, str]) -> int:
def supervise(binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str]) -> int:
try:
return asyncio.run(_supervise(binary, command, host, port, environ))
return asyncio.run(_supervise(binary, command, listener, environ))
except OSError as error:
sys.stderr.write(f"Rust gateway startup failed: {error}\n")
return ExitCode.CHILD_FAILED

View file

@ -12,14 +12,18 @@ from typing import Final, cast
from unittest.mock import MagicMock, patch
import pytest
import uvicorn
from uvicorn.importer import import_from_string
from uvicorn.main import main as uvicorn_main
from gateway.launch import GATEWAY_APP, main, pool_database_url, public_listener, uvicorn_argv
from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings
from litellm.proxy.proxy_server import app as proxy_app
from litellm.proxy.rust_gateway import Supervised
from litellm.proxy.rust_forward.mode import Supervised, server_overrides
from litellm.proxy.rust_forward.supervisor import rust_spec
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
DB_ENV: Final = {
"DATABASE_HOST": "db.internal",
@ -127,34 +131,59 @@ class TestUvicornArgv:
@pytest.mark.parametrize(
("argv", "trust"),
[
((), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true"}),
(
("--forwarded-allow-ips", "10.0.0.5"),
{"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "10.0.0.5"},
),
(("--forwarded-allow-ips=*",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "*"}),
(("--no-proxy-headers",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}),
((), (True, None)),
(("--forwarded-allow-ips", "10.0.0.5"), (True, "10.0.0.5")),
(("--forwarded-allow-ips=*",), (True, "*")),
(("--no-proxy-headers",), (False, None)),
],
)
def test_forwarding_carries_operator_proxy_header_options_to_the_child(
self, argv: tuple[str, ...], trust: dict[str, str]
self, argv: tuple[str, ...], trust: tuple[bool, str | None], monkeypatch: pytest.MonkeyPatch
) -> None:
assert public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") == (
"gateway",
"0.0.0.0",
4000,
trust,
)
monkeypatch.delenv("FORWARDED_ALLOW_IPS", raising=False)
binary, listener = public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway")
assert (binary, listener.host, listener.port) == ("gateway", "0.0.0.0", 4000)
assert (listener.proxy_headers, listener.trusted_proxies) == trust
@pytest.mark.parametrize(
("argv", "environ"),
[(("--timeout-keep-alive", "2"), {}), ((), {"KEEPALIVE_TIMEOUT": "7"}), ((), {})],
)
def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with(
self, argv: tuple[str, ...], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False)
for name, value in environ.items():
monkeypatch.setenv(name, value)
_, listener = public_listener(argv, "gateway")
sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"]
assert sent == str(_uvicorn_params(uvicorn_argv(argv, os.environ, None))["timeout_keep_alive"])
def test_without_a_keepalive_rust_follows_uvicorns_own_default(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False)
_, listener = public_listener((), "gateway")
assert listener.keepalive_seconds == uvicorn.Config(app="", log_config=None).timeout_keep_alive
@pytest.mark.parametrize("argv", [(), ("--timeout-graceful-shutdown", "12")])
def test_rust_drains_for_pythons_graceful_timeout_plus_teardown(
self, argv: tuple[str, ...], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "7.5")
_, listener = public_listener(argv, "gateway")
flag: Final = _uvicorn_params(uvicorn_argv(argv, {}, None))["timeout_graceful_shutdown"]
graceful: Final = GracefulShutdownManager.get_timeout() if flag is None else float(str(flag))
sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"]
assert float(sent) == graceful + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
def test_supervised_listener_uses_the_inherited_loopback_socket(self):
supervised: Final = Supervised(17, "0.0.0.0", True, "127.0.0.1")
supervised: Final = server_overrides(Supervised(17, "0.0.0.0", True, "127.0.0.1"))
params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised))
assert params["fd"] == 17
assert params["workers"] == 2
@pytest.mark.parametrize("argv", [(), ("--proxy-headers",), ("--forwarded-allow-ips=*",)])
def test_private_hop_leaves_proxy_headers_to_the_app(self, argv: tuple[str, ...]):
params: Final = _uvicorn_params(uvicorn_argv(argv, {}, Supervised(17, "0.0.0.0", True, "*")))
params: Final = _uvicorn_params(uvicorn_argv(argv, {}, server_overrides(Supervised(17, "0.0.0.0", True, "*"))))
assert params["proxy_headers"] is False
def test_direct_listener_keeps_explicit_proxy_header_options(self):

View file

@ -0,0 +1,74 @@
from collections.abc import Mapping
from typing import Final
import pytest
import uvicorn
from click.testing import CliRunner
from gunicorn.config import Config as GunicornConfig
from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server
from litellm.proxy.rust_forward.listener import PublicListener, StartupError, startup_check
from litellm.proxy.rust_forward.supervisor import rust_spec
@pytest.mark.parametrize(
"flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"]
)
def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_RUST", "true")
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else []
result: Final = CliRunner().invoke(run_server, [flag, *values])
assert result.exit_code != 0
assert f"LITELLM_RUST does not support {flag}" in result.output
def test_missing_binary_gives_install_instruction() -> None:
assert startup_check(None, {}) == StartupError(
"LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment"
)
def _cli_listener(argv: list[str]) -> tuple[Mapping[str, object], PublicListener]:
params: Final = run_server.make_context("litellm", ["--port", "45999", *argv]).params
return params, ProxyInitializationHelpers.public_listener(
str(params["host"]), int(str(params["port"])), params["keepalive_timeout"], params["run_gunicorn"] is True
)
def _uvicorn_keepalive(options: Mapping[str, object]) -> int:
configured: Final = {"timeout_keep_alive": options["timeout_keep_alive"]} if "timeout_keep_alive" in options else {}
return uvicorn.Config(app="", log_config=None, **configured).timeout_keep_alive
@pytest.mark.parametrize(
("argv", "environ"),
[(["--keepalive_timeout", "2"], {}), ([], {"KEEPALIVE_TIMEOUT": "7"}), ([], {})],
)
def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with(
argv: list[str], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False)
for name, value in environ.items():
monkeypatch.setenv(name, value)
params, listener = _cli_listener(argv)
uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args(
"127.0.0.1", 45555, keepalive_timeout=params["keepalive_timeout"]
)
sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"]
assert sent == str(_uvicorn_keepalive(uvicorn_options))
def test_gunicorn_workers_keep_gunicorns_keepalive_and_rust_follows(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False)
_, listener = _cli_listener(["--run_gunicorn", "--keepalive_timeout", "9"])
assert listener.keepalive_seconds == GunicornConfig().keepalive
def test_rust_drains_for_pythons_graceful_timeout_plus_teardown(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "12.5")
_, listener = _cli_listener([])
uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555)
sent: Final = float(rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"])
assert sent == uvicorn_options["timeout_graceful_shutdown"] + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS

View file

@ -0,0 +1,43 @@
from typing import Final
import pytest
from litellm.proxy.proxy_cli import ProxyInitializationHelpers
from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides
@pytest.mark.parametrize(
("environ", "expected"),
[
({}, Disabled()),
({"LITELLM_RUST": "false"}, Disabled()),
({"LITELLM_RUST": "off"}, Disabled()),
({"LITELLM_RUST": "0"}, Disabled()),
({"LITELLM_RUST": "true"}, Supervisor()),
({"LITELLM_RUST": "1"}, Supervisor()),
({"LITELLM_RUST": "yes"}, Supervisor()),
({"LITELLM_RUST": "on"}, Supervisor()),
({"LITELLM_RUST": " 1 "}, Supervisor()),
({"LITELLM_RUST": "enabled"}, Disabled()),
(
{
"LITELLM_RUST": "true",
"_LITELLM_GATEWAY_FD": "17",
"_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0",
"FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5",
},
Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"),
),
],
)
def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None:
assert rust_forward_mode(environ) == expected
@pytest.mark.parametrize("mode", [None, Supervised(17, "0.0.0.0", True, "127.0.0.1")])
def test_inherited_socket_reaches_uvicorn(mode: Supervised | None) -> None:
options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args(
"127.0.0.1", 45555, forwarded=None if mode is None else server_overrides(mode)
)
assert options.get("fd") == (None if mode is None else mode.listener_fd)
assert options.get("proxy_headers", True) is (mode is None)

View file

@ -0,0 +1,144 @@
import asyncio
import gc
from typing import TYPE_CHECKING, Final
import httpx
import pytest
from fastapi import Request
from fastapi.routing import APIRoute
from uvicorn.importer import import_from_string
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
if TYPE_CHECKING:
from uvicorn._types import (
ASGIReceiveCallable,
ASGISendCallable,
ASGISendEvent,
HTTPDisconnectEvent,
HTTPScope,
Scope,
)
from litellm.proxy.proxy_server import app as proxy_app
from litellm.proxy.rust_forward.mode import Supervised
from litellm.proxy.rust_forward.peer import wrap
Headers = tuple[tuple[bytes, bytes], ...]
async def _receive() -> "HTTPDisconnectEvent":
return {"type": "http.disconnect"}
async def _send(message: "ASGISendEvent") -> None:
return None
async def _seen_by_app(
behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised
) -> tuple[tuple[str, int] | None, str, Headers]:
seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = (
asyncio.get_running_loop().create_future()
)
async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None:
assert scope["type"] == "http"
seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"])))
direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe
app: Final = wrap(probe, mode) if behind_rust else direct
scope: Final[HTTPScope] = {
"type": "http",
"asgi": {"version": "3.0", "spec_version": "2.3"},
"http_version": "1.1",
"method": "GET",
"scheme": "http",
"path": "/",
"raw_path": b"/",
"query_string": b"",
"root_path": "",
"headers": headers,
"client": client,
"server": ("127.0.0.1", 4000),
}
await app(scope, _receive, _send)
return seen.result()
PEER: Final = ("203.0.113.7", 51234)
TRUSTED_PEER: Final = ("10.0.12.34", 40000)
FORWARDED: Final[Headers] = (
(b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"),
(b"x-forwarded-proto", b"https"),
)
@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)])
@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))])
@pytest.mark.parametrize(
"mode",
[
Supervised(17, "0.0.0.0", True, "127.0.0.1"),
Supervised(17, "0.0.0.0", True, "10.0.0.0/8"),
Supervised(17, "0.0.0.0", True, "*"),
Supervised(17, "0.0.0.0", False, "10.0.0.0/8"),
],
)
async def test_python_behind_rust_sees_what_a_direct_connection_would(
peer: tuple[str, int], headers: Headers, mode: Supervised
) -> None:
rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}"
behind_rust: Final = await _seen_by_app(
True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode
)
direct: Final = await _seen_by_app(False, peer, headers, mode)
assert behind_rust == direct
async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None:
client, scheme, _ = await _seen_by_app(
True,
("127.0.0.1", 60000),
((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")),
Supervised(17, "0.0.0.0", True, "127.0.0.1"),
)
assert (client, scheme) == (("203.0.113.7", 51234), "http")
async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None:
client, _, headers = await _seen_by_app(
True,
("192.0.2.9", 1000),
((b"x-litellm-rust-peer", b"10.0.12.34:1"),),
Supervised(17, "0.0.0.0", False, "127.0.0.1"),
)
assert (client, headers) == (("192.0.2.9", 1000), ())
async def _peer_probe(request: Request) -> dict[str, object]:
assert request.client is not None
return {"client": [request.client.host, request.client.port], "peer": request.headers.get("x-litellm-rust-peer")}
@pytest.mark.parametrize("entry_point", ["litellm.proxy.rust_forward.asgi:app", "gateway.launch:worker_app"])
async def test_every_entry_point_hands_routes_the_peer_rust_saw(
entry_point: str, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17")
monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", "0.0.0.0")
monkeypatch.setattr(proxy_app.router, "lifespan_context", proxy_app.router.lifespan_context)
monkeypatch.setattr(
proxy_app.router, "routes", [APIRoute("/__rust_forward_peer_probe", _peer_probe), *proxy_app.router.routes]
)
try:
served: Final = import_from_string(entry_point)()
finally:
gc.unfreeze()
transport: Final = httpx.ASGITransport(app=served, client=("127.0.0.1", 60000))
async with httpx.AsyncClient(transport=transport, base_url="http://proxy") as client:
response: Final = await client.get(
"/__rust_forward_peer_probe", headers={"x-litellm-rust-peer": "203.0.113.7:51234"}
)
assert response.status_code == 200, response.text
assert response.json() == {"client": ["203.0.113.7", 51234], "peer": None}

View file

@ -0,0 +1,139 @@
import functools
import signal
import sys
from pathlib import Path
from typing import Final
import pytest
from litellm.proxy.proxy_cli import run_server
from litellm.proxy.rust_forward.listener import PublicListener
from litellm.proxy.rust_forward.mode import Supervised, rust_forward_mode
from litellm.proxy.rust_forward.supervisor import (
Child,
Event,
ExitCode,
Exited,
Finished,
Running,
Signal,
State,
Stop,
cli_command,
gateway_binary,
python_spec,
rust_spec,
step,
)
TERM: Final = signal.SIGTERM
INT: Final = signal.SIGINT
def _listener(proxy_headers: bool = True, trusted_proxies: str | None = None) -> PublicListener:
return PublicListener("0.0.0.0", 4000, 5, 40.0, proxy_headers, trusted_proxies)
@pytest.mark.parametrize(
("environ", "listener", "expected"),
[
({}, _listener(), Supervised(17, "0.0.0.0", True, "127.0.0.1")),
({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(), Supervised(17, "0.0.0.0", True, "10.0.0.5")),
({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(True, "*"), Supervised(17, "0.0.0.0", True, "*")),
({}, _listener(False), Supervised(17, "0.0.0.0", False, "127.0.0.1")),
],
)
def test_python_child_keeps_the_trust_python_alone_would_use(
environ: dict[str, str], listener: PublicListener, expected: Supervised
) -> None:
spec: Final = python_spec(("python",), 17, 45555, listener, environ)
assert rust_forward_mode(spec.env) == expected
def test_python_child_listens_privately_and_keeps_options() -> None:
arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload")
spec: Final = python_spec(arguments, 17, 45555, _listener(), {})
params: Final = run_server.make_context("litellm", list(spec.argv)).params
assert params["host"] == "127.0.0.1"
assert params["port"] == 45555
assert params["num_workers"] == 2
assert params["reload"] is True
assert spec.pass_fds == (17,)
assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli")
def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None:
spec: Final = rust_spec(
"gateway", 45555, PublicListener("0.0.0.0", 4000, 2, 12.5, True, None), {"RUST_LOG": "info"}
)
assert spec.argv == ("gateway",)
assert spec.pass_fds == ()
assert dict(spec.env) == {
"RUST_LOG": "info",
"LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555",
"LITELLM_FORWARD_HOST": "0.0.0.0",
"LITELLM_FORWARD_PORT": "4000",
"LITELLM_FORWARD_UPSTREAM_KEEPALIVE": "2",
"LITELLM_FORWARD_SHUTDOWN_DRAIN": "12.5",
}
def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
scripts: Final = tmp_path / "scripts"
elsewhere: Final = tmp_path / "elsewhere"
scripts.mkdir()
elsewhere.mkdir()
binary: Final = elsewhere / "litellm-gateway"
binary.write_text("#!/bin/sh\nexit 0\n")
binary.chmod(0o755)
monkeypatch.setenv("PATH", str(elsewhere))
assert gateway_binary(str(scripts)) is None
assert gateway_binary(str(elsewhere)) == str(binary)
def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]:
def advance(acc: tuple[State, tuple[Signal, ...]], event: Event) -> tuple[State, tuple[Signal, ...]]:
state, sent = acc
following, action = step(state, event)
return following, sent if action is None else (*sent, action)
return functools.reduce(advance, events, (Running(), ()))
RUST: Final = Child.RUST
PYTHON: Final = Child.PYTHON
@pytest.mark.parametrize(
("events", "signals", "code"),
[
(
(Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 0)),
(Signal(RUST, TERM), Signal(PYTHON, TERM)),
ExitCode.CLEAN,
),
((Exited(PYTHON, 0),), (), ExitCode.CHILD_FAILED),
((Exited(RUST, 1),), (), ExitCode.CHILD_FAILED),
((Stop(TERM), Exited(PYTHON, 0)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED),
((Stop(TERM), Exited(RUST, 1)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED),
(
(Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 1)),
(Signal(RUST, TERM), Signal(PYTHON, TERM)),
ExitCode.CHILD_FAILED,
),
(
(Stop(INT), Stop(INT), Exited(RUST, 0), Exited(PYTHON, 0)),
(Signal(RUST, INT), Signal(PYTHON, INT)),
ExitCode.CLEAN,
),
(
(Exited(RUST, 1), Stop(TERM), Exited(PYTHON, 0)),
(),
ExitCode.CHILD_FAILED,
),
],
)
def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit(
events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode
) -> None:
assert _fold(events) == (Finished(code), signals)

View file

@ -1550,7 +1550,7 @@ class TestProxyInitializationHelpers:
log_config=None,
keepalive_timeout=30,
timeout_worker_healthcheck=None,
listener_fd=None,
forwarded=None,
)
mock_uvicorn_run.assert_called_once()
@ -1627,7 +1627,7 @@ class TestProxyInitializationHelpers:
log_config=None,
keepalive_timeout=None,
timeout_worker_healthcheck=15,
listener_fd=None,
forwarded=None,
)
@patch("uvicorn.run")

View file

@ -1,293 +0,0 @@
import asyncio
import functools
import signal
import sys
from pathlib import Path
from typing import TYPE_CHECKING, Final
import pytest
from click.testing import CliRunner
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
if TYPE_CHECKING:
from uvicorn._types import (
ASGIReceiveCallable,
ASGISendCallable,
ASGISendEvent,
HTTPDisconnectEvent,
HTTPScope,
Scope,
)
from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server
from litellm.proxy.rust_gateway import (
Child,
Disabled,
Event,
ExitCode,
Exited,
Finished,
Running,
RustPeerMiddleware,
Signal,
State,
Stop,
Supervised,
Supervisor,
cli_command,
gateway_binary,
python_spec,
rust_gateway_mode,
rust_spec,
startup_error,
step,
)
TERM: Final = signal.SIGTERM
INT: Final = signal.SIGINT
@pytest.mark.parametrize(
"flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"]
)
def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_RUST", "true")
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else []
result: Final = CliRunner().invoke(run_server, [flag, *values])
assert result.exit_code != 0
assert f"LITELLM_RUST does not support {flag}" in result.output
@pytest.mark.parametrize(
("environ", "expected"),
[
({}, Disabled()),
({"LITELLM_RUST": "false"}, Disabled()),
({"LITELLM_RUST": "off"}, Disabled()),
({"LITELLM_RUST": "0"}, Disabled()),
({"LITELLM_RUST": "true"}, Supervisor()),
({"LITELLM_RUST": "1"}, Supervisor()),
({"LITELLM_RUST": "yes"}, Supervisor()),
({"LITELLM_RUST": "on"}, Supervisor()),
({"LITELLM_RUST": " 1 "}, Supervisor()),
({"LITELLM_RUST": "enabled"}, Disabled()),
(
{
"LITELLM_RUST": "true",
"_LITELLM_GATEWAY_FD": "17",
"_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0",
"FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5",
},
Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"),
),
],
)
def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None:
assert rust_gateway_mode(environ) == expected
@pytest.mark.parametrize(
("environ", "expected"),
[
({}, Supervised(17, "0.0.0.0", True, "127.0.0.1")),
({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, Supervised(17, "0.0.0.0", True, "10.0.0.5")),
({"FORWARDED_ALLOW_IPS": "*"}, Supervised(17, "0.0.0.0", True, "*")),
({"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}, Supervised(17, "0.0.0.0", False, "127.0.0.1")),
],
)
def test_python_child_keeps_the_trust_python_alone_would_use(environ: dict[str, str], expected: Supervised) -> None:
spec: Final = python_spec(("python",), 17, 45555, "0.0.0.0", environ)
assert rust_gateway_mode(spec.env) == expected
def test_python_child_listens_privately_and_keeps_options() -> None:
arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload")
spec: Final = python_spec(arguments, 17, 45555, "0.0.0.0", {})
params: Final = run_server.make_context("litellm", list(spec.argv)).params
assert params["host"] == "127.0.0.1"
assert params["port"] == 45555
assert params["num_workers"] == 2
assert params["reload"] is True
assert spec.pass_fds == (17,)
assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli")
def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None:
spec: Final = rust_spec("gateway", 45555, "0.0.0.0", 4000, {"RUST_LOG": "info"})
assert spec.argv == ("gateway",)
assert spec.pass_fds == ()
assert dict(spec.env) == {
"RUST_LOG": "info",
"HOST": "0.0.0.0",
"PORT": "4000",
"LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555",
}
def test_missing_binary_gives_install_instruction() -> None:
assert startup_error(None, {}) == (
"LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment"
)
def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
scripts: Final = tmp_path / "scripts"
elsewhere: Final = tmp_path / "elsewhere"
scripts.mkdir()
elsewhere.mkdir()
binary: Final = elsewhere / "litellm-gateway"
binary.write_text("#!/bin/sh\nexit 0\n")
binary.chmod(0o755)
monkeypatch.setenv("PATH", str(elsewhere))
assert gateway_binary(str(scripts)) is None
assert gateway_binary(str(elsewhere)) == str(binary)
@pytest.mark.parametrize("listener_fd", [None, 17])
def test_inherited_socket_reaches_uvicorn(listener_fd: int | None) -> None:
options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args(
"127.0.0.1", 45555, listener_fd=listener_fd
)
assert options.get("fd") == listener_fd
assert options.get("proxy_headers", True) is (listener_fd is None)
def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]:
def advance(acc: tuple[State, tuple[Signal, ...]], event: Event) -> tuple[State, tuple[Signal, ...]]:
state, sent = acc
following, action = step(state, event)
return following, sent if action is None else (*sent, action)
return functools.reduce(advance, events, (Running(), ()))
RUST: Final = Child.RUST
PYTHON: Final = Child.PYTHON
@pytest.mark.parametrize(
("events", "signals", "code"),
[
(
(Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 0)),
(Signal(RUST, TERM), Signal(PYTHON, TERM)),
ExitCode.CLEAN,
),
((Exited(PYTHON, 0),), (), ExitCode.CHILD_FAILED),
((Exited(RUST, 1),), (), ExitCode.CHILD_FAILED),
((Stop(TERM), Exited(PYTHON, 0)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED),
((Stop(TERM), Exited(RUST, 1)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED),
(
(Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 1)),
(Signal(RUST, TERM), Signal(PYTHON, TERM)),
ExitCode.CHILD_FAILED,
),
(
(Stop(INT), Stop(INT), Exited(RUST, 0), Exited(PYTHON, 0)),
(Signal(RUST, INT), Signal(PYTHON, INT)),
ExitCode.CLEAN,
),
(
(Exited(RUST, 1), Stop(TERM), Exited(PYTHON, 0)),
(),
ExitCode.CHILD_FAILED,
),
],
)
def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit(
events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode
) -> None:
assert _fold(events) == (Finished(code), signals)
Headers = tuple[tuple[bytes, bytes], ...]
async def _receive() -> "HTTPDisconnectEvent":
return {"type": "http.disconnect"}
async def _send(message: "ASGISendEvent") -> None:
return None
async def _seen_by_app(
behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised
) -> tuple[tuple[str, int] | None, str, Headers]:
seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = (
asyncio.get_running_loop().create_future()
)
async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None:
assert scope["type"] == "http"
seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"])))
direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe
app: Final = RustPeerMiddleware(probe, mode) if behind_rust else direct
scope: Final[HTTPScope] = {
"type": "http",
"asgi": {"version": "3.0", "spec_version": "2.3"},
"http_version": "1.1",
"method": "GET",
"scheme": "http",
"path": "/",
"raw_path": b"/",
"query_string": b"",
"root_path": "",
"headers": headers,
"client": client,
"server": ("127.0.0.1", 4000),
}
await app(scope, _receive, _send)
return seen.result()
PEER: Final = ("203.0.113.7", 51234)
TRUSTED_PEER: Final = ("10.0.12.34", 40000)
FORWARDED: Final[Headers] = (
(b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"),
(b"x-forwarded-proto", b"https"),
)
@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)])
@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))])
@pytest.mark.parametrize(
"mode",
[
Supervised(17, "0.0.0.0", True, "127.0.0.1"),
Supervised(17, "0.0.0.0", True, "10.0.0.0/8"),
Supervised(17, "0.0.0.0", True, "*"),
Supervised(17, "0.0.0.0", False, "10.0.0.0/8"),
],
)
async def test_python_behind_rust_sees_what_a_direct_connection_would(
peer: tuple[str, int], headers: Headers, mode: Supervised
) -> None:
rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}"
behind_rust: Final = await _seen_by_app(
True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode
)
direct: Final = await _seen_by_app(False, peer, headers, mode)
assert behind_rust == direct
async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None:
client, scheme, _ = await _seen_by_app(
True,
("127.0.0.1", 60000),
((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")),
Supervised(17, "0.0.0.0", True, "127.0.0.1"),
)
assert (client, scheme) == (("203.0.113.7", 51234), "http")
async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None:
client, _, headers = await _seen_by_app(
True,
("192.0.2.9", 1000),
((b"x-litellm-rust-peer", b"10.0.12.34:1"),),
Supervised(17, "0.0.0.0", False, "127.0.0.1"),
)
assert (client, headers) == (("192.0.2.9", 1000), ())