diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 17f68cd7d32..5e61d6ebd4c 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -583,6 +583,7 @@ jobs: tests/unit/proxy/credential_endpoints tests/unit/proxy/discovery_endpoints tests/unit/proxy/shutdown + tests/unit/proxy/rust_forward tests/unit/proxy/public_endpoints tests/unit/proxy/prompts tests/unit/proxy/rag_endpoints diff --git a/gateway/launch.py b/gateway/launch.py index 5dec5de46fa..89341a4cb89 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -29,27 +29,29 @@ from uvicorn.main import main as uvicorn_main from litellm_cli.main import without_gc_pauses if TYPE_CHECKING: - from fastapi import FastAPI - from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings - from litellm.proxy.rust_gateway import Supervised + from litellm.proxy.rust_forward.listener import PublicListener + from litellm.proxy.rust_forward.mode import ServerOverrides + from litellm.proxy.rust_forward.peer import ASGIApp GATEWAY_APP: Final = "gateway.launch:worker_app" KEEPALIVE_FLAG: Final = "--timeout-keep-alive" -def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str], supervised: "Supervised | None") -> tuple[str, ...]: +def uvicorn_argv( + argv: Sequence[str], environ: Mapping[str, str], forwarded: "ServerOverrides | None" +) -> tuple[str, ...]: """Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly.""" keepalive: Final = environ.get("KEEPALIVE_TIMEOUT") inherited_socket: Final = ( - ( + () + if forwarded is None + else ( "--fd", - str(supervised.listener_fd), - "--no-proxy-headers", + str(forwarded.fd), + "--proxy-headers" if forwarded.proxy_headers else "--no-proxy-headers", ) - if supervised is not None - else () ) if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv): return (GATEWAY_APP, "--factory", *argv, *inherited_socket) @@ -77,26 +79,29 @@ def pool_database_url( return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth()) -def worker_app() -> "FastAPI": +def worker_app() -> "ASGIApp": with without_gc_pauses(): from gateway.main import app - return app + from litellm.proxy.rust_forward.mode import rust_forward_mode + from litellm.proxy.rust_forward.peer import wrap + return wrap(app, rust_forward_mode(os.environ)) def _serve(argv: Sequence[str]) -> None: uvicorn_main(tuple(argv), prog_name="uvicorn") -def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, int, Mapping[str, str]]: - """Validate the uvicorn flags for Rust forwarding and return the binary, public address and child environment.""" +def public_listener(argv: Sequence[str], binary: str | None) -> "tuple[str, PublicListener]": + """Validate the uvicorn flags for Rust forwarding and return the binary and the public listener.""" from pydantic import TypeAdapter - from litellm.proxy.rust_gateway import PROXY_HEADERS, TRUSTED_PROXIES, startup_error + from litellm.proxy.rust_forward.listener import PublicListener, StartupError, shutdown_drain_seconds, startup_check + from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager params: Final[Mapping[str, object]] = uvicorn_main.make_context( "uvicorn", list(uvicorn_argv(argv, os.environ, None)) ).params - error: Final = startup_error( + checked: Final = startup_check( binary, { "--ssl-keyfile": params["ssl_keyfile"], @@ -106,18 +111,18 @@ def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, "--fd": params["fd"] is not None, }, ) - if error is not None: - sys.exit(error) - assert binary is not None - host, port = TypeAdapter(tuple[str, int]).validate_python((params["host"], params["port"])) - operator_trust: Final = TypeAdapter(str | None).validate_python(params["forwarded_allow_ips"]) - proxy_headers: Final = TypeAdapter(bool).validate_python(params["proxy_headers"]) - return ( - binary, - host, - port, - {PROXY_HEADERS: str(proxy_headers).lower()} - | ({} if operator_trust is None else {TRUSTED_PROXIES: operator_trust}), + if isinstance(checked, StartupError): + sys.exit(checked.reason) + graceful_timeout: Final = TypeAdapter[int | None](int | None).validate_python(params["timeout_graceful_shutdown"]) + return checked, PublicListener( + host=TypeAdapter(str).validate_python(params["host"]), + port=TypeAdapter(int).validate_python(params["port"]), + keepalive_seconds=TypeAdapter(int).validate_python(params["timeout_keep_alive"]), + shutdown_drain_seconds=shutdown_drain_seconds( + GracefulShutdownManager.get_timeout() if graceful_timeout is None else graceful_timeout + ), + proxy_headers=TypeAdapter(bool).validate_python(params["proxy_headers"]), + trusted_proxies=TypeAdapter[str | None](str | None).validate_python(params["forwarded_allow_ips"]), ) @@ -135,27 +140,21 @@ def configure_database() -> None: def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: - from litellm.proxy.rust_gateway import ( - Disabled, - Supervised, - Supervisor, - gateway_binary, - rust_gateway_mode, - supervise, - ) + from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides + from litellm.proxy.rust_forward.supervisor import gateway_binary, supervise - mode: Final = rust_gateway_mode(os.environ) + mode: Final = rust_forward_mode(os.environ) match mode: case Disabled(): configure_database() serve(uvicorn_argv(argv, os.environ, None)) case Supervisor(): - binary, host, port, trust = public_listener(argv, gateway_binary(sysconfig.get_path("scripts"))) + binary, listener = public_listener(argv, gateway_binary(sysconfig.get_path("scripts"))) configure_database() command: Final = (sys.executable, "-m", "gateway.launch", *argv) - sys.exit(supervise(binary, command, host, port, {**os.environ, **trust})) + sys.exit(supervise(binary, command, listener, os.environ)) case Supervised(): - serve(uvicorn_argv(argv, os.environ, mode)) + serve(uvicorn_argv(argv, os.environ, server_overrides(mode))) case _: assert_never(mode) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index d1520fd56d9..cb8e936f3c5 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -36,6 +36,10 @@ from litellm.types.llms.base import LiteLLMBaseModel if TYPE_CHECKING: from fastapi import FastAPI + + from litellm.proxy.rust_forward.listener import PublicListener + from litellm.proxy.rust_forward.mode import ServerOverrides + from litellm.proxy.rust_forward.peer import ASGIApp else: FastAPI = Any @@ -262,6 +266,7 @@ def _apply_config_logging_settings(litellm_settings: StartupOutputSettings | Non WORKER_APP: Final = "litellm_cli.main:proxy_worker_app" +FORWARD_APP: Final = "litellm.proxy.rust_forward.asgi:app" WORKER_LOG_CONFIG_ENV: Final = "LITELLM_WORKER_LOG_CONFIG" _LOG_CONFIG: Final = TypeAdapter[str | Mapping[str, object]](str | Mapping[str, object]) _LOG_CONFIG_FILE: Final = TypeAdapter[Mapping[str, object]](Mapping[str, object]) @@ -390,12 +395,12 @@ class ProxyInitializationHelpers: logging.config.fileConfig(log_config, disable_existing_loggers=False) @staticmethod - def child_process_uvicorn_overrides(log_config: object) -> Mapping[str, object]: + def child_process_uvicorn_overrides(log_config: object, app: str = WORKER_APP) -> Mapping[str, object]: if log_config is None: - return MappingProxyType({"app": WORKER_APP, "factory": True}) + return MappingProxyType({"app": app, "factory": True}) ProxyInitializationHelpers.apply_log_config(_LOG_CONFIG.validate_python(log_config)) os.environ[WORKER_LOG_CONFIG_ENV] = json.dumps(log_config) - return MappingProxyType({"app": WORKER_APP, "factory": True, "log_config": None}) + return MappingProxyType({"app": app, "factory": True, "log_config": None}) @staticmethod def apply_worker_log_config(environ: Mapping[str, str]) -> None: @@ -410,7 +415,7 @@ class ProxyInitializationHelpers: log_config: str | None = None, keepalive_timeout: int | None = None, timeout_worker_healthcheck: int | None = None, - listener_fd: int | None = None, + forwarded: "ServerOverrides | None" = None, ) -> dict: """ Get the arguments for `uvicorn` worker @@ -429,7 +434,16 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), - **({"fd": listener_fd, "proxy_headers": False} if listener_fd is not None else {}), + **( + {} + if forwarded is None + else { + "app": FORWARD_APP, + "factory": True, + "fd": forwarded.fd, + "proxy_headers": forwarded.proxy_headers, + } + ), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -677,13 +691,13 @@ class ProxyInitializationHelpers: def run_gunicorn_server( host: str, port: int, - app: FastAPI, + app: "ASGIApp", num_workers: int, ssl_certfile_path: str, ssl_keyfile_path: str, max_requests_before_restart: int | None = None, max_requests_before_restart_jitter: int | None = None, - listener_fd: int | None = None, + forwarded: "ServerOverrides | None" = None, ): """ Run litellm with `gunicorn` @@ -756,10 +770,10 @@ class ProxyInitializationHelpers: ) graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout() gunicorn_options: Final = { - "bind": f"fd://{listener_fd}" if listener_fd is not None else f"{host}:{port}", + "bind": f"{host}:{port}" if forwarded is None else f"fd://{forwarded.fd}", "workers": num_workers, # default is 1 "worker_class": build_graceful_shutdown_uvicorn_worker( - graceful_shutdown_timeout, proxy_headers=listener_fd is None + graceful_shutdown_timeout, proxy_headers=True if forwarded is None else forwarded.proxy_headers ), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, "preload": True, # Add the preload flag, @@ -829,6 +843,34 @@ class ProxyInitializationHelpers: _is_port_in_use = is_port_in_use + @staticmethod + def public_listener(host: str, port: int, keepalive_timeout: int | None, run_gunicorn: bool) -> "PublicListener": + """``run_gunicorn_server`` never hands ``--keepalive_timeout`` to gunicorn, so its workers keep gunicorn's.""" + from gunicorn.config import Config as GunicornConfig + + from litellm.proxy.rust_forward.listener import ( + PublicListener, + shutdown_drain_seconds, + uvicorn_default_keepalive, + ) + from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager + + keepalive: Final = ( + TypeAdapter(int).validate_python(GunicornConfig().keepalive) + if run_gunicorn + else (uvicorn_default_keepalive() if keepalive_timeout is None else keepalive_timeout) + ) + return PublicListener( + host=host, + port=random.randint(1024, 49152) + if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) + else port, + keepalive_seconds=keepalive, + shutdown_drain_seconds=shutdown_drain_seconds(GracefulShutdownManager.get_timeout()), + proxy_headers=True, + trusted_proxies=None, + ) + @staticmethod def get_loop_type(): """Helper function to determine the event loop type based on platform""" @@ -1286,17 +1328,17 @@ def run_server( run_setup_wizard() return - from litellm.proxy.rust_gateway import Supervised, Supervisor, rust_gateway_mode + from litellm.proxy.rust_forward.mode import Supervised, Supervisor, rust_forward_mode, server_overrides - gateway_mode: Final = rust_gateway_mode(os.environ) - if isinstance(gateway_mode, Supervisor) and not any( + forward_mode: Final = rust_forward_mode(os.environ) + if isinstance(forward_mode, Supervisor) and not any( (version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data) ): - from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise + from litellm.proxy.rust_forward.listener import StartupError, startup_check + from litellm.proxy.rust_forward.supervisor import cli_command, gateway_binary, supervise - binary: Final = gateway_binary(sysconfig.get_path("scripts")) - gateway_error: Final = startup_error( - binary, + binary: Final = startup_check( + gateway_binary(sysconfig.get_path("scripts")), { "--ssl_keyfile_path": ssl_keyfile_path, "--ssl_certfile_path": ssl_certfile_path, @@ -1305,14 +1347,11 @@ def run_server( "--run_granian": run_granian, }, ) - if gateway_error is not None: - raise click.ClickException(gateway_error) - assert binary is not None - public_port: Final = ( - random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port - ) - raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port, os.environ)) - supervised: Final = gateway_mode if isinstance(gateway_mode, Supervised) else None + if isinstance(binary, StartupError): + raise click.ClickException(binary.reason) + listener: Final = ProxyInitializationHelpers.public_listener(host, port, keepalive_timeout, run_gunicorn) + raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), listener, os.environ)) + forwarded: Final = server_overrides(forward_mode) if isinstance(forward_mode, Supervised) else None args: Final = locals() try: @@ -1763,7 +1802,7 @@ def run_server( if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None: from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process - metrics_bind_host: Final = supervised.metrics_host if supervised is not None else host + metrics_bind_host: Final = host if forwarded is None else forwarded.metrics_host try: metrics_process: Final = start_metrics_server_process( host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir @@ -1782,7 +1821,7 @@ def run_server( log_config=log_config, keepalive_timeout=keepalive_timeout, timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None), - listener_fd=supervised.listener_fd if supervised is not None else None, + forwarded=forwarded, ) # Optional: recycle uvicorn workers after N requests if max_requests_before_restart is not None: @@ -1822,7 +1861,7 @@ def run_server( start_query_engine_reaper() if num_workers > 1 or reload: worker_overrides: Final = ProxyInitializationHelpers.child_process_uvicorn_overrides( - uvicorn_args.get("log_config") + uvicorn_args.get("log_config"), WORKER_APP if forwarded is None else FORWARD_APP ) if reload and num_workers > 1: from litellm.proxy.dev_reload import ( @@ -1848,16 +1887,18 @@ def run_server( workers=num_workers, ) elif run_gunicorn is True: + from litellm.proxy.rust_forward.peer import wrap + ProxyInitializationHelpers.run_gunicorn_server( host=host, port=port, - app=app, + app=wrap(app, forward_mode), num_workers=num_workers, ssl_certfile_path=ssl_certfile_path, ssl_keyfile_path=ssl_keyfile_path, max_requests_before_restart=max_requests_before_restart, max_requests_before_restart_jitter=max_requests_before_restart_jitter, - listener_fd=supervised.listener_fd if supervised is not None else None, + forwarded=forwarded, ) elif run_hypercorn is True: ProxyInitializationHelpers.init_hypercorn_server( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8a50c116125..c5fcbfb2564 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -835,7 +835,6 @@ from litellm.proxy.plugin_routes import ( from litellm.proxy.plugin_routes import ( router as plugin_router, ) -from litellm.proxy.rust_gateway import RustPeerMiddleware, Supervised, rust_gateway_mode from litellm.proxy.spend_tracking.spend_event_producer import ( CollectorSettings, SpendEventProducer, @@ -21925,10 +21924,6 @@ if _server_root_paths: ) app.add_middleware(PerRequestRootPathMiddleware, root_paths=_server_root_paths) -_rust_gateway_mode: Final = rust_gateway_mode(os.environ) -if isinstance(_rust_gateway_mode, Supervised): - app.add_middleware(RustPeerMiddleware, mode=_rust_gateway_mode) - async def _stream_mcp_asgi_response(handle_fn, scope: dict, receive) -> "StreamingResponse": """ diff --git a/litellm/proxy/rust_forward/AGENTS.md b/litellm/proxy/rust_forward/AGENTS.md new file mode 100644 index 00000000000..a2ab93cde5c --- /dev/null +++ b/litellm/proxy/rust_forward/AGENTS.md @@ -0,0 +1,8 @@ +# Rust forwarding, Python half + +With `LITELLM_RUST` on, the supervisor here runs the Rust gateway on the public port and Python on a private loopback listener. This package is the Python half of forwarding and pairs with the crate `litellm-rust/crates/gateway-forward`. Once Rust serves every route, both get deleted together, along with one dispatch branch in each entry point + +- Nothing outside this package imports Rust forwarding concepts, except the dispatch, the flag adapter and the app wrap in `litellm/proxy/proxy_cli.py` and `gateway/launch.py`. The proxy app itself never knows Rust is in front of it +- Python must see each request as if the client had connected directly. `x-litellm-rust-peer` is the only Rust-specific input, and `peer.py` honors it only from the loopback hop +- Python owns every setting Rust needs. The supervisor resolves the host, port, keepalive and shutdown drain Python actually runs with and hands them to Rust as the `LITELLM_FORWARD_*` variables in `rust_spec`. Rust never reads Python's own settings, so a new value Rust needs gets a new variable there +- Entry points turn their own flags into a `PublicListener`, apply `server_overrides` in the supervised child, and wrap the app with `peer.wrap` where they load it diff --git a/litellm/proxy/rust_forward/__init__.py b/litellm/proxy/rust_forward/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/proxy/rust_forward/asgi.py b/litellm/proxy/rust_forward/asgi.py new file mode 100644 index 00000000000..ff9bdc57cf8 --- /dev/null +++ b/litellm/proxy/rust_forward/asgi.py @@ -0,0 +1,10 @@ +import os + +from litellm.proxy.rust_forward.mode import rust_forward_mode +from litellm.proxy.rust_forward.peer import ASGIApp, wrap + + +def app() -> ASGIApp: + from litellm_cli.main import proxy_worker_app + + return wrap(proxy_worker_app(), rust_forward_mode(os.environ)) diff --git a/litellm/proxy/rust_forward/listener.py b/litellm/proxy/rust_forward/listener.py new file mode 100644 index 00000000000..28595a4099e --- /dev/null +++ b/litellm/proxy/rust_forward/listener.py @@ -0,0 +1,46 @@ +import os +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +from litellm.rust_bridge.configuration import ENV_NAME + + +@dataclass(frozen=True, slots=True) +class PublicListener: + host: str + port: int + keepalive_seconds: int + shutdown_drain_seconds: float + proxy_headers: bool + trusted_proxies: str | None + + +@dataclass(frozen=True, slots=True) +class StartupError: + reason: str + + +def startup_check(binary: str | None, unsupported: Mapping[str, object]) -> str | StartupError: + flag: Final = next((name for name, value in unsupported.items() if value), None) + if flag is not None: + return StartupError(f"{ENV_NAME} does not support {flag}") + if os.name != "posix": + return StartupError(f"{ENV_NAME} requires POSIX inherited sockets and process signals") + if binary is None: + return StartupError( + f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment" + ) + return binary + + +def uvicorn_default_keepalive() -> int: + import uvicorn + + return uvicorn.Config(app="", log_config=None).timeout_keep_alive + + +def shutdown_drain_seconds(graceful_timeout: float) -> float: + from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS + + return graceful_timeout + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS diff --git a/litellm/proxy/rust_forward/mode.py b/litellm/proxy/rust_forward/mode.py new file mode 100644 index 00000000000..a75287688ea --- /dev/null +++ b/litellm/proxy/rust_forward/mode.py @@ -0,0 +1,68 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final, TypeAlias + +from pydantic import TypeAdapter + +from litellm.rust_bridge.configuration import environment_switch + +GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" +METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" +PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" +TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" +DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1" +RUST_HOP: Final = "127.0.0.1" + +_FD: Final = TypeAdapter(int) +_ENABLED: Final = TypeAdapter(bool) + + +@dataclass(frozen=True, slots=True) +class Disabled: + pass + + +@dataclass(frozen=True, slots=True) +class Supervisor: + pass + + +@dataclass(frozen=True, slots=True) +class Supervised: + listener_fd: int + metrics_host: str + proxy_headers: bool + trusted_proxies: str + + +RustForwardMode: TypeAlias = Disabled | Supervisor | Supervised + + +def rust_forward_mode(environ: Mapping[str, str]) -> RustForwardMode: + fd: Final = environ.get(GATEWAY_FD) + if fd is not None: + return Supervised( + _FD.validate_python(fd), + environ[METRICS_HOST], + _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), + environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), + ) + if environment_switch(environ): + return Supervisor() + return Disabled() + + +@dataclass(frozen=True, slots=True) +class ServerOverrides: + fd: int + proxy_headers: bool + metrics_host: str + + +def server_overrides(mode: Supervised) -> ServerOverrides: + """The private listener the supervised server binds instead of its own flags. + + Proxy headers stay off at the server, since it would judge Rust's loopback address instead of the real peer. + ``peer.wrap`` applies the operator's trust settings to the restored peer. + """ + return ServerOverrides(fd=mode.listener_fd, proxy_headers=False, metrics_host=mode.metrics_host) diff --git a/litellm/proxy/rust_forward/peer.py b/litellm/proxy/rust_forward/peer.py new file mode 100644 index 00000000000..2d03d6ca35c --- /dev/null +++ b/litellm/proxy/rust_forward/peer.py @@ -0,0 +1,57 @@ +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Final, TypeAlias + +from typing_extensions import assert_never + +from litellm.proxy.rust_forward.mode import RUST_HOP, Disabled, RustForwardMode, Supervised, Supervisor + +if TYPE_CHECKING: + from uvicorn._types import ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope + +PEER_HEADER: Final = b"x-litellm-rust-peer" + +ASGIApp: TypeAlias = Callable[..., Awaitable[None]] + + +def _restore_peer(scope: "WWWScope") -> None: + headers: Final = tuple(scope["headers"]) + peer: Final = next((value for name, value in headers if name == PEER_HEADER), None) + if peer is None: + return + remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER) + scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop + client: Final = scope["client"] + if client is None or client[0] != RUST_HOP: + return + host, _, port = peer.decode("latin-1").rpartition(":") + scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope + + +class RustPeerMiddleware: + """Make Python see each request as if the client had connected to it directly. + + Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real + TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the + real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust + settings Python would have used on the public port. + """ + + def __init__(self, app: ASGIApp, mode: Supervised) -> None: + from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + + self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app + + async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + if scope["type"] != "lifespan": + _restore_peer(scope) + await self.app(scope, receive, send) + + +def wrap(app: ASGIApp, mode: RustForwardMode) -> ASGIApp: + match mode: + case Supervised(): + return RustPeerMiddleware(app, mode) + case Disabled() | Supervisor(): + return app + case _: + assert_never(mode) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_forward/supervisor.py similarity index 53% rename from litellm/proxy/rust_gateway.py rename to litellm/proxy/rust_forward/supervisor.py index 8be060e362c..6ab4b86dca3 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_forward/supervisor.py @@ -9,115 +9,22 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from enum import Enum, IntEnum from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeAlias +from typing import Final, TypeAlias from pydantic import TypeAdapter from typing_extensions import assert_never -from litellm.rust_bridge.configuration import ENV_NAME, environment_switch +from litellm.proxy.rust_forward.listener import PublicListener +from litellm.proxy.rust_forward.mode import GATEWAY_FD, METRICS_HOST, PROXY_HEADERS, RUST_HOP, TRUSTED_PROXIES -if TYPE_CHECKING: - from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope - -GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" -METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" -PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" -TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" -DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1" -RUST_HOP: Final = "127.0.0.1" -PEER_HEADER: Final = b"x-litellm-rust-peer" - -_FD: Final = TypeAdapter(int) -_ENABLED: Final = TypeAdapter(bool) _ADDRESS: Final = TypeAdapter(tuple[str, int]) -@dataclass(frozen=True, slots=True) -class Disabled: - pass - - -@dataclass(frozen=True, slots=True) -class Supervisor: - pass - - -@dataclass(frozen=True, slots=True) -class Supervised: - listener_fd: int - metrics_host: str - proxy_headers: bool - trusted_proxies: str - - -RustGatewayMode: TypeAlias = Disabled | Supervisor | Supervised - - -def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode: - fd: Final = environ.get(GATEWAY_FD) - if fd is not None: - return Supervised( - _FD.validate_python(fd), - environ[METRICS_HOST], - _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), - environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), - ) - if environment_switch(environ): - return Supervisor() - return Disabled() - - -def _restore_peer(scope: "WWWScope") -> None: - headers: Final = tuple(scope["headers"]) - peer: Final = next((value for name, value in headers if name == PEER_HEADER), None) - if peer is None: - return - remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER) - scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop - client: Final = scope["client"] - if client is None or client[0] != RUST_HOP: - return - host, _, port = peer.decode("latin-1").rpartition(":") - scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope - - -class RustPeerMiddleware: - """Make Python see each request as if the client had connected to it directly. - - Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real - TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the - real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust - settings Python would have used on the public port. The server's proxy-header handling is off in - this mode, since it would judge Rust's loopback address instead of the real peer. - """ - - def __init__(self, app: "ASGI3Application", mode: Supervised) -> None: - from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware - - self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app - - async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: - if scope["type"] != "lifespan": - _restore_peer(scope) - await self.app(scope, receive, send) - - def gateway_binary(scripts_dir: str) -> str | None: name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway" return shutil.which(name, path=scripts_dir) -def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: - flag: Final = next((name for name, value in unsupported.items() if value), None) - if flag is not None: - return f"{ENV_NAME} does not support {flag}" - if os.name != "posix": - return f"{ENV_NAME} requires POSIX inherited sockets and process signals" - if binary is None: - return f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment" - return None - - class Child(Enum): RUST = "rust" PYTHON = "python" @@ -209,7 +116,7 @@ class ChildSpec: def python_spec( - command: Sequence[str], listener_fd: int, upstream_port: int, public_host: str, environ: Mapping[str, str] + command: Sequence[str], listener_fd: int, upstream_port: int, listener: PublicListener, environ: Mapping[str, str] ) -> ChildSpec: return ChildSpec( argv=(*command, "--host", RUST_HOP, "--port", str(upstream_port)), @@ -217,22 +124,26 @@ def python_spec( { **environ, GATEWAY_FD: str(listener_fd), - METRICS_HOST: public_host, + METRICS_HOST: listener.host, + PROXY_HEADERS: str(listener.proxy_headers).lower(), + **({} if listener.trusted_proxies is None else {TRUSTED_PROXIES: listener.trusted_proxies}), } ), pass_fds=(listener_fd,), ) -def rust_spec(binary: str, upstream_port: int, host: str, port: int, environ: Mapping[str, str]) -> ChildSpec: +def rust_spec(binary: str, upstream_port: int, listener: PublicListener, environ: Mapping[str, str]) -> ChildSpec: return ChildSpec( argv=(binary,), env=MappingProxyType( { **environ, - "HOST": host, - "PORT": str(port), "LITELLM_FORWARD_UPSTREAM": f"http://{RUST_HOP}:{upstream_port}", + "LITELLM_FORWARD_HOST": listener.host, + "LITELLM_FORWARD_PORT": str(listener.port), + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE": str(listener.keepalive_seconds), + "LITELLM_FORWARD_SHUTDOWN_DRAIN": str(listener.shutdown_drain_seconds), } ), pass_fds=(), @@ -266,18 +177,20 @@ async def _drive( return await _drive(following, events, children) -async def _supervise(binary: str, command: Sequence[str], host: str, port: int, environ: Mapping[str, str]) -> ExitCode: +async def _supervise( + binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str] +) -> ExitCode: loop: Final = asyncio.get_running_loop() events: Final[asyncio.Queue[Event]] = asyncio.Queue() for signum in (signal.SIGTERM, signal.SIGINT): loop.add_signal_handler(signum, events.put_nowait, Stop(signum)) - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: - listener.bind((RUST_HOP, 0)) - listener.listen() - _, upstream_port = _ADDRESS.validate_python(listener.getsockname()) - python: Final = await _spawn(python_spec(command, listener.fileno(), upstream_port, host, environ)) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as private: + private.bind((RUST_HOP, 0)) + private.listen() + _, upstream_port = _ADDRESS.validate_python(private.getsockname()) + python: Final = await _spawn(python_spec(command, private.fileno(), upstream_port, listener, environ)) try: - rust: Final = await _spawn(rust_spec(binary, upstream_port, host, port, environ)) + rust: Final = await _spawn(rust_spec(binary, upstream_port, listener, environ)) except OSError: _terminate_group(python) await python.wait() @@ -294,9 +207,9 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int, await asyncio.gather(*waiters) -def supervise(binary: str, command: Sequence[str], host: str, port: int, environ: Mapping[str, str]) -> int: +def supervise(binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str]) -> int: try: - return asyncio.run(_supervise(binary, command, host, port, environ)) + return asyncio.run(_supervise(binary, command, listener, environ)) except OSError as error: sys.stderr.write(f"Rust gateway startup failed: {error}\n") return ExitCode.CHILD_FAILED diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_forward.py similarity index 100% rename from tests/integration/configuration/test_rust_gateway.py rename to tests/integration/configuration/test_rust_forward.py diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index 418e0358b64..36eaabfb029 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -12,14 +12,18 @@ from typing import Final, cast from unittest.mock import MagicMock, patch import pytest +import uvicorn from uvicorn.importer import import_from_string from uvicorn.main import main as uvicorn_main from gateway.launch import GATEWAY_APP, main, pool_database_url, public_listener, uvicorn_argv +from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings from litellm.proxy.proxy_server import app as proxy_app -from litellm.proxy.rust_gateway import Supervised +from litellm.proxy.rust_forward.mode import Supervised, server_overrides +from litellm.proxy.rust_forward.supervisor import rust_spec +from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager DB_ENV: Final = { "DATABASE_HOST": "db.internal", @@ -127,34 +131,59 @@ class TestUvicornArgv: @pytest.mark.parametrize( ("argv", "trust"), [ - ((), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true"}), - ( - ("--forwarded-allow-ips", "10.0.0.5"), - {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "10.0.0.5"}, - ), - (("--forwarded-allow-ips=*",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "*"}), - (("--no-proxy-headers",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}), + ((), (True, None)), + (("--forwarded-allow-ips", "10.0.0.5"), (True, "10.0.0.5")), + (("--forwarded-allow-ips=*",), (True, "*")), + (("--no-proxy-headers",), (False, None)), ], ) def test_forwarding_carries_operator_proxy_header_options_to_the_child( - self, argv: tuple[str, ...], trust: dict[str, str] + self, argv: tuple[str, ...], trust: tuple[bool, str | None], monkeypatch: pytest.MonkeyPatch ) -> None: - assert public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") == ( - "gateway", - "0.0.0.0", - 4000, - trust, - ) + monkeypatch.delenv("FORWARDED_ALLOW_IPS", raising=False) + binary, listener = public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") + assert (binary, listener.host, listener.port) == ("gateway", "0.0.0.0", 4000) + assert (listener.proxy_headers, listener.trusted_proxies) == trust + + @pytest.mark.parametrize( + ("argv", "environ"), + [(("--timeout-keep-alive", "2"), {}), ((), {"KEEPALIVE_TIMEOUT": "7"}), ((), {})], + ) + def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with( + self, argv: tuple[str, ...], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + for name, value in environ.items(): + monkeypatch.setenv(name, value) + _, listener = public_listener(argv, "gateway") + sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"] + assert sent == str(_uvicorn_params(uvicorn_argv(argv, os.environ, None))["timeout_keep_alive"]) + + def test_without_a_keepalive_rust_follows_uvicorns_own_default(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + _, listener = public_listener((), "gateway") + assert listener.keepalive_seconds == uvicorn.Config(app="", log_config=None).timeout_keep_alive + + @pytest.mark.parametrize("argv", [(), ("--timeout-graceful-shutdown", "12")]) + def test_rust_drains_for_pythons_graceful_timeout_plus_teardown( + self, argv: tuple[str, ...], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "7.5") + _, listener = public_listener(argv, "gateway") + flag: Final = _uvicorn_params(uvicorn_argv(argv, {}, None))["timeout_graceful_shutdown"] + graceful: Final = GracefulShutdownManager.get_timeout() if flag is None else float(str(flag)) + sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"] + assert float(sent) == graceful + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS def test_supervised_listener_uses_the_inherited_loopback_socket(self): - supervised: Final = Supervised(17, "0.0.0.0", True, "127.0.0.1") + supervised: Final = server_overrides(Supervised(17, "0.0.0.0", True, "127.0.0.1")) params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised)) assert params["fd"] == 17 assert params["workers"] == 2 @pytest.mark.parametrize("argv", [(), ("--proxy-headers",), ("--forwarded-allow-ips=*",)]) def test_private_hop_leaves_proxy_headers_to_the_app(self, argv: tuple[str, ...]): - params: Final = _uvicorn_params(uvicorn_argv(argv, {}, Supervised(17, "0.0.0.0", True, "*"))) + params: Final = _uvicorn_params(uvicorn_argv(argv, {}, server_overrides(Supervised(17, "0.0.0.0", True, "*")))) assert params["proxy_headers"] is False def test_direct_listener_keeps_explicit_proxy_header_options(self): diff --git a/tests/unit/proxy/rust_forward/__init__.py b/tests/unit/proxy/rust_forward/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/rust_forward/test_listener.py b/tests/unit/proxy/rust_forward/test_listener.py new file mode 100644 index 00000000000..d17b0e74e14 --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_listener.py @@ -0,0 +1,74 @@ +from collections.abc import Mapping +from typing import Final + +import pytest +import uvicorn +from click.testing import CliRunner +from gunicorn.config import Config as GunicornConfig + +from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS +from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server +from litellm.proxy.rust_forward.listener import PublicListener, StartupError, startup_check +from litellm.proxy.rust_forward.supervisor import rust_spec + + +@pytest.mark.parametrize( + "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] +) +def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "true") + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] + result: Final = CliRunner().invoke(run_server, [flag, *values]) + assert result.exit_code != 0 + assert f"LITELLM_RUST does not support {flag}" in result.output + + +def test_missing_binary_gives_install_instruction() -> None: + assert startup_check(None, {}) == StartupError( + "LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment" + ) + + +def _cli_listener(argv: list[str]) -> tuple[Mapping[str, object], PublicListener]: + params: Final = run_server.make_context("litellm", ["--port", "45999", *argv]).params + return params, ProxyInitializationHelpers.public_listener( + str(params["host"]), int(str(params["port"])), params["keepalive_timeout"], params["run_gunicorn"] is True + ) + + +def _uvicorn_keepalive(options: Mapping[str, object]) -> int: + configured: Final = {"timeout_keep_alive": options["timeout_keep_alive"]} if "timeout_keep_alive" in options else {} + return uvicorn.Config(app="", log_config=None, **configured).timeout_keep_alive + + +@pytest.mark.parametrize( + ("argv", "environ"), + [(["--keepalive_timeout", "2"], {}), ([], {"KEEPALIVE_TIMEOUT": "7"}), ([], {})], +) +def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with( + argv: list[str], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + for name, value in environ.items(): + monkeypatch.setenv(name, value) + params, listener = _cli_listener(argv) + uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 45555, keepalive_timeout=params["keepalive_timeout"] + ) + sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"] + assert sent == str(_uvicorn_keepalive(uvicorn_options)) + + +def test_gunicorn_workers_keep_gunicorns_keepalive_and_rust_follows(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + _, listener = _cli_listener(["--run_gunicorn", "--keepalive_timeout", "9"]) + assert listener.keepalive_seconds == GunicornConfig().keepalive + + +def test_rust_drains_for_pythons_graceful_timeout_plus_teardown(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "12.5") + _, listener = _cli_listener([]) + uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555) + sent: Final = float(rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"]) + assert sent == uvicorn_options["timeout_graceful_shutdown"] + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS diff --git a/tests/unit/proxy/rust_forward/test_mode.py b/tests/unit/proxy/rust_forward/test_mode.py new file mode 100644 index 00000000000..20005a21255 --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_mode.py @@ -0,0 +1,43 @@ +from typing import Final + +import pytest + +from litellm.proxy.proxy_cli import ProxyInitializationHelpers +from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides + + +@pytest.mark.parametrize( + ("environ", "expected"), + [ + ({}, Disabled()), + ({"LITELLM_RUST": "false"}, Disabled()), + ({"LITELLM_RUST": "off"}, Disabled()), + ({"LITELLM_RUST": "0"}, Disabled()), + ({"LITELLM_RUST": "true"}, Supervisor()), + ({"LITELLM_RUST": "1"}, Supervisor()), + ({"LITELLM_RUST": "yes"}, Supervisor()), + ({"LITELLM_RUST": "on"}, Supervisor()), + ({"LITELLM_RUST": " 1 "}, Supervisor()), + ({"LITELLM_RUST": "enabled"}, Disabled()), + ( + { + "LITELLM_RUST": "true", + "_LITELLM_GATEWAY_FD": "17", + "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", + "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", + }, + Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"), + ), + ], +) +def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None: + assert rust_forward_mode(environ) == expected + + +@pytest.mark.parametrize("mode", [None, Supervised(17, "0.0.0.0", True, "127.0.0.1")]) +def test_inherited_socket_reaches_uvicorn(mode: Supervised | None) -> None: + options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 45555, forwarded=None if mode is None else server_overrides(mode) + ) + assert options.get("fd") == (None if mode is None else mode.listener_fd) + assert options.get("proxy_headers", True) is (mode is None) diff --git a/tests/unit/proxy/rust_forward/test_peer.py b/tests/unit/proxy/rust_forward/test_peer.py new file mode 100644 index 00000000000..e96d9cf2b6a --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_peer.py @@ -0,0 +1,144 @@ +import asyncio +import gc +from typing import TYPE_CHECKING, Final + +import httpx +import pytest +from fastapi import Request +from fastapi.routing import APIRoute +from uvicorn.importer import import_from_string +from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + +if TYPE_CHECKING: + from uvicorn._types import ( + ASGIReceiveCallable, + ASGISendCallable, + ASGISendEvent, + HTTPDisconnectEvent, + HTTPScope, + Scope, + ) + +from litellm.proxy.proxy_server import app as proxy_app +from litellm.proxy.rust_forward.mode import Supervised +from litellm.proxy.rust_forward.peer import wrap + + +Headers = tuple[tuple[bytes, bytes], ...] + + +async def _receive() -> "HTTPDisconnectEvent": + return {"type": "http.disconnect"} + + +async def _send(message: "ASGISendEvent") -> None: + return None + + +async def _seen_by_app( + behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised +) -> tuple[tuple[str, int] | None, str, Headers]: + seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = ( + asyncio.get_running_loop().create_future() + ) + + async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + assert scope["type"] == "http" + seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"]))) + + direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe + app: Final = wrap(probe, mode) if behind_rust else direct + scope: Final[HTTPScope] = { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": "/", + "raw_path": b"/", + "query_string": b"", + "root_path": "", + "headers": headers, + "client": client, + "server": ("127.0.0.1", 4000), + } + await app(scope, _receive, _send) + return seen.result() + + +PEER: Final = ("203.0.113.7", 51234) +TRUSTED_PEER: Final = ("10.0.12.34", 40000) +FORWARDED: Final[Headers] = ( + (b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"), + (b"x-forwarded-proto", b"https"), +) + + +@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)]) +@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))]) +@pytest.mark.parametrize( + "mode", + [ + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + Supervised(17, "0.0.0.0", True, "10.0.0.0/8"), + Supervised(17, "0.0.0.0", True, "*"), + Supervised(17, "0.0.0.0", False, "10.0.0.0/8"), + ], +) +async def test_python_behind_rust_sees_what_a_direct_connection_would( + peer: tuple[str, int], headers: Headers, mode: Supervised +) -> None: + rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}" + behind_rust: Final = await _seen_by_app( + True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode + ) + direct: Final = await _seen_by_app(False, peer, headers, mode) + assert behind_rust == direct + + +async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None: + client, scheme, _ = await _seen_by_app( + True, + ("127.0.0.1", 60000), + ((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")), + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + ) + assert (client, scheme) == (("203.0.113.7", 51234), "http") + + +async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None: + client, _, headers = await _seen_by_app( + True, + ("192.0.2.9", 1000), + ((b"x-litellm-rust-peer", b"10.0.12.34:1"),), + Supervised(17, "0.0.0.0", False, "127.0.0.1"), + ) + assert (client, headers) == (("192.0.2.9", 1000), ()) + + +async def _peer_probe(request: Request) -> dict[str, object]: + assert request.client is not None + return {"client": [request.client.host, request.client.port], "peer": request.headers.get("x-litellm-rust-peer")} + + +@pytest.mark.parametrize("entry_point", ["litellm.proxy.rust_forward.asgi:app", "gateway.launch:worker_app"]) +async def test_every_entry_point_hands_routes_the_peer_rust_saw( + entry_point: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17") + monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", "0.0.0.0") + monkeypatch.setattr(proxy_app.router, "lifespan_context", proxy_app.router.lifespan_context) + monkeypatch.setattr( + proxy_app.router, "routes", [APIRoute("/__rust_forward_peer_probe", _peer_probe), *proxy_app.router.routes] + ) + try: + served: Final = import_from_string(entry_point)() + finally: + gc.unfreeze() + transport: Final = httpx.ASGITransport(app=served, client=("127.0.0.1", 60000)) + async with httpx.AsyncClient(transport=transport, base_url="http://proxy") as client: + response: Final = await client.get( + "/__rust_forward_peer_probe", headers={"x-litellm-rust-peer": "203.0.113.7:51234"} + ) + assert response.status_code == 200, response.text + assert response.json() == {"client": ["203.0.113.7", 51234], "peer": None} diff --git a/tests/unit/proxy/rust_forward/test_supervisor.py b/tests/unit/proxy/rust_forward/test_supervisor.py new file mode 100644 index 00000000000..224d97f349d --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_supervisor.py @@ -0,0 +1,139 @@ +import functools +import signal +import sys +from pathlib import Path +from typing import Final + +import pytest + +from litellm.proxy.proxy_cli import run_server +from litellm.proxy.rust_forward.listener import PublicListener +from litellm.proxy.rust_forward.mode import Supervised, rust_forward_mode +from litellm.proxy.rust_forward.supervisor import ( + Child, + Event, + ExitCode, + Exited, + Finished, + Running, + Signal, + State, + Stop, + cli_command, + gateway_binary, + python_spec, + rust_spec, + step, +) + +TERM: Final = signal.SIGTERM +INT: Final = signal.SIGINT + + +def _listener(proxy_headers: bool = True, trusted_proxies: str | None = None) -> PublicListener: + return PublicListener("0.0.0.0", 4000, 5, 40.0, proxy_headers, trusted_proxies) + + +@pytest.mark.parametrize( + ("environ", "listener", "expected"), + [ + ({}, _listener(), Supervised(17, "0.0.0.0", True, "127.0.0.1")), + ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(), Supervised(17, "0.0.0.0", True, "10.0.0.5")), + ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(True, "*"), Supervised(17, "0.0.0.0", True, "*")), + ({}, _listener(False), Supervised(17, "0.0.0.0", False, "127.0.0.1")), + ], +) +def test_python_child_keeps_the_trust_python_alone_would_use( + environ: dict[str, str], listener: PublicListener, expected: Supervised +) -> None: + spec: Final = python_spec(("python",), 17, 45555, listener, environ) + assert rust_forward_mode(spec.env) == expected + + +def test_python_child_listens_privately_and_keeps_options() -> None: + arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") + spec: Final = python_spec(arguments, 17, 45555, _listener(), {}) + params: Final = run_server.make_context("litellm", list(spec.argv)).params + assert params["host"] == "127.0.0.1" + assert params["port"] == 45555 + assert params["num_workers"] == 2 + assert params["reload"] is True + assert spec.pass_fds == (17,) + assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli") + + +def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None: + spec: Final = rust_spec( + "gateway", 45555, PublicListener("0.0.0.0", 4000, 2, 12.5, True, None), {"RUST_LOG": "info"} + ) + assert spec.argv == ("gateway",) + assert spec.pass_fds == () + assert dict(spec.env) == { + "RUST_LOG": "info", + "LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555", + "LITELLM_FORWARD_HOST": "0.0.0.0", + "LITELLM_FORWARD_PORT": "4000", + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE": "2", + "LITELLM_FORWARD_SHUTDOWN_DRAIN": "12.5", + } + + +def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + scripts: Final = tmp_path / "scripts" + elsewhere: Final = tmp_path / "elsewhere" + scripts.mkdir() + elsewhere.mkdir() + binary: Final = elsewhere / "litellm-gateway" + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o755) + monkeypatch.setenv("PATH", str(elsewhere)) + assert gateway_binary(str(scripts)) is None + assert gateway_binary(str(elsewhere)) == str(binary) + + +def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]: + def advance(acc: tuple[State, tuple[Signal, ...]], event: Event) -> tuple[State, tuple[Signal, ...]]: + state, sent = acc + following, action = step(state, event) + return following, sent if action is None else (*sent, action) + + return functools.reduce(advance, events, (Running(), ())) + + +RUST: Final = Child.RUST +PYTHON: Final = Child.PYTHON + + +@pytest.mark.parametrize( + ("events", "signals", "code"), + [ + ( + (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 0)), + (Signal(RUST, TERM), Signal(PYTHON, TERM)), + ExitCode.CLEAN, + ), + ((Exited(PYTHON, 0),), (), ExitCode.CHILD_FAILED), + ((Exited(RUST, 1),), (), ExitCode.CHILD_FAILED), + ((Stop(TERM), Exited(PYTHON, 0)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), + ((Stop(TERM), Exited(RUST, 1)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), + ( + (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 1)), + (Signal(RUST, TERM), Signal(PYTHON, TERM)), + ExitCode.CHILD_FAILED, + ), + ( + (Stop(INT), Stop(INT), Exited(RUST, 0), Exited(PYTHON, 0)), + (Signal(RUST, INT), Signal(PYTHON, INT)), + ExitCode.CLEAN, + ), + ( + (Exited(RUST, 1), Stop(TERM), Exited(PYTHON, 0)), + (), + ExitCode.CHILD_FAILED, + ), + ], +) +def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit( + events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode +) -> None: + assert _fold(events) == (Finished(code), signals) diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index db36f0d681f..0ce5cc58191 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -1550,7 +1550,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=30, timeout_worker_healthcheck=None, - listener_fd=None, + forwarded=None, ) mock_uvicorn_run.assert_called_once() @@ -1627,7 +1627,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=None, timeout_worker_healthcheck=15, - listener_fd=None, + forwarded=None, ) @patch("uvicorn.run") diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py deleted file mode 100644 index 1072148be9f..00000000000 --- a/tests/unit/proxy/test_rust_gateway.py +++ /dev/null @@ -1,293 +0,0 @@ -import asyncio -import functools -import signal -import sys -from pathlib import Path -from typing import TYPE_CHECKING, Final - -import pytest -from click.testing import CliRunner -from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware - -if TYPE_CHECKING: - from uvicorn._types import ( - ASGIReceiveCallable, - ASGISendCallable, - ASGISendEvent, - HTTPDisconnectEvent, - HTTPScope, - Scope, - ) - -from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server -from litellm.proxy.rust_gateway import ( - Child, - Disabled, - Event, - ExitCode, - Exited, - Finished, - Running, - RustPeerMiddleware, - Signal, - State, - Stop, - Supervised, - Supervisor, - cli_command, - gateway_binary, - python_spec, - rust_gateway_mode, - rust_spec, - startup_error, - step, -) - -TERM: Final = signal.SIGTERM -INT: Final = signal.SIGINT - - -@pytest.mark.parametrize( - "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] -) -def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("LITELLM_RUST", "true") - monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) - values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] - result: Final = CliRunner().invoke(run_server, [flag, *values]) - assert result.exit_code != 0 - assert f"LITELLM_RUST does not support {flag}" in result.output - - -@pytest.mark.parametrize( - ("environ", "expected"), - [ - ({}, Disabled()), - ({"LITELLM_RUST": "false"}, Disabled()), - ({"LITELLM_RUST": "off"}, Disabled()), - ({"LITELLM_RUST": "0"}, Disabled()), - ({"LITELLM_RUST": "true"}, Supervisor()), - ({"LITELLM_RUST": "1"}, Supervisor()), - ({"LITELLM_RUST": "yes"}, Supervisor()), - ({"LITELLM_RUST": "on"}, Supervisor()), - ({"LITELLM_RUST": " 1 "}, Supervisor()), - ({"LITELLM_RUST": "enabled"}, Disabled()), - ( - { - "LITELLM_RUST": "true", - "_LITELLM_GATEWAY_FD": "17", - "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", - "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", - }, - Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"), - ), - ], -) -def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None: - assert rust_gateway_mode(environ) == expected - - -@pytest.mark.parametrize( - ("environ", "expected"), - [ - ({}, Supervised(17, "0.0.0.0", True, "127.0.0.1")), - ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, Supervised(17, "0.0.0.0", True, "10.0.0.5")), - ({"FORWARDED_ALLOW_IPS": "*"}, Supervised(17, "0.0.0.0", True, "*")), - ({"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}, Supervised(17, "0.0.0.0", False, "127.0.0.1")), - ], -) -def test_python_child_keeps_the_trust_python_alone_would_use(environ: dict[str, str], expected: Supervised) -> None: - spec: Final = python_spec(("python",), 17, 45555, "0.0.0.0", environ) - assert rust_gateway_mode(spec.env) == expected - - -def test_python_child_listens_privately_and_keeps_options() -> None: - arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") - spec: Final = python_spec(arguments, 17, 45555, "0.0.0.0", {}) - params: Final = run_server.make_context("litellm", list(spec.argv)).params - assert params["host"] == "127.0.0.1" - assert params["port"] == 45555 - assert params["num_workers"] == 2 - assert params["reload"] is True - assert spec.pass_fds == (17,) - assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli") - - -def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None: - spec: Final = rust_spec("gateway", 45555, "0.0.0.0", 4000, {"RUST_LOG": "info"}) - assert spec.argv == ("gateway",) - assert spec.pass_fds == () - assert dict(spec.env) == { - "RUST_LOG": "info", - "HOST": "0.0.0.0", - "PORT": "4000", - "LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555", - } - - -def test_missing_binary_gives_install_instruction() -> None: - assert startup_error(None, {}) == ( - "LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment" - ) - - -def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - scripts: Final = tmp_path / "scripts" - elsewhere: Final = tmp_path / "elsewhere" - scripts.mkdir() - elsewhere.mkdir() - binary: Final = elsewhere / "litellm-gateway" - binary.write_text("#!/bin/sh\nexit 0\n") - binary.chmod(0o755) - monkeypatch.setenv("PATH", str(elsewhere)) - assert gateway_binary(str(scripts)) is None - assert gateway_binary(str(elsewhere)) == str(binary) - - -@pytest.mark.parametrize("listener_fd", [None, 17]) -def test_inherited_socket_reaches_uvicorn(listener_fd: int | None) -> None: - options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( - "127.0.0.1", 45555, listener_fd=listener_fd - ) - assert options.get("fd") == listener_fd - assert options.get("proxy_headers", True) is (listener_fd is None) - - -def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]: - def advance(acc: tuple[State, tuple[Signal, ...]], event: Event) -> tuple[State, tuple[Signal, ...]]: - state, sent = acc - following, action = step(state, event) - return following, sent if action is None else (*sent, action) - - return functools.reduce(advance, events, (Running(), ())) - - -RUST: Final = Child.RUST -PYTHON: Final = Child.PYTHON - - -@pytest.mark.parametrize( - ("events", "signals", "code"), - [ - ( - (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 0)), - (Signal(RUST, TERM), Signal(PYTHON, TERM)), - ExitCode.CLEAN, - ), - ((Exited(PYTHON, 0),), (), ExitCode.CHILD_FAILED), - ((Exited(RUST, 1),), (), ExitCode.CHILD_FAILED), - ((Stop(TERM), Exited(PYTHON, 0)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), - ((Stop(TERM), Exited(RUST, 1)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), - ( - (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 1)), - (Signal(RUST, TERM), Signal(PYTHON, TERM)), - ExitCode.CHILD_FAILED, - ), - ( - (Stop(INT), Stop(INT), Exited(RUST, 0), Exited(PYTHON, 0)), - (Signal(RUST, INT), Signal(PYTHON, INT)), - ExitCode.CLEAN, - ), - ( - (Exited(RUST, 1), Stop(TERM), Exited(PYTHON, 0)), - (), - ExitCode.CHILD_FAILED, - ), - ], -) -def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit( - events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode -) -> None: - assert _fold(events) == (Finished(code), signals) - - -Headers = tuple[tuple[bytes, bytes], ...] - - -async def _receive() -> "HTTPDisconnectEvent": - return {"type": "http.disconnect"} - - -async def _send(message: "ASGISendEvent") -> None: - return None - - -async def _seen_by_app( - behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised -) -> tuple[tuple[str, int] | None, str, Headers]: - seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = ( - asyncio.get_running_loop().create_future() - ) - - async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: - assert scope["type"] == "http" - seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"]))) - - direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe - app: Final = RustPeerMiddleware(probe, mode) if behind_rust else direct - scope: Final[HTTPScope] = { - "type": "http", - "asgi": {"version": "3.0", "spec_version": "2.3"}, - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": "/", - "raw_path": b"/", - "query_string": b"", - "root_path": "", - "headers": headers, - "client": client, - "server": ("127.0.0.1", 4000), - } - await app(scope, _receive, _send) - return seen.result() - - -PEER: Final = ("203.0.113.7", 51234) -TRUSTED_PEER: Final = ("10.0.12.34", 40000) -FORWARDED: Final[Headers] = ( - (b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"), - (b"x-forwarded-proto", b"https"), -) - - -@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)]) -@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))]) -@pytest.mark.parametrize( - "mode", - [ - Supervised(17, "0.0.0.0", True, "127.0.0.1"), - Supervised(17, "0.0.0.0", True, "10.0.0.0/8"), - Supervised(17, "0.0.0.0", True, "*"), - Supervised(17, "0.0.0.0", False, "10.0.0.0/8"), - ], -) -async def test_python_behind_rust_sees_what_a_direct_connection_would( - peer: tuple[str, int], headers: Headers, mode: Supervised -) -> None: - rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}" - behind_rust: Final = await _seen_by_app( - True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode - ) - direct: Final = await _seen_by_app(False, peer, headers, mode) - assert behind_rust == direct - - -async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None: - client, scheme, _ = await _seen_by_app( - True, - ("127.0.0.1", 60000), - ((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")), - Supervised(17, "0.0.0.0", True, "127.0.0.1"), - ) - assert (client, scheme) == (("203.0.113.7", 51234), "http") - - -async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None: - client, _, headers = await _seen_by_app( - True, - ("192.0.2.9", 1000), - ((b"x-litellm-rust-peer", b"10.0.12.34:1"),), - Supervised(17, "0.0.0.0", False, "127.0.0.1"), - ) - assert (client, headers) == (("192.0.2.9", 1000), ())