diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 17f68cd7d32..5e61d6ebd4c 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -583,6 +583,7 @@ jobs: tests/unit/proxy/credential_endpoints tests/unit/proxy/discovery_endpoints tests/unit/proxy/shutdown + tests/unit/proxy/rust_forward tests/unit/proxy/public_endpoints tests/unit/proxy/prompts tests/unit/proxy/rag_endpoints diff --git a/gateway/launch.py b/gateway/launch.py index 59f7121e513..89341a4cb89 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -19,29 +19,43 @@ Run with: import os import sys +import sysconfig from collections.abc import Callable, Mapping, Sequence from typing import TYPE_CHECKING, Final +from typing_extensions import assert_never from uvicorn.main import main as uvicorn_main from litellm_cli.main import without_gc_pauses if TYPE_CHECKING: - from fastapi import FastAPI - from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings + from litellm.proxy.rust_forward.listener import PublicListener + from litellm.proxy.rust_forward.mode import ServerOverrides + from litellm.proxy.rust_forward.peer import ASGIApp GATEWAY_APP: Final = "gateway.launch:worker_app" KEEPALIVE_FLAG: Final = "--timeout-keep-alive" -def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str, ...]: +def uvicorn_argv( + argv: Sequence[str], environ: Mapping[str, str], forwarded: "ServerOverrides | None" +) -> tuple[str, ...]: """Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly.""" keepalive: Final = environ.get("KEEPALIVE_TIMEOUT") + inherited_socket: Final = ( + () + if forwarded is None + else ( + "--fd", + str(forwarded.fd), + "--proxy-headers" if forwarded.proxy_headers else "--no-proxy-headers", + ) + ) if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv): - return (GATEWAY_APP, "--factory", *argv) - return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive) + return (GATEWAY_APP, "--factory", *argv, *inherited_socket) + return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive, *inherited_socket) def pool_database_url( @@ -65,17 +79,54 @@ def pool_database_url( return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth()) -def worker_app() -> "FastAPI": +def worker_app() -> "ASGIApp": with without_gc_pauses(): from gateway.main import app - return app + from litellm.proxy.rust_forward.mode import rust_forward_mode + from litellm.proxy.rust_forward.peer import wrap + return wrap(app, rust_forward_mode(os.environ)) def _serve(argv: Sequence[str]) -> None: uvicorn_main(tuple(argv), prog_name="uvicorn") -def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: +def public_listener(argv: Sequence[str], binary: str | None) -> "tuple[str, PublicListener]": + """Validate the uvicorn flags for Rust forwarding and return the binary and the public listener.""" + from pydantic import TypeAdapter + + from litellm.proxy.rust_forward.listener import PublicListener, StartupError, shutdown_drain_seconds, startup_check + from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager + + params: Final[Mapping[str, object]] = uvicorn_main.make_context( + "uvicorn", list(uvicorn_argv(argv, os.environ, None)) + ).params + checked: Final = startup_check( + binary, + { + "--ssl-keyfile": params["ssl_keyfile"], + "--ssl-certfile": params["ssl_certfile"], + "--ssl-ciphers": any(arg.split("=")[0] == "--ssl-ciphers" for arg in argv), + "--uds": params["uds"] is not None, + "--fd": params["fd"] is not None, + }, + ) + if isinstance(checked, StartupError): + sys.exit(checked.reason) + graceful_timeout: Final = TypeAdapter[int | None](int | None).validate_python(params["timeout_graceful_shutdown"]) + return checked, PublicListener( + host=TypeAdapter(str).validate_python(params["host"]), + port=TypeAdapter(int).validate_python(params["port"]), + keepalive_seconds=TypeAdapter(int).validate_python(params["timeout_keep_alive"]), + shutdown_drain_seconds=shutdown_drain_seconds( + GracefulShutdownManager.get_timeout() if graceful_timeout is None else graceful_timeout + ), + proxy_headers=TypeAdapter(bool).validate_python(params["proxy_headers"]), + trusted_proxies=TypeAdapter[str | None](str | None).validate_python(params["forwarded_allow_ips"]), + ) + + +def configure_database() -> None: from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url @@ -86,7 +137,26 @@ def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) - sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}") if pooled_url is not None: export_pooled_database_url(pooled_url) - serve(uvicorn_argv(argv, os.environ)) + + +def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: + from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides + from litellm.proxy.rust_forward.supervisor import gateway_binary, supervise + + mode: Final = rust_forward_mode(os.environ) + match mode: + case Disabled(): + configure_database() + serve(uvicorn_argv(argv, os.environ, None)) + case Supervisor(): + binary, listener = public_listener(argv, gateway_binary(sysconfig.get_path("scripts"))) + configure_database() + command: Final = (sys.executable, "-m", "gateway.launch", *argv) + sys.exit(supervise(binary, command, listener, os.environ)) + case Supervised(): + serve(uvicorn_argv(argv, os.environ, server_overrides(mode))) + case _: + assert_never(mode) if __name__ == "__main__": diff --git a/litellm/proxy/dev_reload.py b/litellm/proxy/dev_reload.py index 500f581b34d..7b057271cb6 100644 --- a/litellm/proxy/dev_reload.py +++ b/litellm/proxy/dev_reload.py @@ -13,7 +13,7 @@ from socket import socket from typing import Final, Literal from pydantic import ConfigDict, ImportString, JsonValue, TypeAdapter, with_config -from typing_extensions import ReadOnly, Required, TypedDict +from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from uvicorn import Config, Server from uvicorn import config as uvicorn_config from uvicorn.supervisors import ChangeReload, Multiprocess @@ -42,6 +42,7 @@ class ServerOptions(TypedDict, total=False): @with_config(ConfigDict(extra="allow")) class ReloadOptions(ServerOptions): app: Required[ReadOnly[str]] + factory: NotRequired[ReadOnly[bool]] reload: ReadOnly[bool] reload_dirs: ReadOnly[tuple[str, ...]] reload_includes: ReadOnly[tuple[str, ...]] @@ -92,7 +93,9 @@ def load_worker_app(import_path: str, worker_app_factory: bool) -> Callable[..., def pool_config(options: ReloadOptions, workers: int, worker_overrides: Mapping[str, object] | None = None) -> Config: overrides: Final = worker_overrides or {} worker_app: Final = TypeAdapter(str).validate_python(overrides.get("app", options["app"])) - worker_app_factory: Final = TypeAdapter(bool).validate_python(overrides.get("factory", False)) + worker_app_factory: Final = TypeAdapter(bool).validate_python( + overrides.get("factory", options.get("factory", False)) + ) return Config( partial(load_worker_app, worker_app, worker_app_factory), factory=True, diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 0fbfa0b06a7..cb8e936f3c5 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -8,6 +8,7 @@ import random import re import subprocess import sys +import sysconfig import urllib.parse as urlparse from collections.abc import Callable, Iterable, Mapping, Sequence from pathlib import Path @@ -35,6 +36,10 @@ from litellm.types.llms.base import LiteLLMBaseModel if TYPE_CHECKING: from fastapi import FastAPI + + from litellm.proxy.rust_forward.listener import PublicListener + from litellm.proxy.rust_forward.mode import ServerOverrides + from litellm.proxy.rust_forward.peer import ASGIApp else: FastAPI = Any @@ -261,6 +266,7 @@ def _apply_config_logging_settings(litellm_settings: StartupOutputSettings | Non WORKER_APP: Final = "litellm_cli.main:proxy_worker_app" +FORWARD_APP: Final = "litellm.proxy.rust_forward.asgi:app" WORKER_LOG_CONFIG_ENV: Final = "LITELLM_WORKER_LOG_CONFIG" _LOG_CONFIG: Final = TypeAdapter[str | Mapping[str, object]](str | Mapping[str, object]) _LOG_CONFIG_FILE: Final = TypeAdapter[Mapping[str, object]](Mapping[str, object]) @@ -389,12 +395,12 @@ class ProxyInitializationHelpers: logging.config.fileConfig(log_config, disable_existing_loggers=False) @staticmethod - def child_process_uvicorn_overrides(log_config: object) -> Mapping[str, object]: + def child_process_uvicorn_overrides(log_config: object, app: str = WORKER_APP) -> Mapping[str, object]: if log_config is None: - return MappingProxyType({"app": WORKER_APP, "factory": True}) + return MappingProxyType({"app": app, "factory": True}) ProxyInitializationHelpers.apply_log_config(_LOG_CONFIG.validate_python(log_config)) os.environ[WORKER_LOG_CONFIG_ENV] = json.dumps(log_config) - return MappingProxyType({"app": WORKER_APP, "factory": True, "log_config": None}) + return MappingProxyType({"app": app, "factory": True, "log_config": None}) @staticmethod def apply_worker_log_config(environ: Mapping[str, str]) -> None: @@ -409,6 +415,7 @@ class ProxyInitializationHelpers: log_config: str | None = None, keepalive_timeout: int | None = None, timeout_worker_healthcheck: int | None = None, + forwarded: "ServerOverrides | None" = None, ) -> dict: """ Get the arguments for `uvicorn` worker @@ -427,6 +434,16 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), + **( + {} + if forwarded is None + else { + "app": FORWARD_APP, + "factory": True, + "fd": forwarded.fd, + "proxy_headers": forwarded.proxy_headers, + } + ), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -674,12 +691,13 @@ class ProxyInitializationHelpers: def run_gunicorn_server( host: str, port: int, - app: FastAPI, + app: "ASGIApp", num_workers: int, ssl_certfile_path: str, ssl_keyfile_path: str, max_requests_before_restart: int | None = None, max_requests_before_restart_jitter: int | None = None, + forwarded: "ServerOverrides | None" = None, ): """ Run litellm with `gunicorn` @@ -752,9 +770,11 @@ class ProxyInitializationHelpers: ) graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout() gunicorn_options: Final = { - "bind": f"{host}:{port}", + "bind": f"{host}:{port}" if forwarded is None else f"fd://{forwarded.fd}", "workers": num_workers, # default is 1 - "worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout), + "worker_class": build_graceful_shutdown_uvicorn_worker( + graceful_shutdown_timeout, proxy_headers=True if forwarded is None else forwarded.proxy_headers + ), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, "preload": True, # Add the preload flag, "accesslog": "-", # Log to stdout @@ -815,7 +835,7 @@ class ProxyInitializationHelpers: _run_ollama_serve = run_ollama_serve @staticmethod - def is_port_in_use(port): + def is_port_in_use(port: int): import socket with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: @@ -823,6 +843,34 @@ class ProxyInitializationHelpers: _is_port_in_use = is_port_in_use + @staticmethod + def public_listener(host: str, port: int, keepalive_timeout: int | None, run_gunicorn: bool) -> "PublicListener": + """``run_gunicorn_server`` never hands ``--keepalive_timeout`` to gunicorn, so its workers keep gunicorn's.""" + from gunicorn.config import Config as GunicornConfig + + from litellm.proxy.rust_forward.listener import ( + PublicListener, + shutdown_drain_seconds, + uvicorn_default_keepalive, + ) + from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager + + keepalive: Final = ( + TypeAdapter(int).validate_python(GunicornConfig().keepalive) + if run_gunicorn + else (uvicorn_default_keepalive() if keepalive_timeout is None else keepalive_timeout) + ) + return PublicListener( + host=host, + port=random.randint(1024, 49152) + if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) + else port, + keepalive_seconds=keepalive, + shutdown_drain_seconds=shutdown_drain_seconds(GracefulShutdownManager.get_timeout()), + proxy_headers=True, + trusted_proxies=None, + ) + @staticmethod def get_loop_type(): """Helper function to determine the event loop type based on platform""" @@ -1210,8 +1258,8 @@ class ProxyInitializationHelpers: ) def run_server( cli_args, - host, - port, + host: str, + port: int, api_base, api_version, model, @@ -1280,6 +1328,31 @@ def run_server( run_setup_wizard() return + from litellm.proxy.rust_forward.mode import Supervised, Supervisor, rust_forward_mode, server_overrides + + forward_mode: Final = rust_forward_mode(os.environ) + if isinstance(forward_mode, Supervisor) and not any( + (version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data) + ): + from litellm.proxy.rust_forward.listener import StartupError, startup_check + from litellm.proxy.rust_forward.supervisor import cli_command, gateway_binary, supervise + + binary: Final = startup_check( + gateway_binary(sysconfig.get_path("scripts")), + { + "--ssl_keyfile_path": ssl_keyfile_path, + "--ssl_certfile_path": ssl_certfile_path, + "--ciphers": ciphers, + "--run_hypercorn": run_hypercorn, + "--run_granian": run_granian, + }, + ) + if isinstance(binary, StartupError): + raise click.ClickException(binary.reason) + listener: Final = ProxyInitializationHelpers.public_listener(host, port, keepalive_timeout, run_gunicorn) + raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), listener, os.environ)) + forwarded: Final = server_overrides(forward_mode) if isinstance(forward_mode, Supervised) else None + args: Final = locals() try: from litellm.proxy.proxy_server import ( @@ -1729,14 +1802,15 @@ def run_server( if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None: from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process + metrics_bind_host: Final = host if forwarded is None else forwarded.metrics_host try: metrics_process: Final = start_metrics_server_process( - host=host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir + host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir ) except MetricsServerStartupError as error: raise click.ClickException(str(error)) from error print_startup_message( - f"\033[1;32mLiteLLM: Serving Prometheus metrics on {host}:{prometheus_metrics_port}/metrics " + f"\033[1;32mLiteLLM: Serving Prometheus metrics on {metrics_bind_host}:{prometheus_metrics_port}/metrics " f"(pid {metrics_process.pid})\033[0m" ) @@ -1747,6 +1821,7 @@ def run_server( log_config=log_config, keepalive_timeout=keepalive_timeout, timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None), + forwarded=forwarded, ) # Optional: recycle uvicorn workers after N requests if max_requests_before_restart is not None: @@ -1786,7 +1861,7 @@ def run_server( start_query_engine_reaper() if num_workers > 1 or reload: worker_overrides: Final = ProxyInitializationHelpers.child_process_uvicorn_overrides( - uvicorn_args.get("log_config") + uvicorn_args.get("log_config"), WORKER_APP if forwarded is None else FORWARD_APP ) if reload and num_workers > 1: from litellm.proxy.dev_reload import ( @@ -1812,15 +1887,18 @@ def run_server( workers=num_workers, ) elif run_gunicorn is True: + from litellm.proxy.rust_forward.peer import wrap + ProxyInitializationHelpers.run_gunicorn_server( host=host, port=port, - app=app, + app=wrap(app, forward_mode), num_workers=num_workers, ssl_certfile_path=ssl_certfile_path, ssl_keyfile_path=ssl_keyfile_path, max_requests_before_restart=max_requests_before_restart, max_requests_before_restart_jitter=max_requests_before_restart_jitter, + forwarded=forwarded, ) elif run_hypercorn is True: ProxyInitializationHelpers.init_hypercorn_server( diff --git a/litellm/proxy/rust_forward/AGENTS.md b/litellm/proxy/rust_forward/AGENTS.md new file mode 100644 index 00000000000..a2ab93cde5c --- /dev/null +++ b/litellm/proxy/rust_forward/AGENTS.md @@ -0,0 +1,8 @@ +# Rust forwarding, Python half + +With `LITELLM_RUST` on, the supervisor here runs the Rust gateway on the public port and Python on a private loopback listener. This package is the Python half of forwarding and pairs with the crate `litellm-rust/crates/gateway-forward`. Once Rust serves every route, both get deleted together, along with one dispatch branch in each entry point + +- Nothing outside this package imports Rust forwarding concepts, except the dispatch, the flag adapter and the app wrap in `litellm/proxy/proxy_cli.py` and `gateway/launch.py`. The proxy app itself never knows Rust is in front of it +- Python must see each request as if the client had connected directly. `x-litellm-rust-peer` is the only Rust-specific input, and `peer.py` honors it only from the loopback hop +- Python owns every setting Rust needs. The supervisor resolves the host, port, keepalive and shutdown drain Python actually runs with and hands them to Rust as the `LITELLM_FORWARD_*` variables in `rust_spec`. Rust never reads Python's own settings, so a new value Rust needs gets a new variable there +- Entry points turn their own flags into a `PublicListener`, apply `server_overrides` in the supervised child, and wrap the app with `peer.wrap` where they load it diff --git a/litellm/proxy/rust_forward/__init__.py b/litellm/proxy/rust_forward/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/proxy/rust_forward/asgi.py b/litellm/proxy/rust_forward/asgi.py new file mode 100644 index 00000000000..ff9bdc57cf8 --- /dev/null +++ b/litellm/proxy/rust_forward/asgi.py @@ -0,0 +1,10 @@ +import os + +from litellm.proxy.rust_forward.mode import rust_forward_mode +from litellm.proxy.rust_forward.peer import ASGIApp, wrap + + +def app() -> ASGIApp: + from litellm_cli.main import proxy_worker_app + + return wrap(proxy_worker_app(), rust_forward_mode(os.environ)) diff --git a/litellm/proxy/rust_forward/listener.py b/litellm/proxy/rust_forward/listener.py new file mode 100644 index 00000000000..28595a4099e --- /dev/null +++ b/litellm/proxy/rust_forward/listener.py @@ -0,0 +1,46 @@ +import os +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +from litellm.rust_bridge.configuration import ENV_NAME + + +@dataclass(frozen=True, slots=True) +class PublicListener: + host: str + port: int + keepalive_seconds: int + shutdown_drain_seconds: float + proxy_headers: bool + trusted_proxies: str | None + + +@dataclass(frozen=True, slots=True) +class StartupError: + reason: str + + +def startup_check(binary: str | None, unsupported: Mapping[str, object]) -> str | StartupError: + flag: Final = next((name for name, value in unsupported.items() if value), None) + if flag is not None: + return StartupError(f"{ENV_NAME} does not support {flag}") + if os.name != "posix": + return StartupError(f"{ENV_NAME} requires POSIX inherited sockets and process signals") + if binary is None: + return StartupError( + f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment" + ) + return binary + + +def uvicorn_default_keepalive() -> int: + import uvicorn + + return uvicorn.Config(app="", log_config=None).timeout_keep_alive + + +def shutdown_drain_seconds(graceful_timeout: float) -> float: + from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS + + return graceful_timeout + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS diff --git a/litellm/proxy/rust_forward/mode.py b/litellm/proxy/rust_forward/mode.py new file mode 100644 index 00000000000..a75287688ea --- /dev/null +++ b/litellm/proxy/rust_forward/mode.py @@ -0,0 +1,68 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final, TypeAlias + +from pydantic import TypeAdapter + +from litellm.rust_bridge.configuration import environment_switch + +GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" +METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" +PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" +TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" +DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1" +RUST_HOP: Final = "127.0.0.1" + +_FD: Final = TypeAdapter(int) +_ENABLED: Final = TypeAdapter(bool) + + +@dataclass(frozen=True, slots=True) +class Disabled: + pass + + +@dataclass(frozen=True, slots=True) +class Supervisor: + pass + + +@dataclass(frozen=True, slots=True) +class Supervised: + listener_fd: int + metrics_host: str + proxy_headers: bool + trusted_proxies: str + + +RustForwardMode: TypeAlias = Disabled | Supervisor | Supervised + + +def rust_forward_mode(environ: Mapping[str, str]) -> RustForwardMode: + fd: Final = environ.get(GATEWAY_FD) + if fd is not None: + return Supervised( + _FD.validate_python(fd), + environ[METRICS_HOST], + _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), + environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), + ) + if environment_switch(environ): + return Supervisor() + return Disabled() + + +@dataclass(frozen=True, slots=True) +class ServerOverrides: + fd: int + proxy_headers: bool + metrics_host: str + + +def server_overrides(mode: Supervised) -> ServerOverrides: + """The private listener the supervised server binds instead of its own flags. + + Proxy headers stay off at the server, since it would judge Rust's loopback address instead of the real peer. + ``peer.wrap`` applies the operator's trust settings to the restored peer. + """ + return ServerOverrides(fd=mode.listener_fd, proxy_headers=False, metrics_host=mode.metrics_host) diff --git a/litellm/proxy/rust_forward/peer.py b/litellm/proxy/rust_forward/peer.py new file mode 100644 index 00000000000..2d03d6ca35c --- /dev/null +++ b/litellm/proxy/rust_forward/peer.py @@ -0,0 +1,57 @@ +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Final, TypeAlias + +from typing_extensions import assert_never + +from litellm.proxy.rust_forward.mode import RUST_HOP, Disabled, RustForwardMode, Supervised, Supervisor + +if TYPE_CHECKING: + from uvicorn._types import ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope + +PEER_HEADER: Final = b"x-litellm-rust-peer" + +ASGIApp: TypeAlias = Callable[..., Awaitable[None]] + + +def _restore_peer(scope: "WWWScope") -> None: + headers: Final = tuple(scope["headers"]) + peer: Final = next((value for name, value in headers if name == PEER_HEADER), None) + if peer is None: + return + remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER) + scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop + client: Final = scope["client"] + if client is None or client[0] != RUST_HOP: + return + host, _, port = peer.decode("latin-1").rpartition(":") + scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope + + +class RustPeerMiddleware: + """Make Python see each request as if the client had connected to it directly. + + Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real + TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the + real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust + settings Python would have used on the public port. + """ + + def __init__(self, app: ASGIApp, mode: Supervised) -> None: + from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + + self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app + + async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + if scope["type"] != "lifespan": + _restore_peer(scope) + await self.app(scope, receive, send) + + +def wrap(app: ASGIApp, mode: RustForwardMode) -> ASGIApp: + match mode: + case Supervised(): + return RustPeerMiddleware(app, mode) + case Disabled() | Supervisor(): + return app + case _: + assert_never(mode) diff --git a/litellm/proxy/rust_forward/supervisor.py b/litellm/proxy/rust_forward/supervisor.py new file mode 100644 index 00000000000..6ab4b86dca3 --- /dev/null +++ b/litellm/proxy/rust_forward/supervisor.py @@ -0,0 +1,219 @@ +import asyncio +import contextlib +import os +import shutil +import signal +import socket +import sys +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from enum import Enum, IntEnum +from types import MappingProxyType +from typing import Final, TypeAlias + +from pydantic import TypeAdapter +from typing_extensions import assert_never + +from litellm.proxy.rust_forward.listener import PublicListener +from litellm.proxy.rust_forward.mode import GATEWAY_FD, METRICS_HOST, PROXY_HEADERS, RUST_HOP, TRUSTED_PROXIES + +_ADDRESS: Final = TypeAdapter(tuple[str, int]) + + +def gateway_binary(scripts_dir: str) -> str | None: + name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway" + return shutil.which(name, path=scripts_dir) + + +class Child(Enum): + RUST = "rust" + PYTHON = "python" + + +DRAIN_ORDER: Final = (Child.RUST, Child.PYTHON) + + +class ExitCode(IntEnum): + CLEAN = 0 + CHILD_FAILED = 1 + + +@dataclass(frozen=True, slots=True) +class Stop: + signum: signal.Signals + + +@dataclass(frozen=True, slots=True) +class Exited: + child: Child + code: int + + +@dataclass(frozen=True, slots=True) +class Running: + pass + + +@dataclass(frozen=True, slots=True) +class Draining: + signum: signal.Signals + signalled: Child + remaining: tuple[Child, ...] + + +@dataclass(frozen=True, slots=True) +class Finished: + code: ExitCode + + +@dataclass(frozen=True, slots=True) +class Signal: + child: Child + signum: signal.Signals + + +Event: TypeAlias = Stop | Exited +State: TypeAlias = Running | Draining | Finished + + +def _signal_next(signum: signal.Signals, order: tuple[Child, ...]) -> tuple[State, Signal | None]: + if not order: + return Finished(ExitCode.CLEAN), None + return Draining(signum, order[0], order[1:]), Signal(order[0], signum) + + +def step(state: State, event: Event) -> tuple[State, Signal | None]: + match state: + case Running(): + match event: + case Stop(signum): + return _signal_next(signum, DRAIN_ORDER) + case Exited(): + return Finished(ExitCode.CHILD_FAILED), None + case _: + assert_never(event) + case Draining(signum, signalled, remaining): + match event: + case Stop(): + return state, None + case Exited(child, code) if child is signalled and code == 0: + return _signal_next(signum, remaining) + case Exited(): + return Finished(ExitCode.CHILD_FAILED), None + case _: + assert_never(event) + case Finished(): + return state, None + case _: + assert_never(state) + + +@dataclass(frozen=True, slots=True) +class ChildSpec: + argv: tuple[str, ...] + env: Mapping[str, str] + pass_fds: tuple[int, ...] + + +def python_spec( + command: Sequence[str], listener_fd: int, upstream_port: int, listener: PublicListener, environ: Mapping[str, str] +) -> ChildSpec: + return ChildSpec( + argv=(*command, "--host", RUST_HOP, "--port", str(upstream_port)), + env=MappingProxyType( + { + **environ, + GATEWAY_FD: str(listener_fd), + METRICS_HOST: listener.host, + PROXY_HEADERS: str(listener.proxy_headers).lower(), + **({} if listener.trusted_proxies is None else {TRUSTED_PROXIES: listener.trusted_proxies}), + } + ), + pass_fds=(listener_fd,), + ) + + +def rust_spec(binary: str, upstream_port: int, listener: PublicListener, environ: Mapping[str, str]) -> ChildSpec: + return ChildSpec( + argv=(binary,), + env=MappingProxyType( + { + **environ, + "LITELLM_FORWARD_UPSTREAM": f"http://{RUST_HOP}:{upstream_port}", + "LITELLM_FORWARD_HOST": listener.host, + "LITELLM_FORWARD_PORT": str(listener.port), + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE": str(listener.keepalive_seconds), + "LITELLM_FORWARD_SHUTDOWN_DRAIN": str(listener.shutdown_drain_seconds), + } + ), + pass_fds=(), + ) + + +async def _spawn(spec: ChildSpec) -> asyncio.subprocess.Process: + return await asyncio.create_subprocess_exec( + *spec.argv, env=dict(spec.env), pass_fds=spec.pass_fds, start_new_session=True + ) + + +def _terminate_group(process: asyncio.subprocess.Process) -> None: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGTERM) + + +async def _report_exit(child: Child, process: asyncio.subprocess.Process, events: asyncio.Queue[Event]) -> None: + events.put_nowait(Exited(child, await process.wait())) + + +async def _drive( + state: State, events: asyncio.Queue[Event], children: Mapping[Child, asyncio.subprocess.Process] +) -> ExitCode: + if isinstance(state, Finished): + return state.code + following, action = step(state, await events.get()) + if action is not None: + with contextlib.suppress(ProcessLookupError): + children[action.child].send_signal(action.signum) + return await _drive(following, events, children) + + +async def _supervise( + binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str] +) -> ExitCode: + loop: Final = asyncio.get_running_loop() + events: Final[asyncio.Queue[Event]] = asyncio.Queue() + for signum in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(signum, events.put_nowait, Stop(signum)) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as private: + private.bind((RUST_HOP, 0)) + private.listen() + _, upstream_port = _ADDRESS.validate_python(private.getsockname()) + python: Final = await _spawn(python_spec(command, private.fileno(), upstream_port, listener, environ)) + try: + rust: Final = await _spawn(rust_spec(binary, upstream_port, listener, environ)) + except OSError: + _terminate_group(python) + await python.wait() + raise + children: Final = MappingProxyType({Child.RUST: rust, Child.PYTHON: python}) + waiters: Final = tuple( + asyncio.create_task(_report_exit(child, process, events)) for child, process in children.items() + ) + try: + return await _drive(Running(), events, children) + finally: + for process in children.values(): + _terminate_group(process) + await asyncio.gather(*waiters) + + +def supervise(binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str]) -> int: + try: + return asyncio.run(_supervise(binary, command, listener, environ)) + except OSError as error: + sys.stderr.write(f"Rust gateway startup failed: {error}\n") + return ExitCode.CHILD_FAILED + + +def cli_command(argv: Sequence[str]) -> tuple[str, ...]: + return (sys.executable, "-m", "litellm.proxy.proxy_cli", *argv) diff --git a/litellm/proxy/shutdown/graceful_uvicorn_worker.py b/litellm/proxy/shutdown/graceful_uvicorn_worker.py index 6ae5ae60c3f..8bbd64190f8 100644 --- a/litellm/proxy/shutdown/graceful_uvicorn_worker.py +++ b/litellm/proxy/shutdown/graceful_uvicorn_worker.py @@ -1,10 +1,16 @@ from uvicorn.workers import UvicornWorker -def build_graceful_shutdown_uvicorn_worker(timeout_graceful_shutdown: float) -> type[UvicornWorker]: +def build_graceful_shutdown_uvicorn_worker( + timeout_graceful_shutdown: float, *, proxy_headers: bool = True +) -> type[UvicornWorker]: class GracefulShutdownUvicornWorker(UvicornWorker): def __init__(self, *args: object) -> None: - self.CONFIG_KWARGS = {**UvicornWorker.CONFIG_KWARGS, "timeout_graceful_shutdown": timeout_graceful_shutdown} + self.CONFIG_KWARGS = { + **UvicornWorker.CONFIG_KWARGS, + "timeout_graceful_shutdown": timeout_graceful_shutdown, + "proxy_headers": proxy_headers, + } super().__init__(*args) return GracefulShutdownUvicornWorker diff --git a/litellm/rust_bridge/configuration.py b/litellm/rust_bridge/configuration.py index 152ba632996..e36527e9f4e 100644 --- a/litellm/rust_bridge/configuration.py +++ b/litellm/rust_bridge/configuration.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +from collections.abc import Mapping from enum import Enum, auto from functools import lru_cache from typing import Final @@ -8,7 +9,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never -_GLOBAL_ENV_NAME: Final = "LITELLM_RUST" +ENV_NAME: Final = "LITELLM_RUST" _ENV_BOOL: Final = TypeAdapter(bool) @@ -44,6 +45,10 @@ def _parse_env_bool(value: str | None) -> bool | None: return None +def environment_switch(environ: Mapping[str, str]) -> bool | None: + return _parse_env_bool(environ.get(ENV_NAME)) + + def decide( rollout: Rollout, *, @@ -72,7 +77,7 @@ def decision(rollout: Rollout) -> Decision: return decide( rollout, process_override=_CONFIGURATION.override, - environment_override=_parse_env_bool(os.getenv(_GLOBAL_ENV_NAME)), + environment_override=environment_switch(os.environ), ) diff --git a/tests/integration/configuration/test_rust_forward.py b/tests/integration/configuration/test_rust_forward.py new file mode 100644 index 00000000000..b75da286bf7 --- /dev/null +++ b/tests/integration/configuration/test_rust_forward.py @@ -0,0 +1,180 @@ +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Final + +import httpx +import pytest + +from tests.integration._support.client import Gateway, eventually +from tests.integration._support.ports import allocate_port +from tests.integration._support.process import stop_root_process + +TRUSTED_BALANCER: Final = "127.0.0.3" + + +def ready(client: httpx.Client) -> bool: + try: + return client.get("/health/liveliness").status_code == 200 + except httpx.TransportError: + return False + + +def client_ip(base_url: str, local_address: str, key: str, forwarded_for: str) -> httpx.Response: + with httpx.Client( + base_url=base_url, + transport=httpx.HTTPTransport(local_address=local_address), + trust_env=False, + timeout=5, + ) as client: + return client.get( + "/v1/mcp/network/client-ip", + headers={"Authorization": f"Bearer {key}", "X-Forwarded-For": forwarded_for}, + ) + + +@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) +def test_gateway_preserves_metrics_bind_and_mcp_client_ip( + gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...] +) -> None: + config: Final = tmp_path / "config.yaml" + config.write_text( + json.dumps( + { + "model_list": [], + "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY", "use_x_forwarded_for": False}, + } + ) + ) + metrics_port: Final = allocate_port() + public_port: Final = allocate_port() + environment: Final = { + **{key: value for key, value in os.environ.items() if key not in ("DATABASE_URL", "DATABASE_URL_READ_REPLICA")}, + "LITELLM_MASTER_KEY": gateway.key, + "STORE_MODEL_IN_DB": "False", + "PYTHON_DOTENV_DISABLED": "1", + "LITELLM_RUST": "true", + "FORWARDED_ALLOW_IPS": TRUSTED_BALANCER, + } + with (tmp_path / "proxy.log").open("wb") as output: + with subprocess.Popen( + ( + sys.executable, + "-I", + "-m", + "litellm.proxy.proxy_cli", + "--config", + str(config), + "--host", + "0.0.0.0", + "--port", + str(public_port), + "--num_workers", + "2", + "--timeout_worker_healthcheck", + "120", + "--prometheus_metrics_port", + str(metrics_port), + *server_arguments, + ), + env=environment, + stdout=output, + stderr=output, + ) as supervisor: + try: + with httpx.Client( + base_url=f"http://127.0.0.1:{public_port}", + transport=httpx.HTTPTransport(local_address="127.0.0.2"), + trust_env=False, + follow_redirects=True, + timeout=5, + ) as external: + assert eventually(lambda: ready(external), bool, seconds=180), (tmp_path / "proxy.log").read_text() + response: Final = external.get(f"http://127.0.0.2:{metrics_port}/metrics") + assert response.status_code == 200, response.text + public: Final = f"http://127.0.0.1:{public_port}" + spoofed: Final = client_ip(public, "127.0.0.2", gateway.key, "1.2.3.4") + assert spoofed.status_code == 200, spoofed.text + assert spoofed.json() == {"ip": "127.0.0.2"} + behind_balancer: Final = client_ip(public, TRUSTED_BALANCER, gateway.key, "203.0.113.9") + assert behind_balancer.status_code == 200, behind_balancer.text + assert behind_balancer.json() == {"ip": "203.0.113.9"} + borrowed: Final = client_ip(public, "127.0.0.2", gateway.key, f"203.0.113.9, {TRUSTED_BALANCER}") + assert borrowed.status_code == 200, borrowed.text + assert borrowed.json() == {"ip": "127.0.0.2"} + finally: + assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text() + assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text() + + +@pytest.mark.parametrize( + "server_arguments", + [("--no-proxy-headers",), ("--forwarded-allow-ips", "127.0.0.1")], +) +def test_gateway_launcher_private_hop_preserves_client_address( + tmp_path: Path, server_arguments: tuple[str, ...] +) -> None: + config: Final = tmp_path / "config.yaml" + config.write_text( + json.dumps({"model_list": [], "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY"}}) + ) + public_port: Final = allocate_port() + environment: Final = { + **{key: value for key, value in os.environ.items() if not key.startswith("DATABASE_") and key != "DIRECT_URL"}, + "LITELLM_MASTER_KEY": "sk-rust-forwarding-integration-key-not-a-secret", + "LITELLM_RUST": "true", + "LITELLM_PGBOUNCER_ENABLED": "false", + "CONFIG_FILE_PATH": str(config), + "STORE_MODEL_IN_DB": "False", + "PYTHON_DOTENV_DISABLED": "1", + } + log: Final = tmp_path / "gateway.log" + with log.open("wb") as output: + with subprocess.Popen( + ( + sys.executable, + "-I", + "-m", + "gateway.launch", + "--host", + "0.0.0.0", + "--port", + str(public_port), + "--workers", + "2", + "--timeout-worker-healthcheck", + "120", + *server_arguments, + ), + env=environment, + stdout=output, + stderr=output, + ) as supervisor: + try: + with httpx.Client( + base_url=f"http://127.0.0.1:{public_port}", + transport=httpx.HTTPTransport(local_address="127.0.0.2"), + trust_env=False, + timeout=5, + ) as external: + assert eventually(lambda: ready(external), bool, seconds=180), log.read_text() + response: Final = external.get( + "/health/liveliness?client-ip-proof", headers={"X-Forwarded-For": "198.51.100.10"} + ) + assert response.status_code == 200, response.text + events: Final = eventually( + lambda: tuple( + line + for line in log.read_text().splitlines() + if "GET /health/liveliness?client-ip-proof " in line + ), + bool, + seconds=10, + ) + assert len(events) == 1, log.read_text() + assert "127.0.0.2:" in events[0], events + finally: + assert stop_root_process(supervisor), log.read_text() + assert supervisor.returncode == 0, log.read_text() diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index d682fcbad2d..36eaabfb029 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -12,13 +12,18 @@ from typing import Final, cast from unittest.mock import MagicMock, patch import pytest +import uvicorn from uvicorn.importer import import_from_string from uvicorn.main import main as uvicorn_main -from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv +from gateway.launch import GATEWAY_APP, main, pool_database_url, public_listener, uvicorn_argv +from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings from litellm.proxy.proxy_server import app as proxy_app +from litellm.proxy.rust_forward.mode import Supervised, server_overrides +from litellm.proxy.rust_forward.supervisor import rust_spec +from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager DB_ENV: Final = { "DATABASE_HOST": "db.internal", @@ -114,17 +119,89 @@ def _uvicorn_params(argv: tuple[str, ...]) -> dict[str, object]: class TestUvicornArgv: + @pytest.mark.parametrize( + "arguments", [("--uds", "/tmp/gateway.sock"), ("--fd", "0"), ("--fd=17",), ("--ssl-keyfile", "unused")] + ) + def test_forwarding_rejects_unsupported_listeners_before_starting_processes( + self, arguments: tuple[str, ...] + ) -> None: + with pytest.raises(SystemExit, match=f"does not support {arguments[0].split('=')[0]}"): + public_listener(arguments, "gateway") + + @pytest.mark.parametrize( + ("argv", "trust"), + [ + ((), (True, None)), + (("--forwarded-allow-ips", "10.0.0.5"), (True, "10.0.0.5")), + (("--forwarded-allow-ips=*",), (True, "*")), + (("--no-proxy-headers",), (False, None)), + ], + ) + def test_forwarding_carries_operator_proxy_header_options_to_the_child( + self, argv: tuple[str, ...], trust: tuple[bool, str | None], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("FORWARDED_ALLOW_IPS", raising=False) + binary, listener = public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") + assert (binary, listener.host, listener.port) == ("gateway", "0.0.0.0", 4000) + assert (listener.proxy_headers, listener.trusted_proxies) == trust + + @pytest.mark.parametrize( + ("argv", "environ"), + [(("--timeout-keep-alive", "2"), {}), ((), {"KEEPALIVE_TIMEOUT": "7"}), ((), {})], + ) + def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with( + self, argv: tuple[str, ...], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + for name, value in environ.items(): + monkeypatch.setenv(name, value) + _, listener = public_listener(argv, "gateway") + sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"] + assert sent == str(_uvicorn_params(uvicorn_argv(argv, os.environ, None))["timeout_keep_alive"]) + + def test_without_a_keepalive_rust_follows_uvicorns_own_default(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + _, listener = public_listener((), "gateway") + assert listener.keepalive_seconds == uvicorn.Config(app="", log_config=None).timeout_keep_alive + + @pytest.mark.parametrize("argv", [(), ("--timeout-graceful-shutdown", "12")]) + def test_rust_drains_for_pythons_graceful_timeout_plus_teardown( + self, argv: tuple[str, ...], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "7.5") + _, listener = public_listener(argv, "gateway") + flag: Final = _uvicorn_params(uvicorn_argv(argv, {}, None))["timeout_graceful_shutdown"] + graceful: Final = GracefulShutdownManager.get_timeout() if flag is None else float(str(flag)) + sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"] + assert float(sent) == graceful + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS + + def test_supervised_listener_uses_the_inherited_loopback_socket(self): + supervised: Final = server_overrides(Supervised(17, "0.0.0.0", True, "127.0.0.1")) + params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised)) + assert params["fd"] == 17 + assert params["workers"] == 2 + + @pytest.mark.parametrize("argv", [(), ("--proxy-headers",), ("--forwarded-allow-ips=*",)]) + def test_private_hop_leaves_proxy_headers_to_the_app(self, argv: tuple[str, ...]): + params: Final = _uvicorn_params(uvicorn_argv(argv, {}, server_overrides(Supervised(17, "0.0.0.0", True, "*")))) + assert params["proxy_headers"] is False + + def test_direct_listener_keeps_explicit_proxy_header_options(self): + params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {}, None)) + assert params["proxy_headers"] is False + assert params["forwarded_allow_ips"] == "*" + def test_keepalive_env_reaches_uvicorn(self): - params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"})) + params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"}, None)) assert params["app"] == GATEWAY_APP assert params["workers"] == 4 assert params["timeout_keep_alive"] == 75 def test_unset_env_keeps_the_uvicorn_default(self): - assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}))["timeout_keep_alive"] == 5 + assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}, None))["timeout_keep_alive"] == 5 def test_an_explicit_flag_wins_over_the_env(self): - argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}) + argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}, None) assert _uvicorn_params(argv)["timeout_keep_alive"] == 30 def test_the_app_uvicorn_is_told_to_serve_is_the_trimmed_gateway(self, monkeypatch: pytest.MonkeyPatch) -> None: @@ -141,7 +218,7 @@ class TestUvicornArgv: ): monkeypatch.delenv(key, raising=False) - params: Final = _uvicorn_params(uvicorn_argv((), {})) + params: Final = _uvicorn_params(uvicorn_argv((), {}, None)) assert params["factory"] is True try: served: Final = import_from_string(cast(str, params["app"]))() @@ -229,6 +306,16 @@ class TestMain: DatabaseURLSettings.from_env().apply_to_env() assert os.environ["DATABASE_URL"] == pooled + def test_forwarding_disabled_keeps_uvicorn_socket_options( + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LITELLM_RUST", "false") + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False) + served: Final[list[tuple[str, ...]]] = [] + main(("--uds", "/tmp/gateway.sock"), serve=lambda argv: served.append(tuple(argv))) + assert _uvicorn_params(served[0])["uds"] == "/tmp/gateway.sock" + def test_a_pooler_that_cannot_start_stops_the_gateway_before_uvicorn( self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path ): diff --git a/tests/unit/proxy/rust_forward/__init__.py b/tests/unit/proxy/rust_forward/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/rust_forward/test_listener.py b/tests/unit/proxy/rust_forward/test_listener.py new file mode 100644 index 00000000000..d17b0e74e14 --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_listener.py @@ -0,0 +1,74 @@ +from collections.abc import Mapping +from typing import Final + +import pytest +import uvicorn +from click.testing import CliRunner +from gunicorn.config import Config as GunicornConfig + +from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS +from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server +from litellm.proxy.rust_forward.listener import PublicListener, StartupError, startup_check +from litellm.proxy.rust_forward.supervisor import rust_spec + + +@pytest.mark.parametrize( + "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] +) +def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "true") + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] + result: Final = CliRunner().invoke(run_server, [flag, *values]) + assert result.exit_code != 0 + assert f"LITELLM_RUST does not support {flag}" in result.output + + +def test_missing_binary_gives_install_instruction() -> None: + assert startup_check(None, {}) == StartupError( + "LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment" + ) + + +def _cli_listener(argv: list[str]) -> tuple[Mapping[str, object], PublicListener]: + params: Final = run_server.make_context("litellm", ["--port", "45999", *argv]).params + return params, ProxyInitializationHelpers.public_listener( + str(params["host"]), int(str(params["port"])), params["keepalive_timeout"], params["run_gunicorn"] is True + ) + + +def _uvicorn_keepalive(options: Mapping[str, object]) -> int: + configured: Final = {"timeout_keep_alive": options["timeout_keep_alive"]} if "timeout_keep_alive" in options else {} + return uvicorn.Config(app="", log_config=None, **configured).timeout_keep_alive + + +@pytest.mark.parametrize( + ("argv", "environ"), + [(["--keepalive_timeout", "2"], {}), ([], {"KEEPALIVE_TIMEOUT": "7"}), ([], {})], +) +def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with( + argv: list[str], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + for name, value in environ.items(): + monkeypatch.setenv(name, value) + params, listener = _cli_listener(argv) + uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 45555, keepalive_timeout=params["keepalive_timeout"] + ) + sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"] + assert sent == str(_uvicorn_keepalive(uvicorn_options)) + + +def test_gunicorn_workers_keep_gunicorns_keepalive_and_rust_follows(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + _, listener = _cli_listener(["--run_gunicorn", "--keepalive_timeout", "9"]) + assert listener.keepalive_seconds == GunicornConfig().keepalive + + +def test_rust_drains_for_pythons_graceful_timeout_plus_teardown(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "12.5") + _, listener = _cli_listener([]) + uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555) + sent: Final = float(rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"]) + assert sent == uvicorn_options["timeout_graceful_shutdown"] + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS diff --git a/tests/unit/proxy/rust_forward/test_mode.py b/tests/unit/proxy/rust_forward/test_mode.py new file mode 100644 index 00000000000..20005a21255 --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_mode.py @@ -0,0 +1,43 @@ +from typing import Final + +import pytest + +from litellm.proxy.proxy_cli import ProxyInitializationHelpers +from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides + + +@pytest.mark.parametrize( + ("environ", "expected"), + [ + ({}, Disabled()), + ({"LITELLM_RUST": "false"}, Disabled()), + ({"LITELLM_RUST": "off"}, Disabled()), + ({"LITELLM_RUST": "0"}, Disabled()), + ({"LITELLM_RUST": "true"}, Supervisor()), + ({"LITELLM_RUST": "1"}, Supervisor()), + ({"LITELLM_RUST": "yes"}, Supervisor()), + ({"LITELLM_RUST": "on"}, Supervisor()), + ({"LITELLM_RUST": " 1 "}, Supervisor()), + ({"LITELLM_RUST": "enabled"}, Disabled()), + ( + { + "LITELLM_RUST": "true", + "_LITELLM_GATEWAY_FD": "17", + "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", + "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", + }, + Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"), + ), + ], +) +def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None: + assert rust_forward_mode(environ) == expected + + +@pytest.mark.parametrize("mode", [None, Supervised(17, "0.0.0.0", True, "127.0.0.1")]) +def test_inherited_socket_reaches_uvicorn(mode: Supervised | None) -> None: + options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 45555, forwarded=None if mode is None else server_overrides(mode) + ) + assert options.get("fd") == (None if mode is None else mode.listener_fd) + assert options.get("proxy_headers", True) is (mode is None) diff --git a/tests/unit/proxy/rust_forward/test_peer.py b/tests/unit/proxy/rust_forward/test_peer.py new file mode 100644 index 00000000000..e96d9cf2b6a --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_peer.py @@ -0,0 +1,144 @@ +import asyncio +import gc +from typing import TYPE_CHECKING, Final + +import httpx +import pytest +from fastapi import Request +from fastapi.routing import APIRoute +from uvicorn.importer import import_from_string +from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + +if TYPE_CHECKING: + from uvicorn._types import ( + ASGIReceiveCallable, + ASGISendCallable, + ASGISendEvent, + HTTPDisconnectEvent, + HTTPScope, + Scope, + ) + +from litellm.proxy.proxy_server import app as proxy_app +from litellm.proxy.rust_forward.mode import Supervised +from litellm.proxy.rust_forward.peer import wrap + + +Headers = tuple[tuple[bytes, bytes], ...] + + +async def _receive() -> "HTTPDisconnectEvent": + return {"type": "http.disconnect"} + + +async def _send(message: "ASGISendEvent") -> None: + return None + + +async def _seen_by_app( + behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised +) -> tuple[tuple[str, int] | None, str, Headers]: + seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = ( + asyncio.get_running_loop().create_future() + ) + + async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + assert scope["type"] == "http" + seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"]))) + + direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe + app: Final = wrap(probe, mode) if behind_rust else direct + scope: Final[HTTPScope] = { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": "/", + "raw_path": b"/", + "query_string": b"", + "root_path": "", + "headers": headers, + "client": client, + "server": ("127.0.0.1", 4000), + } + await app(scope, _receive, _send) + return seen.result() + + +PEER: Final = ("203.0.113.7", 51234) +TRUSTED_PEER: Final = ("10.0.12.34", 40000) +FORWARDED: Final[Headers] = ( + (b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"), + (b"x-forwarded-proto", b"https"), +) + + +@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)]) +@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))]) +@pytest.mark.parametrize( + "mode", + [ + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + Supervised(17, "0.0.0.0", True, "10.0.0.0/8"), + Supervised(17, "0.0.0.0", True, "*"), + Supervised(17, "0.0.0.0", False, "10.0.0.0/8"), + ], +) +async def test_python_behind_rust_sees_what_a_direct_connection_would( + peer: tuple[str, int], headers: Headers, mode: Supervised +) -> None: + rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}" + behind_rust: Final = await _seen_by_app( + True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode + ) + direct: Final = await _seen_by_app(False, peer, headers, mode) + assert behind_rust == direct + + +async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None: + client, scheme, _ = await _seen_by_app( + True, + ("127.0.0.1", 60000), + ((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")), + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + ) + assert (client, scheme) == (("203.0.113.7", 51234), "http") + + +async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None: + client, _, headers = await _seen_by_app( + True, + ("192.0.2.9", 1000), + ((b"x-litellm-rust-peer", b"10.0.12.34:1"),), + Supervised(17, "0.0.0.0", False, "127.0.0.1"), + ) + assert (client, headers) == (("192.0.2.9", 1000), ()) + + +async def _peer_probe(request: Request) -> dict[str, object]: + assert request.client is not None + return {"client": [request.client.host, request.client.port], "peer": request.headers.get("x-litellm-rust-peer")} + + +@pytest.mark.parametrize("entry_point", ["litellm.proxy.rust_forward.asgi:app", "gateway.launch:worker_app"]) +async def test_every_entry_point_hands_routes_the_peer_rust_saw( + entry_point: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17") + monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", "0.0.0.0") + monkeypatch.setattr(proxy_app.router, "lifespan_context", proxy_app.router.lifespan_context) + monkeypatch.setattr( + proxy_app.router, "routes", [APIRoute("/__rust_forward_peer_probe", _peer_probe), *proxy_app.router.routes] + ) + try: + served: Final = import_from_string(entry_point)() + finally: + gc.unfreeze() + transport: Final = httpx.ASGITransport(app=served, client=("127.0.0.1", 60000)) + async with httpx.AsyncClient(transport=transport, base_url="http://proxy") as client: + response: Final = await client.get( + "/__rust_forward_peer_probe", headers={"x-litellm-rust-peer": "203.0.113.7:51234"} + ) + assert response.status_code == 200, response.text + assert response.json() == {"client": ["203.0.113.7", 51234], "peer": None} diff --git a/tests/unit/proxy/rust_forward/test_supervisor.py b/tests/unit/proxy/rust_forward/test_supervisor.py new file mode 100644 index 00000000000..224d97f349d --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_supervisor.py @@ -0,0 +1,139 @@ +import functools +import signal +import sys +from pathlib import Path +from typing import Final + +import pytest + +from litellm.proxy.proxy_cli import run_server +from litellm.proxy.rust_forward.listener import PublicListener +from litellm.proxy.rust_forward.mode import Supervised, rust_forward_mode +from litellm.proxy.rust_forward.supervisor import ( + Child, + Event, + ExitCode, + Exited, + Finished, + Running, + Signal, + State, + Stop, + cli_command, + gateway_binary, + python_spec, + rust_spec, + step, +) + +TERM: Final = signal.SIGTERM +INT: Final = signal.SIGINT + + +def _listener(proxy_headers: bool = True, trusted_proxies: str | None = None) -> PublicListener: + return PublicListener("0.0.0.0", 4000, 5, 40.0, proxy_headers, trusted_proxies) + + +@pytest.mark.parametrize( + ("environ", "listener", "expected"), + [ + ({}, _listener(), Supervised(17, "0.0.0.0", True, "127.0.0.1")), + ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(), Supervised(17, "0.0.0.0", True, "10.0.0.5")), + ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(True, "*"), Supervised(17, "0.0.0.0", True, "*")), + ({}, _listener(False), Supervised(17, "0.0.0.0", False, "127.0.0.1")), + ], +) +def test_python_child_keeps_the_trust_python_alone_would_use( + environ: dict[str, str], listener: PublicListener, expected: Supervised +) -> None: + spec: Final = python_spec(("python",), 17, 45555, listener, environ) + assert rust_forward_mode(spec.env) == expected + + +def test_python_child_listens_privately_and_keeps_options() -> None: + arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") + spec: Final = python_spec(arguments, 17, 45555, _listener(), {}) + params: Final = run_server.make_context("litellm", list(spec.argv)).params + assert params["host"] == "127.0.0.1" + assert params["port"] == 45555 + assert params["num_workers"] == 2 + assert params["reload"] is True + assert spec.pass_fds == (17,) + assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli") + + +def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None: + spec: Final = rust_spec( + "gateway", 45555, PublicListener("0.0.0.0", 4000, 2, 12.5, True, None), {"RUST_LOG": "info"} + ) + assert spec.argv == ("gateway",) + assert spec.pass_fds == () + assert dict(spec.env) == { + "RUST_LOG": "info", + "LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555", + "LITELLM_FORWARD_HOST": "0.0.0.0", + "LITELLM_FORWARD_PORT": "4000", + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE": "2", + "LITELLM_FORWARD_SHUTDOWN_DRAIN": "12.5", + } + + +def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + scripts: Final = tmp_path / "scripts" + elsewhere: Final = tmp_path / "elsewhere" + scripts.mkdir() + elsewhere.mkdir() + binary: Final = elsewhere / "litellm-gateway" + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o755) + monkeypatch.setenv("PATH", str(elsewhere)) + assert gateway_binary(str(scripts)) is None + assert gateway_binary(str(elsewhere)) == str(binary) + + +def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]: + def advance(acc: tuple[State, tuple[Signal, ...]], event: Event) -> tuple[State, tuple[Signal, ...]]: + state, sent = acc + following, action = step(state, event) + return following, sent if action is None else (*sent, action) + + return functools.reduce(advance, events, (Running(), ())) + + +RUST: Final = Child.RUST +PYTHON: Final = Child.PYTHON + + +@pytest.mark.parametrize( + ("events", "signals", "code"), + [ + ( + (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 0)), + (Signal(RUST, TERM), Signal(PYTHON, TERM)), + ExitCode.CLEAN, + ), + ((Exited(PYTHON, 0),), (), ExitCode.CHILD_FAILED), + ((Exited(RUST, 1),), (), ExitCode.CHILD_FAILED), + ((Stop(TERM), Exited(PYTHON, 0)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), + ((Stop(TERM), Exited(RUST, 1)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), + ( + (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 1)), + (Signal(RUST, TERM), Signal(PYTHON, TERM)), + ExitCode.CHILD_FAILED, + ), + ( + (Stop(INT), Stop(INT), Exited(RUST, 0), Exited(PYTHON, 0)), + (Signal(RUST, INT), Signal(PYTHON, INT)), + ExitCode.CLEAN, + ), + ( + (Exited(RUST, 1), Stop(TERM), Exited(PYTHON, 0)), + (), + ExitCode.CHILD_FAILED, + ), + ], +) +def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit( + events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode +) -> None: + assert _fold(events) == (Finished(code), signals) diff --git a/tests/unit/proxy/test_dev_reload.py b/tests/unit/proxy/test_dev_reload.py index 60a7c53820b..6650c1cd3e3 100644 --- a/tests/unit/proxy/test_dev_reload.py +++ b/tests/unit/proxy/test_dev_reload.py @@ -86,6 +86,39 @@ def test_a_broken_factory_exits_the_worker_as_a_startup_failure( assert exit_info.value.code == uvicorn_config.STARTUP_FAILURE +@pytest.mark.parametrize("worker_overrides", [None, {"factory": True}]) +def test_a_factory_flag_in_the_options_reaches_the_workers( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, worker_overrides: dict[str, object] | None +) -> None: + (tmp_path / "reload_factory_flag_app.py").write_text( + "async def served(scope, receive, send):\n" + " await send({'type': 'http.response.start', 'status': 204, 'headers': []})\n" + " await send({'type': 'http.response.body', 'body': b'', 'more_body': False})\n" + "\n" + "def factory():\n" + " return served\n" + ) + monkeypatch.syspath_prepend(str(tmp_path)) + config: Final = pool_config( + RELOAD_OPTIONS.validate_python({**_options("reload_factory_flag_app:factory"), "factory": True}), + workers=POOL_WORKERS, + worker_overrides=worker_overrides, + ) + config.load() + loaded_app: Final = config.loaded_app + assert loaded_app is not None + sent: Final[list[Message]] = [] + + async def receive() -> Message: + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(message: Message) -> None: + sent.append(message) + + asyncio.run(loaded_app({"type": "http", "method": "GET", "path": "/", "headers": []}, receive, send)) + assert sent[0] == {"type": "http.response.start", "status": 204, "headers": []} + + @REQUIRES_UVICORN_STARTUP_FAILURE @pytest.mark.parametrize( ("module", "source"), @@ -118,7 +151,9 @@ def test_the_reload_pool_defaults_the_worker_healthcheck_and_keeps_an_explicit_o explicit: Final = RELOAD_WORKER_HEALTHCHECK_SECONDS + 7 assert pool_config(_options("app:app"), workers=2).timeout_worker_healthcheck == RELOAD_WORKER_HEALTHCHECK_SECONDS assert ( - pool_config({**_options("app:app"), "timeout_worker_healthcheck": explicit}, workers=2).timeout_worker_healthcheck + pool_config( + {**_options("app:app"), "timeout_worker_healthcheck": explicit}, workers=2 + ).timeout_worker_healthcheck == explicit ) @@ -142,7 +177,9 @@ def test_the_reload_pool_accepts_and_keeps_every_uvicorn_arg_the_proxy_cli_build def test_the_reload_pool_passes_a_uvicorn_arg_it_does_not_list_through_to_the_workers() -> None: - config: Final = pool_config(RELOAD_OPTIONS.validate_python({**_options("app:app"), "backlog": 123}), workers=POOL_WORKERS) + config: Final = pool_config( + RELOAD_OPTIONS.validate_python({**_options("app:app"), "backlog": 123}), workers=POOL_WORKERS + ) assert (config.backlog, config.reload) == (123, False) diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index 643c01e3e7f..0ce5cc58191 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -1550,6 +1550,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=30, timeout_worker_healthcheck=None, + forwarded=None, ) mock_uvicorn_run.assert_called_once() @@ -1626,6 +1627,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=None, timeout_worker_healthcheck=15, + forwarded=None, ) @patch("uvicorn.run")