From b2e18b93a39d47843841dfa7d9bb7860f2f46183 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 13:21:05 +0900 Subject: [PATCH] fix(proxy): gather Rust forwarding into rust_forward and hand Rust Python's keepalive and drain Every Python line that exists only because Rust forwards to Python now lives in litellm/proxy/rust_forward (mode, peer, listener, supervisor, asgi), paired with the gateway-forward crate. proxy_server no longer knows about Rust. proxy_cli and gateway.launch wrap the app where they load it, apply server_overrides in the supervised child, and turn their own flags into one PublicListener The supervisor now resolves the keepalive and shutdown drain Python actually runs with and passes them to Rust as LITELLM_FORWARD_UPSTREAM_KEEPALIVE and LITELLM_FORWARD_SHUTDOWN_DRAIN, along with LITELLM_FORWARD_HOST and LITELLM_FORWARD_PORT. Before, `--keepalive_timeout 2` gave uvicorn 2s while Rust kept its default 5s pool and reused connections uvicorn was closing Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test-unit.yml | 1 + gateway/launch.py | 77 +++-- litellm/proxy/proxy_cli.py | 99 ++++-- litellm/proxy/proxy_server.py | 5 - litellm/proxy/rust_forward/AGENTS.md | 8 + litellm/proxy/rust_forward/__init__.py | 0 litellm/proxy/rust_forward/asgi.py | 10 + litellm/proxy/rust_forward/listener.py | 46 +++ litellm/proxy/rust_forward/mode.py | 68 ++++ litellm/proxy/rust_forward/peer.py | 57 ++++ .../supervisor.py} | 133 ++------ ...t_rust_gateway.py => test_rust_forward.py} | 0 tests/unit/gateway/test_launch.py | 63 +++- tests/unit/proxy/rust_forward/__init__.py | 0 .../unit/proxy/rust_forward/test_listener.py | 74 +++++ tests/unit/proxy/rust_forward/test_mode.py | 43 +++ tests/unit/proxy/rust_forward/test_peer.py | 144 +++++++++ .../proxy/rust_forward/test_supervisor.py | 139 +++++++++ tests/unit/proxy/test_proxy_cli.py | 4 +- tests/unit/proxy/test_rust_gateway.py | 293 ------------------ 20 files changed, 769 insertions(+), 495 deletions(-) create mode 100644 litellm/proxy/rust_forward/AGENTS.md create mode 100644 litellm/proxy/rust_forward/__init__.py create mode 100644 litellm/proxy/rust_forward/asgi.py create mode 100644 litellm/proxy/rust_forward/listener.py create mode 100644 litellm/proxy/rust_forward/mode.py create mode 100644 litellm/proxy/rust_forward/peer.py rename litellm/proxy/{rust_gateway.py => rust_forward/supervisor.py} (53%) rename tests/integration/configuration/{test_rust_gateway.py => test_rust_forward.py} (100%) create mode 100644 tests/unit/proxy/rust_forward/__init__.py create mode 100644 tests/unit/proxy/rust_forward/test_listener.py create mode 100644 tests/unit/proxy/rust_forward/test_mode.py create mode 100644 tests/unit/proxy/rust_forward/test_peer.py create mode 100644 tests/unit/proxy/rust_forward/test_supervisor.py delete mode 100644 tests/unit/proxy/test_rust_gateway.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 17f68cd7d32..5e61d6ebd4c 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -583,6 +583,7 @@ jobs: tests/unit/proxy/credential_endpoints tests/unit/proxy/discovery_endpoints tests/unit/proxy/shutdown + tests/unit/proxy/rust_forward tests/unit/proxy/public_endpoints tests/unit/proxy/prompts tests/unit/proxy/rag_endpoints diff --git a/gateway/launch.py b/gateway/launch.py index 5dec5de46fa..89341a4cb89 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -29,27 +29,29 @@ from uvicorn.main import main as uvicorn_main from litellm_cli.main import without_gc_pauses if TYPE_CHECKING: - from fastapi import FastAPI - from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings - from litellm.proxy.rust_gateway import Supervised + from litellm.proxy.rust_forward.listener import PublicListener + from litellm.proxy.rust_forward.mode import ServerOverrides + from litellm.proxy.rust_forward.peer import ASGIApp GATEWAY_APP: Final = "gateway.launch:worker_app" KEEPALIVE_FLAG: Final = "--timeout-keep-alive" -def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str], supervised: "Supervised | None") -> tuple[str, ...]: +def uvicorn_argv( + argv: Sequence[str], environ: Mapping[str, str], forwarded: "ServerOverrides | None" +) -> tuple[str, ...]: """Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly.""" keepalive: Final = environ.get("KEEPALIVE_TIMEOUT") inherited_socket: Final = ( - ( + () + if forwarded is None + else ( "--fd", - str(supervised.listener_fd), - "--no-proxy-headers", + str(forwarded.fd), + "--proxy-headers" if forwarded.proxy_headers else "--no-proxy-headers", ) - if supervised is not None - else () ) if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv): return (GATEWAY_APP, "--factory", *argv, *inherited_socket) @@ -77,26 +79,29 @@ def pool_database_url( return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth()) -def worker_app() -> "FastAPI": +def worker_app() -> "ASGIApp": with without_gc_pauses(): from gateway.main import app - return app + from litellm.proxy.rust_forward.mode import rust_forward_mode + from litellm.proxy.rust_forward.peer import wrap + return wrap(app, rust_forward_mode(os.environ)) def _serve(argv: Sequence[str]) -> None: uvicorn_main(tuple(argv), prog_name="uvicorn") -def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, int, Mapping[str, str]]: - """Validate the uvicorn flags for Rust forwarding and return the binary, public address and child environment.""" +def public_listener(argv: Sequence[str], binary: str | None) -> "tuple[str, PublicListener]": + """Validate the uvicorn flags for Rust forwarding and return the binary and the public listener.""" from pydantic import TypeAdapter - from litellm.proxy.rust_gateway import PROXY_HEADERS, TRUSTED_PROXIES, startup_error + from litellm.proxy.rust_forward.listener import PublicListener, StartupError, shutdown_drain_seconds, startup_check + from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager params: Final[Mapping[str, object]] = uvicorn_main.make_context( "uvicorn", list(uvicorn_argv(argv, os.environ, None)) ).params - error: Final = startup_error( + checked: Final = startup_check( binary, { "--ssl-keyfile": params["ssl_keyfile"], @@ -106,18 +111,18 @@ def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, "--fd": params["fd"] is not None, }, ) - if error is not None: - sys.exit(error) - assert binary is not None - host, port = TypeAdapter(tuple[str, int]).validate_python((params["host"], params["port"])) - operator_trust: Final = TypeAdapter(str | None).validate_python(params["forwarded_allow_ips"]) - proxy_headers: Final = TypeAdapter(bool).validate_python(params["proxy_headers"]) - return ( - binary, - host, - port, - {PROXY_HEADERS: str(proxy_headers).lower()} - | ({} if operator_trust is None else {TRUSTED_PROXIES: operator_trust}), + if isinstance(checked, StartupError): + sys.exit(checked.reason) + graceful_timeout: Final = TypeAdapter[int | None](int | None).validate_python(params["timeout_graceful_shutdown"]) + return checked, PublicListener( + host=TypeAdapter(str).validate_python(params["host"]), + port=TypeAdapter(int).validate_python(params["port"]), + keepalive_seconds=TypeAdapter(int).validate_python(params["timeout_keep_alive"]), + shutdown_drain_seconds=shutdown_drain_seconds( + GracefulShutdownManager.get_timeout() if graceful_timeout is None else graceful_timeout + ), + proxy_headers=TypeAdapter(bool).validate_python(params["proxy_headers"]), + trusted_proxies=TypeAdapter[str | None](str | None).validate_python(params["forwarded_allow_ips"]), ) @@ -135,27 +140,21 @@ def configure_database() -> None: def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: - from litellm.proxy.rust_gateway import ( - Disabled, - Supervised, - Supervisor, - gateway_binary, - rust_gateway_mode, - supervise, - ) + from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides + from litellm.proxy.rust_forward.supervisor import gateway_binary, supervise - mode: Final = rust_gateway_mode(os.environ) + mode: Final = rust_forward_mode(os.environ) match mode: case Disabled(): configure_database() serve(uvicorn_argv(argv, os.environ, None)) case Supervisor(): - binary, host, port, trust = public_listener(argv, gateway_binary(sysconfig.get_path("scripts"))) + binary, listener = public_listener(argv, gateway_binary(sysconfig.get_path("scripts"))) configure_database() command: Final = (sys.executable, "-m", "gateway.launch", *argv) - sys.exit(supervise(binary, command, host, port, {**os.environ, **trust})) + sys.exit(supervise(binary, command, listener, os.environ)) case Supervised(): - serve(uvicorn_argv(argv, os.environ, mode)) + serve(uvicorn_argv(argv, os.environ, server_overrides(mode))) case _: assert_never(mode) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index d1520fd56d9..cb8e936f3c5 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -36,6 +36,10 @@ from litellm.types.llms.base import LiteLLMBaseModel if TYPE_CHECKING: from fastapi import FastAPI + + from litellm.proxy.rust_forward.listener import PublicListener + from litellm.proxy.rust_forward.mode import ServerOverrides + from litellm.proxy.rust_forward.peer import ASGIApp else: FastAPI = Any @@ -262,6 +266,7 @@ def _apply_config_logging_settings(litellm_settings: StartupOutputSettings | Non WORKER_APP: Final = "litellm_cli.main:proxy_worker_app" +FORWARD_APP: Final = "litellm.proxy.rust_forward.asgi:app" WORKER_LOG_CONFIG_ENV: Final = "LITELLM_WORKER_LOG_CONFIG" _LOG_CONFIG: Final = TypeAdapter[str | Mapping[str, object]](str | Mapping[str, object]) _LOG_CONFIG_FILE: Final = TypeAdapter[Mapping[str, object]](Mapping[str, object]) @@ -390,12 +395,12 @@ class ProxyInitializationHelpers: logging.config.fileConfig(log_config, disable_existing_loggers=False) @staticmethod - def child_process_uvicorn_overrides(log_config: object) -> Mapping[str, object]: + def child_process_uvicorn_overrides(log_config: object, app: str = WORKER_APP) -> Mapping[str, object]: if log_config is None: - return MappingProxyType({"app": WORKER_APP, "factory": True}) + return MappingProxyType({"app": app, "factory": True}) ProxyInitializationHelpers.apply_log_config(_LOG_CONFIG.validate_python(log_config)) os.environ[WORKER_LOG_CONFIG_ENV] = json.dumps(log_config) - return MappingProxyType({"app": WORKER_APP, "factory": True, "log_config": None}) + return MappingProxyType({"app": app, "factory": True, "log_config": None}) @staticmethod def apply_worker_log_config(environ: Mapping[str, str]) -> None: @@ -410,7 +415,7 @@ class ProxyInitializationHelpers: log_config: str | None = None, keepalive_timeout: int | None = None, timeout_worker_healthcheck: int | None = None, - listener_fd: int | None = None, + forwarded: "ServerOverrides | None" = None, ) -> dict: """ Get the arguments for `uvicorn` worker @@ -429,7 +434,16 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), - **({"fd": listener_fd, "proxy_headers": False} if listener_fd is not None else {}), + **( + {} + if forwarded is None + else { + "app": FORWARD_APP, + "factory": True, + "fd": forwarded.fd, + "proxy_headers": forwarded.proxy_headers, + } + ), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -677,13 +691,13 @@ class ProxyInitializationHelpers: def run_gunicorn_server( host: str, port: int, - app: FastAPI, + app: "ASGIApp", num_workers: int, ssl_certfile_path: str, ssl_keyfile_path: str, max_requests_before_restart: int | None = None, max_requests_before_restart_jitter: int | None = None, - listener_fd: int | None = None, + forwarded: "ServerOverrides | None" = None, ): """ Run litellm with `gunicorn` @@ -756,10 +770,10 @@ class ProxyInitializationHelpers: ) graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout() gunicorn_options: Final = { - "bind": f"fd://{listener_fd}" if listener_fd is not None else f"{host}:{port}", + "bind": f"{host}:{port}" if forwarded is None else f"fd://{forwarded.fd}", "workers": num_workers, # default is 1 "worker_class": build_graceful_shutdown_uvicorn_worker( - graceful_shutdown_timeout, proxy_headers=listener_fd is None + graceful_shutdown_timeout, proxy_headers=True if forwarded is None else forwarded.proxy_headers ), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, "preload": True, # Add the preload flag, @@ -829,6 +843,34 @@ class ProxyInitializationHelpers: _is_port_in_use = is_port_in_use + @staticmethod + def public_listener(host: str, port: int, keepalive_timeout: int | None, run_gunicorn: bool) -> "PublicListener": + """``run_gunicorn_server`` never hands ``--keepalive_timeout`` to gunicorn, so its workers keep gunicorn's.""" + from gunicorn.config import Config as GunicornConfig + + from litellm.proxy.rust_forward.listener import ( + PublicListener, + shutdown_drain_seconds, + uvicorn_default_keepalive, + ) + from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager + + keepalive: Final = ( + TypeAdapter(int).validate_python(GunicornConfig().keepalive) + if run_gunicorn + else (uvicorn_default_keepalive() if keepalive_timeout is None else keepalive_timeout) + ) + return PublicListener( + host=host, + port=random.randint(1024, 49152) + if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) + else port, + keepalive_seconds=keepalive, + shutdown_drain_seconds=shutdown_drain_seconds(GracefulShutdownManager.get_timeout()), + proxy_headers=True, + trusted_proxies=None, + ) + @staticmethod def get_loop_type(): """Helper function to determine the event loop type based on platform""" @@ -1286,17 +1328,17 @@ def run_server( run_setup_wizard() return - from litellm.proxy.rust_gateway import Supervised, Supervisor, rust_gateway_mode + from litellm.proxy.rust_forward.mode import Supervised, Supervisor, rust_forward_mode, server_overrides - gateway_mode: Final = rust_gateway_mode(os.environ) - if isinstance(gateway_mode, Supervisor) and not any( + forward_mode: Final = rust_forward_mode(os.environ) + if isinstance(forward_mode, Supervisor) and not any( (version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data) ): - from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise + from litellm.proxy.rust_forward.listener import StartupError, startup_check + from litellm.proxy.rust_forward.supervisor import cli_command, gateway_binary, supervise - binary: Final = gateway_binary(sysconfig.get_path("scripts")) - gateway_error: Final = startup_error( - binary, + binary: Final = startup_check( + gateway_binary(sysconfig.get_path("scripts")), { "--ssl_keyfile_path": ssl_keyfile_path, "--ssl_certfile_path": ssl_certfile_path, @@ -1305,14 +1347,11 @@ def run_server( "--run_granian": run_granian, }, ) - if gateway_error is not None: - raise click.ClickException(gateway_error) - assert binary is not None - public_port: Final = ( - random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port - ) - raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port, os.environ)) - supervised: Final = gateway_mode if isinstance(gateway_mode, Supervised) else None + if isinstance(binary, StartupError): + raise click.ClickException(binary.reason) + listener: Final = ProxyInitializationHelpers.public_listener(host, port, keepalive_timeout, run_gunicorn) + raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), listener, os.environ)) + forwarded: Final = server_overrides(forward_mode) if isinstance(forward_mode, Supervised) else None args: Final = locals() try: @@ -1763,7 +1802,7 @@ def run_server( if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None: from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process - metrics_bind_host: Final = supervised.metrics_host if supervised is not None else host + metrics_bind_host: Final = host if forwarded is None else forwarded.metrics_host try: metrics_process: Final = start_metrics_server_process( host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir @@ -1782,7 +1821,7 @@ def run_server( log_config=log_config, keepalive_timeout=keepalive_timeout, timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None), - listener_fd=supervised.listener_fd if supervised is not None else None, + forwarded=forwarded, ) # Optional: recycle uvicorn workers after N requests if max_requests_before_restart is not None: @@ -1822,7 +1861,7 @@ def run_server( start_query_engine_reaper() if num_workers > 1 or reload: worker_overrides: Final = ProxyInitializationHelpers.child_process_uvicorn_overrides( - uvicorn_args.get("log_config") + uvicorn_args.get("log_config"), WORKER_APP if forwarded is None else FORWARD_APP ) if reload and num_workers > 1: from litellm.proxy.dev_reload import ( @@ -1848,16 +1887,18 @@ def run_server( workers=num_workers, ) elif run_gunicorn is True: + from litellm.proxy.rust_forward.peer import wrap + ProxyInitializationHelpers.run_gunicorn_server( host=host, port=port, - app=app, + app=wrap(app, forward_mode), num_workers=num_workers, ssl_certfile_path=ssl_certfile_path, ssl_keyfile_path=ssl_keyfile_path, max_requests_before_restart=max_requests_before_restart, max_requests_before_restart_jitter=max_requests_before_restart_jitter, - listener_fd=supervised.listener_fd if supervised is not None else None, + forwarded=forwarded, ) elif run_hypercorn is True: ProxyInitializationHelpers.init_hypercorn_server( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8a50c116125..c5fcbfb2564 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -835,7 +835,6 @@ from litellm.proxy.plugin_routes import ( from litellm.proxy.plugin_routes import ( router as plugin_router, ) -from litellm.proxy.rust_gateway import RustPeerMiddleware, Supervised, rust_gateway_mode from litellm.proxy.spend_tracking.spend_event_producer import ( CollectorSettings, SpendEventProducer, @@ -21925,10 +21924,6 @@ if _server_root_paths: ) app.add_middleware(PerRequestRootPathMiddleware, root_paths=_server_root_paths) -_rust_gateway_mode: Final = rust_gateway_mode(os.environ) -if isinstance(_rust_gateway_mode, Supervised): - app.add_middleware(RustPeerMiddleware, mode=_rust_gateway_mode) - async def _stream_mcp_asgi_response(handle_fn, scope: dict, receive) -> "StreamingResponse": """ diff --git a/litellm/proxy/rust_forward/AGENTS.md b/litellm/proxy/rust_forward/AGENTS.md new file mode 100644 index 00000000000..a2ab93cde5c --- /dev/null +++ b/litellm/proxy/rust_forward/AGENTS.md @@ -0,0 +1,8 @@ +# Rust forwarding, Python half + +With `LITELLM_RUST` on, the supervisor here runs the Rust gateway on the public port and Python on a private loopback listener. This package is the Python half of forwarding and pairs with the crate `litellm-rust/crates/gateway-forward`. Once Rust serves every route, both get deleted together, along with one dispatch branch in each entry point + +- Nothing outside this package imports Rust forwarding concepts, except the dispatch, the flag adapter and the app wrap in `litellm/proxy/proxy_cli.py` and `gateway/launch.py`. The proxy app itself never knows Rust is in front of it +- Python must see each request as if the client had connected directly. `x-litellm-rust-peer` is the only Rust-specific input, and `peer.py` honors it only from the loopback hop +- Python owns every setting Rust needs. The supervisor resolves the host, port, keepalive and shutdown drain Python actually runs with and hands them to Rust as the `LITELLM_FORWARD_*` variables in `rust_spec`. Rust never reads Python's own settings, so a new value Rust needs gets a new variable there +- Entry points turn their own flags into a `PublicListener`, apply `server_overrides` in the supervised child, and wrap the app with `peer.wrap` where they load it diff --git a/litellm/proxy/rust_forward/__init__.py b/litellm/proxy/rust_forward/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/proxy/rust_forward/asgi.py b/litellm/proxy/rust_forward/asgi.py new file mode 100644 index 00000000000..ff9bdc57cf8 --- /dev/null +++ b/litellm/proxy/rust_forward/asgi.py @@ -0,0 +1,10 @@ +import os + +from litellm.proxy.rust_forward.mode import rust_forward_mode +from litellm.proxy.rust_forward.peer import ASGIApp, wrap + + +def app() -> ASGIApp: + from litellm_cli.main import proxy_worker_app + + return wrap(proxy_worker_app(), rust_forward_mode(os.environ)) diff --git a/litellm/proxy/rust_forward/listener.py b/litellm/proxy/rust_forward/listener.py new file mode 100644 index 00000000000..28595a4099e --- /dev/null +++ b/litellm/proxy/rust_forward/listener.py @@ -0,0 +1,46 @@ +import os +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +from litellm.rust_bridge.configuration import ENV_NAME + + +@dataclass(frozen=True, slots=True) +class PublicListener: + host: str + port: int + keepalive_seconds: int + shutdown_drain_seconds: float + proxy_headers: bool + trusted_proxies: str | None + + +@dataclass(frozen=True, slots=True) +class StartupError: + reason: str + + +def startup_check(binary: str | None, unsupported: Mapping[str, object]) -> str | StartupError: + flag: Final = next((name for name, value in unsupported.items() if value), None) + if flag is not None: + return StartupError(f"{ENV_NAME} does not support {flag}") + if os.name != "posix": + return StartupError(f"{ENV_NAME} requires POSIX inherited sockets and process signals") + if binary is None: + return StartupError( + f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment" + ) + return binary + + +def uvicorn_default_keepalive() -> int: + import uvicorn + + return uvicorn.Config(app="", log_config=None).timeout_keep_alive + + +def shutdown_drain_seconds(graceful_timeout: float) -> float: + from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS + + return graceful_timeout + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS diff --git a/litellm/proxy/rust_forward/mode.py b/litellm/proxy/rust_forward/mode.py new file mode 100644 index 00000000000..a75287688ea --- /dev/null +++ b/litellm/proxy/rust_forward/mode.py @@ -0,0 +1,68 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final, TypeAlias + +from pydantic import TypeAdapter + +from litellm.rust_bridge.configuration import environment_switch + +GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" +METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" +PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" +TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" +DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1" +RUST_HOP: Final = "127.0.0.1" + +_FD: Final = TypeAdapter(int) +_ENABLED: Final = TypeAdapter(bool) + + +@dataclass(frozen=True, slots=True) +class Disabled: + pass + + +@dataclass(frozen=True, slots=True) +class Supervisor: + pass + + +@dataclass(frozen=True, slots=True) +class Supervised: + listener_fd: int + metrics_host: str + proxy_headers: bool + trusted_proxies: str + + +RustForwardMode: TypeAlias = Disabled | Supervisor | Supervised + + +def rust_forward_mode(environ: Mapping[str, str]) -> RustForwardMode: + fd: Final = environ.get(GATEWAY_FD) + if fd is not None: + return Supervised( + _FD.validate_python(fd), + environ[METRICS_HOST], + _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), + environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), + ) + if environment_switch(environ): + return Supervisor() + return Disabled() + + +@dataclass(frozen=True, slots=True) +class ServerOverrides: + fd: int + proxy_headers: bool + metrics_host: str + + +def server_overrides(mode: Supervised) -> ServerOverrides: + """The private listener the supervised server binds instead of its own flags. + + Proxy headers stay off at the server, since it would judge Rust's loopback address instead of the real peer. + ``peer.wrap`` applies the operator's trust settings to the restored peer. + """ + return ServerOverrides(fd=mode.listener_fd, proxy_headers=False, metrics_host=mode.metrics_host) diff --git a/litellm/proxy/rust_forward/peer.py b/litellm/proxy/rust_forward/peer.py new file mode 100644 index 00000000000..2d03d6ca35c --- /dev/null +++ b/litellm/proxy/rust_forward/peer.py @@ -0,0 +1,57 @@ +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Final, TypeAlias + +from typing_extensions import assert_never + +from litellm.proxy.rust_forward.mode import RUST_HOP, Disabled, RustForwardMode, Supervised, Supervisor + +if TYPE_CHECKING: + from uvicorn._types import ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope + +PEER_HEADER: Final = b"x-litellm-rust-peer" + +ASGIApp: TypeAlias = Callable[..., Awaitable[None]] + + +def _restore_peer(scope: "WWWScope") -> None: + headers: Final = tuple(scope["headers"]) + peer: Final = next((value for name, value in headers if name == PEER_HEADER), None) + if peer is None: + return + remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER) + scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop + client: Final = scope["client"] + if client is None or client[0] != RUST_HOP: + return + host, _, port = peer.decode("latin-1").rpartition(":") + scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope + + +class RustPeerMiddleware: + """Make Python see each request as if the client had connected to it directly. + + Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real + TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the + real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust + settings Python would have used on the public port. + """ + + def __init__(self, app: ASGIApp, mode: Supervised) -> None: + from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + + self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app + + async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + if scope["type"] != "lifespan": + _restore_peer(scope) + await self.app(scope, receive, send) + + +def wrap(app: ASGIApp, mode: RustForwardMode) -> ASGIApp: + match mode: + case Supervised(): + return RustPeerMiddleware(app, mode) + case Disabled() | Supervisor(): + return app + case _: + assert_never(mode) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_forward/supervisor.py similarity index 53% rename from litellm/proxy/rust_gateway.py rename to litellm/proxy/rust_forward/supervisor.py index 8be060e362c..6ab4b86dca3 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_forward/supervisor.py @@ -9,115 +9,22 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from enum import Enum, IntEnum from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeAlias +from typing import Final, TypeAlias from pydantic import TypeAdapter from typing_extensions import assert_never -from litellm.rust_bridge.configuration import ENV_NAME, environment_switch +from litellm.proxy.rust_forward.listener import PublicListener +from litellm.proxy.rust_forward.mode import GATEWAY_FD, METRICS_HOST, PROXY_HEADERS, RUST_HOP, TRUSTED_PROXIES -if TYPE_CHECKING: - from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope - -GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" -METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" -PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" -TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" -DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1" -RUST_HOP: Final = "127.0.0.1" -PEER_HEADER: Final = b"x-litellm-rust-peer" - -_FD: Final = TypeAdapter(int) -_ENABLED: Final = TypeAdapter(bool) _ADDRESS: Final = TypeAdapter(tuple[str, int]) -@dataclass(frozen=True, slots=True) -class Disabled: - pass - - -@dataclass(frozen=True, slots=True) -class Supervisor: - pass - - -@dataclass(frozen=True, slots=True) -class Supervised: - listener_fd: int - metrics_host: str - proxy_headers: bool - trusted_proxies: str - - -RustGatewayMode: TypeAlias = Disabled | Supervisor | Supervised - - -def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode: - fd: Final = environ.get(GATEWAY_FD) - if fd is not None: - return Supervised( - _FD.validate_python(fd), - environ[METRICS_HOST], - _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), - environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), - ) - if environment_switch(environ): - return Supervisor() - return Disabled() - - -def _restore_peer(scope: "WWWScope") -> None: - headers: Final = tuple(scope["headers"]) - peer: Final = next((value for name, value in headers if name == PEER_HEADER), None) - if peer is None: - return - remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER) - scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop - client: Final = scope["client"] - if client is None or client[0] != RUST_HOP: - return - host, _, port = peer.decode("latin-1").rpartition(":") - scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope - - -class RustPeerMiddleware: - """Make Python see each request as if the client had connected to it directly. - - Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real - TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the - real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust - settings Python would have used on the public port. The server's proxy-header handling is off in - this mode, since it would judge Rust's loopback address instead of the real peer. - """ - - def __init__(self, app: "ASGI3Application", mode: Supervised) -> None: - from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware - - self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app - - async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: - if scope["type"] != "lifespan": - _restore_peer(scope) - await self.app(scope, receive, send) - - def gateway_binary(scripts_dir: str) -> str | None: name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway" return shutil.which(name, path=scripts_dir) -def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: - flag: Final = next((name for name, value in unsupported.items() if value), None) - if flag is not None: - return f"{ENV_NAME} does not support {flag}" - if os.name != "posix": - return f"{ENV_NAME} requires POSIX inherited sockets and process signals" - if binary is None: - return f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment" - return None - - class Child(Enum): RUST = "rust" PYTHON = "python" @@ -209,7 +116,7 @@ class ChildSpec: def python_spec( - command: Sequence[str], listener_fd: int, upstream_port: int, public_host: str, environ: Mapping[str, str] + command: Sequence[str], listener_fd: int, upstream_port: int, listener: PublicListener, environ: Mapping[str, str] ) -> ChildSpec: return ChildSpec( argv=(*command, "--host", RUST_HOP, "--port", str(upstream_port)), @@ -217,22 +124,26 @@ def python_spec( { **environ, GATEWAY_FD: str(listener_fd), - METRICS_HOST: public_host, + METRICS_HOST: listener.host, + PROXY_HEADERS: str(listener.proxy_headers).lower(), + **({} if listener.trusted_proxies is None else {TRUSTED_PROXIES: listener.trusted_proxies}), } ), pass_fds=(listener_fd,), ) -def rust_spec(binary: str, upstream_port: int, host: str, port: int, environ: Mapping[str, str]) -> ChildSpec: +def rust_spec(binary: str, upstream_port: int, listener: PublicListener, environ: Mapping[str, str]) -> ChildSpec: return ChildSpec( argv=(binary,), env=MappingProxyType( { **environ, - "HOST": host, - "PORT": str(port), "LITELLM_FORWARD_UPSTREAM": f"http://{RUST_HOP}:{upstream_port}", + "LITELLM_FORWARD_HOST": listener.host, + "LITELLM_FORWARD_PORT": str(listener.port), + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE": str(listener.keepalive_seconds), + "LITELLM_FORWARD_SHUTDOWN_DRAIN": str(listener.shutdown_drain_seconds), } ), pass_fds=(), @@ -266,18 +177,20 @@ async def _drive( return await _drive(following, events, children) -async def _supervise(binary: str, command: Sequence[str], host: str, port: int, environ: Mapping[str, str]) -> ExitCode: +async def _supervise( + binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str] +) -> ExitCode: loop: Final = asyncio.get_running_loop() events: Final[asyncio.Queue[Event]] = asyncio.Queue() for signum in (signal.SIGTERM, signal.SIGINT): loop.add_signal_handler(signum, events.put_nowait, Stop(signum)) - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: - listener.bind((RUST_HOP, 0)) - listener.listen() - _, upstream_port = _ADDRESS.validate_python(listener.getsockname()) - python: Final = await _spawn(python_spec(command, listener.fileno(), upstream_port, host, environ)) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as private: + private.bind((RUST_HOP, 0)) + private.listen() + _, upstream_port = _ADDRESS.validate_python(private.getsockname()) + python: Final = await _spawn(python_spec(command, private.fileno(), upstream_port, listener, environ)) try: - rust: Final = await _spawn(rust_spec(binary, upstream_port, host, port, environ)) + rust: Final = await _spawn(rust_spec(binary, upstream_port, listener, environ)) except OSError: _terminate_group(python) await python.wait() @@ -294,9 +207,9 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int, await asyncio.gather(*waiters) -def supervise(binary: str, command: Sequence[str], host: str, port: int, environ: Mapping[str, str]) -> int: +def supervise(binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str]) -> int: try: - return asyncio.run(_supervise(binary, command, host, port, environ)) + return asyncio.run(_supervise(binary, command, listener, environ)) except OSError as error: sys.stderr.write(f"Rust gateway startup failed: {error}\n") return ExitCode.CHILD_FAILED diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_forward.py similarity index 100% rename from tests/integration/configuration/test_rust_gateway.py rename to tests/integration/configuration/test_rust_forward.py diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index 418e0358b64..36eaabfb029 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -12,14 +12,18 @@ from typing import Final, cast from unittest.mock import MagicMock, patch import pytest +import uvicorn from uvicorn.importer import import_from_string from uvicorn.main import main as uvicorn_main from gateway.launch import GATEWAY_APP, main, pool_database_url, public_listener, uvicorn_argv +from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings from litellm.proxy.proxy_server import app as proxy_app -from litellm.proxy.rust_gateway import Supervised +from litellm.proxy.rust_forward.mode import Supervised, server_overrides +from litellm.proxy.rust_forward.supervisor import rust_spec +from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager DB_ENV: Final = { "DATABASE_HOST": "db.internal", @@ -127,34 +131,59 @@ class TestUvicornArgv: @pytest.mark.parametrize( ("argv", "trust"), [ - ((), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true"}), - ( - ("--forwarded-allow-ips", "10.0.0.5"), - {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "10.0.0.5"}, - ), - (("--forwarded-allow-ips=*",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "*"}), - (("--no-proxy-headers",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}), + ((), (True, None)), + (("--forwarded-allow-ips", "10.0.0.5"), (True, "10.0.0.5")), + (("--forwarded-allow-ips=*",), (True, "*")), + (("--no-proxy-headers",), (False, None)), ], ) def test_forwarding_carries_operator_proxy_header_options_to_the_child( - self, argv: tuple[str, ...], trust: dict[str, str] + self, argv: tuple[str, ...], trust: tuple[bool, str | None], monkeypatch: pytest.MonkeyPatch ) -> None: - assert public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") == ( - "gateway", - "0.0.0.0", - 4000, - trust, - ) + monkeypatch.delenv("FORWARDED_ALLOW_IPS", raising=False) + binary, listener = public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") + assert (binary, listener.host, listener.port) == ("gateway", "0.0.0.0", 4000) + assert (listener.proxy_headers, listener.trusted_proxies) == trust + + @pytest.mark.parametrize( + ("argv", "environ"), + [(("--timeout-keep-alive", "2"), {}), ((), {"KEEPALIVE_TIMEOUT": "7"}), ((), {})], + ) + def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with( + self, argv: tuple[str, ...], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + for name, value in environ.items(): + monkeypatch.setenv(name, value) + _, listener = public_listener(argv, "gateway") + sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"] + assert sent == str(_uvicorn_params(uvicorn_argv(argv, os.environ, None))["timeout_keep_alive"]) + + def test_without_a_keepalive_rust_follows_uvicorns_own_default(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + _, listener = public_listener((), "gateway") + assert listener.keepalive_seconds == uvicorn.Config(app="", log_config=None).timeout_keep_alive + + @pytest.mark.parametrize("argv", [(), ("--timeout-graceful-shutdown", "12")]) + def test_rust_drains_for_pythons_graceful_timeout_plus_teardown( + self, argv: tuple[str, ...], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "7.5") + _, listener = public_listener(argv, "gateway") + flag: Final = _uvicorn_params(uvicorn_argv(argv, {}, None))["timeout_graceful_shutdown"] + graceful: Final = GracefulShutdownManager.get_timeout() if flag is None else float(str(flag)) + sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"] + assert float(sent) == graceful + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS def test_supervised_listener_uses_the_inherited_loopback_socket(self): - supervised: Final = Supervised(17, "0.0.0.0", True, "127.0.0.1") + supervised: Final = server_overrides(Supervised(17, "0.0.0.0", True, "127.0.0.1")) params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised)) assert params["fd"] == 17 assert params["workers"] == 2 @pytest.mark.parametrize("argv", [(), ("--proxy-headers",), ("--forwarded-allow-ips=*",)]) def test_private_hop_leaves_proxy_headers_to_the_app(self, argv: tuple[str, ...]): - params: Final = _uvicorn_params(uvicorn_argv(argv, {}, Supervised(17, "0.0.0.0", True, "*"))) + params: Final = _uvicorn_params(uvicorn_argv(argv, {}, server_overrides(Supervised(17, "0.0.0.0", True, "*")))) assert params["proxy_headers"] is False def test_direct_listener_keeps_explicit_proxy_header_options(self): diff --git a/tests/unit/proxy/rust_forward/__init__.py b/tests/unit/proxy/rust_forward/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/rust_forward/test_listener.py b/tests/unit/proxy/rust_forward/test_listener.py new file mode 100644 index 00000000000..d17b0e74e14 --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_listener.py @@ -0,0 +1,74 @@ +from collections.abc import Mapping +from typing import Final + +import pytest +import uvicorn +from click.testing import CliRunner +from gunicorn.config import Config as GunicornConfig + +from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS +from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server +from litellm.proxy.rust_forward.listener import PublicListener, StartupError, startup_check +from litellm.proxy.rust_forward.supervisor import rust_spec + + +@pytest.mark.parametrize( + "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] +) +def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "true") + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] + result: Final = CliRunner().invoke(run_server, [flag, *values]) + assert result.exit_code != 0 + assert f"LITELLM_RUST does not support {flag}" in result.output + + +def test_missing_binary_gives_install_instruction() -> None: + assert startup_check(None, {}) == StartupError( + "LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment" + ) + + +def _cli_listener(argv: list[str]) -> tuple[Mapping[str, object], PublicListener]: + params: Final = run_server.make_context("litellm", ["--port", "45999", *argv]).params + return params, ProxyInitializationHelpers.public_listener( + str(params["host"]), int(str(params["port"])), params["keepalive_timeout"], params["run_gunicorn"] is True + ) + + +def _uvicorn_keepalive(options: Mapping[str, object]) -> int: + configured: Final = {"timeout_keep_alive": options["timeout_keep_alive"]} if "timeout_keep_alive" in options else {} + return uvicorn.Config(app="", log_config=None, **configured).timeout_keep_alive + + +@pytest.mark.parametrize( + ("argv", "environ"), + [(["--keepalive_timeout", "2"], {}), ([], {"KEEPALIVE_TIMEOUT": "7"}), ([], {})], +) +def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with( + argv: list[str], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + for name, value in environ.items(): + monkeypatch.setenv(name, value) + params, listener = _cli_listener(argv) + uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 45555, keepalive_timeout=params["keepalive_timeout"] + ) + sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"] + assert sent == str(_uvicorn_keepalive(uvicorn_options)) + + +def test_gunicorn_workers_keep_gunicorns_keepalive_and_rust_follows(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + _, listener = _cli_listener(["--run_gunicorn", "--keepalive_timeout", "9"]) + assert listener.keepalive_seconds == GunicornConfig().keepalive + + +def test_rust_drains_for_pythons_graceful_timeout_plus_teardown(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "12.5") + _, listener = _cli_listener([]) + uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555) + sent: Final = float(rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"]) + assert sent == uvicorn_options["timeout_graceful_shutdown"] + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS diff --git a/tests/unit/proxy/rust_forward/test_mode.py b/tests/unit/proxy/rust_forward/test_mode.py new file mode 100644 index 00000000000..20005a21255 --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_mode.py @@ -0,0 +1,43 @@ +from typing import Final + +import pytest + +from litellm.proxy.proxy_cli import ProxyInitializationHelpers +from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides + + +@pytest.mark.parametrize( + ("environ", "expected"), + [ + ({}, Disabled()), + ({"LITELLM_RUST": "false"}, Disabled()), + ({"LITELLM_RUST": "off"}, Disabled()), + ({"LITELLM_RUST": "0"}, Disabled()), + ({"LITELLM_RUST": "true"}, Supervisor()), + ({"LITELLM_RUST": "1"}, Supervisor()), + ({"LITELLM_RUST": "yes"}, Supervisor()), + ({"LITELLM_RUST": "on"}, Supervisor()), + ({"LITELLM_RUST": " 1 "}, Supervisor()), + ({"LITELLM_RUST": "enabled"}, Disabled()), + ( + { + "LITELLM_RUST": "true", + "_LITELLM_GATEWAY_FD": "17", + "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", + "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", + }, + Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"), + ), + ], +) +def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None: + assert rust_forward_mode(environ) == expected + + +@pytest.mark.parametrize("mode", [None, Supervised(17, "0.0.0.0", True, "127.0.0.1")]) +def test_inherited_socket_reaches_uvicorn(mode: Supervised | None) -> None: + options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 45555, forwarded=None if mode is None else server_overrides(mode) + ) + assert options.get("fd") == (None if mode is None else mode.listener_fd) + assert options.get("proxy_headers", True) is (mode is None) diff --git a/tests/unit/proxy/rust_forward/test_peer.py b/tests/unit/proxy/rust_forward/test_peer.py new file mode 100644 index 00000000000..e96d9cf2b6a --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_peer.py @@ -0,0 +1,144 @@ +import asyncio +import gc +from typing import TYPE_CHECKING, Final + +import httpx +import pytest +from fastapi import Request +from fastapi.routing import APIRoute +from uvicorn.importer import import_from_string +from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + +if TYPE_CHECKING: + from uvicorn._types import ( + ASGIReceiveCallable, + ASGISendCallable, + ASGISendEvent, + HTTPDisconnectEvent, + HTTPScope, + Scope, + ) + +from litellm.proxy.proxy_server import app as proxy_app +from litellm.proxy.rust_forward.mode import Supervised +from litellm.proxy.rust_forward.peer import wrap + + +Headers = tuple[tuple[bytes, bytes], ...] + + +async def _receive() -> "HTTPDisconnectEvent": + return {"type": "http.disconnect"} + + +async def _send(message: "ASGISendEvent") -> None: + return None + + +async def _seen_by_app( + behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised +) -> tuple[tuple[str, int] | None, str, Headers]: + seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = ( + asyncio.get_running_loop().create_future() + ) + + async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + assert scope["type"] == "http" + seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"]))) + + direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe + app: Final = wrap(probe, mode) if behind_rust else direct + scope: Final[HTTPScope] = { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": "/", + "raw_path": b"/", + "query_string": b"", + "root_path": "", + "headers": headers, + "client": client, + "server": ("127.0.0.1", 4000), + } + await app(scope, _receive, _send) + return seen.result() + + +PEER: Final = ("203.0.113.7", 51234) +TRUSTED_PEER: Final = ("10.0.12.34", 40000) +FORWARDED: Final[Headers] = ( + (b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"), + (b"x-forwarded-proto", b"https"), +) + + +@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)]) +@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))]) +@pytest.mark.parametrize( + "mode", + [ + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + Supervised(17, "0.0.0.0", True, "10.0.0.0/8"), + Supervised(17, "0.0.0.0", True, "*"), + Supervised(17, "0.0.0.0", False, "10.0.0.0/8"), + ], +) +async def test_python_behind_rust_sees_what_a_direct_connection_would( + peer: tuple[str, int], headers: Headers, mode: Supervised +) -> None: + rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}" + behind_rust: Final = await _seen_by_app( + True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode + ) + direct: Final = await _seen_by_app(False, peer, headers, mode) + assert behind_rust == direct + + +async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None: + client, scheme, _ = await _seen_by_app( + True, + ("127.0.0.1", 60000), + ((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")), + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + ) + assert (client, scheme) == (("203.0.113.7", 51234), "http") + + +async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None: + client, _, headers = await _seen_by_app( + True, + ("192.0.2.9", 1000), + ((b"x-litellm-rust-peer", b"10.0.12.34:1"),), + Supervised(17, "0.0.0.0", False, "127.0.0.1"), + ) + assert (client, headers) == (("192.0.2.9", 1000), ()) + + +async def _peer_probe(request: Request) -> dict[str, object]: + assert request.client is not None + return {"client": [request.client.host, request.client.port], "peer": request.headers.get("x-litellm-rust-peer")} + + +@pytest.mark.parametrize("entry_point", ["litellm.proxy.rust_forward.asgi:app", "gateway.launch:worker_app"]) +async def test_every_entry_point_hands_routes_the_peer_rust_saw( + entry_point: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17") + monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", "0.0.0.0") + monkeypatch.setattr(proxy_app.router, "lifespan_context", proxy_app.router.lifespan_context) + monkeypatch.setattr( + proxy_app.router, "routes", [APIRoute("/__rust_forward_peer_probe", _peer_probe), *proxy_app.router.routes] + ) + try: + served: Final = import_from_string(entry_point)() + finally: + gc.unfreeze() + transport: Final = httpx.ASGITransport(app=served, client=("127.0.0.1", 60000)) + async with httpx.AsyncClient(transport=transport, base_url="http://proxy") as client: + response: Final = await client.get( + "/__rust_forward_peer_probe", headers={"x-litellm-rust-peer": "203.0.113.7:51234"} + ) + assert response.status_code == 200, response.text + assert response.json() == {"client": ["203.0.113.7", 51234], "peer": None} diff --git a/tests/unit/proxy/rust_forward/test_supervisor.py b/tests/unit/proxy/rust_forward/test_supervisor.py new file mode 100644 index 00000000000..224d97f349d --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_supervisor.py @@ -0,0 +1,139 @@ +import functools +import signal +import sys +from pathlib import Path +from typing import Final + +import pytest + +from litellm.proxy.proxy_cli import run_server +from litellm.proxy.rust_forward.listener import PublicListener +from litellm.proxy.rust_forward.mode import Supervised, rust_forward_mode +from litellm.proxy.rust_forward.supervisor import ( + Child, + Event, + ExitCode, + Exited, + Finished, + Running, + Signal, + State, + Stop, + cli_command, + gateway_binary, + python_spec, + rust_spec, + step, +) + +TERM: Final = signal.SIGTERM +INT: Final = signal.SIGINT + + +def _listener(proxy_headers: bool = True, trusted_proxies: str | None = None) -> PublicListener: + return PublicListener("0.0.0.0", 4000, 5, 40.0, proxy_headers, trusted_proxies) + + +@pytest.mark.parametrize( + ("environ", "listener", "expected"), + [ + ({}, _listener(), Supervised(17, "0.0.0.0", True, "127.0.0.1")), + ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(), Supervised(17, "0.0.0.0", True, "10.0.0.5")), + ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(True, "*"), Supervised(17, "0.0.0.0", True, "*")), + ({}, _listener(False), Supervised(17, "0.0.0.0", False, "127.0.0.1")), + ], +) +def test_python_child_keeps_the_trust_python_alone_would_use( + environ: dict[str, str], listener: PublicListener, expected: Supervised +) -> None: + spec: Final = python_spec(("python",), 17, 45555, listener, environ) + assert rust_forward_mode(spec.env) == expected + + +def test_python_child_listens_privately_and_keeps_options() -> None: + arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") + spec: Final = python_spec(arguments, 17, 45555, _listener(), {}) + params: Final = run_server.make_context("litellm", list(spec.argv)).params + assert params["host"] == "127.0.0.1" + assert params["port"] == 45555 + assert params["num_workers"] == 2 + assert params["reload"] is True + assert spec.pass_fds == (17,) + assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli") + + +def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None: + spec: Final = rust_spec( + "gateway", 45555, PublicListener("0.0.0.0", 4000, 2, 12.5, True, None), {"RUST_LOG": "info"} + ) + assert spec.argv == ("gateway",) + assert spec.pass_fds == () + assert dict(spec.env) == { + "RUST_LOG": "info", + "LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555", + "LITELLM_FORWARD_HOST": "0.0.0.0", + "LITELLM_FORWARD_PORT": "4000", + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE": "2", + "LITELLM_FORWARD_SHUTDOWN_DRAIN": "12.5", + } + + +def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + scripts: Final = tmp_path / "scripts" + elsewhere: Final = tmp_path / "elsewhere" + scripts.mkdir() + elsewhere.mkdir() + binary: Final = elsewhere / "litellm-gateway" + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o755) + monkeypatch.setenv("PATH", str(elsewhere)) + assert gateway_binary(str(scripts)) is None + assert gateway_binary(str(elsewhere)) == str(binary) + + +def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]: + def advance(acc: tuple[State, tuple[Signal, ...]], event: Event) -> tuple[State, tuple[Signal, ...]]: + state, sent = acc + following, action = step(state, event) + return following, sent if action is None else (*sent, action) + + return functools.reduce(advance, events, (Running(), ())) + + +RUST: Final = Child.RUST +PYTHON: Final = Child.PYTHON + + +@pytest.mark.parametrize( + ("events", "signals", "code"), + [ + ( + (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 0)), + (Signal(RUST, TERM), Signal(PYTHON, TERM)), + ExitCode.CLEAN, + ), + ((Exited(PYTHON, 0),), (), ExitCode.CHILD_FAILED), + ((Exited(RUST, 1),), (), ExitCode.CHILD_FAILED), + ((Stop(TERM), Exited(PYTHON, 0)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), + ((Stop(TERM), Exited(RUST, 1)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), + ( + (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 1)), + (Signal(RUST, TERM), Signal(PYTHON, TERM)), + ExitCode.CHILD_FAILED, + ), + ( + (Stop(INT), Stop(INT), Exited(RUST, 0), Exited(PYTHON, 0)), + (Signal(RUST, INT), Signal(PYTHON, INT)), + ExitCode.CLEAN, + ), + ( + (Exited(RUST, 1), Stop(TERM), Exited(PYTHON, 0)), + (), + ExitCode.CHILD_FAILED, + ), + ], +) +def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit( + events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode +) -> None: + assert _fold(events) == (Finished(code), signals) diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index db36f0d681f..0ce5cc58191 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -1550,7 +1550,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=30, timeout_worker_healthcheck=None, - listener_fd=None, + forwarded=None, ) mock_uvicorn_run.assert_called_once() @@ -1627,7 +1627,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=None, timeout_worker_healthcheck=15, - listener_fd=None, + forwarded=None, ) @patch("uvicorn.run") diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py deleted file mode 100644 index 1072148be9f..00000000000 --- a/tests/unit/proxy/test_rust_gateway.py +++ /dev/null @@ -1,293 +0,0 @@ -import asyncio -import functools -import signal -import sys -from pathlib import Path -from typing import TYPE_CHECKING, Final - -import pytest -from click.testing import CliRunner -from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware - -if TYPE_CHECKING: - from uvicorn._types import ( - ASGIReceiveCallable, - ASGISendCallable, - ASGISendEvent, - HTTPDisconnectEvent, - HTTPScope, - Scope, - ) - -from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server -from litellm.proxy.rust_gateway import ( - Child, - Disabled, - Event, - ExitCode, - Exited, - Finished, - Running, - RustPeerMiddleware, - Signal, - State, - Stop, - Supervised, - Supervisor, - cli_command, - gateway_binary, - python_spec, - rust_gateway_mode, - rust_spec, - startup_error, - step, -) - -TERM: Final = signal.SIGTERM -INT: Final = signal.SIGINT - - -@pytest.mark.parametrize( - "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] -) -def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("LITELLM_RUST", "true") - monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) - values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] - result: Final = CliRunner().invoke(run_server, [flag, *values]) - assert result.exit_code != 0 - assert f"LITELLM_RUST does not support {flag}" in result.output - - -@pytest.mark.parametrize( - ("environ", "expected"), - [ - ({}, Disabled()), - ({"LITELLM_RUST": "false"}, Disabled()), - ({"LITELLM_RUST": "off"}, Disabled()), - ({"LITELLM_RUST": "0"}, Disabled()), - ({"LITELLM_RUST": "true"}, Supervisor()), - ({"LITELLM_RUST": "1"}, Supervisor()), - ({"LITELLM_RUST": "yes"}, Supervisor()), - ({"LITELLM_RUST": "on"}, Supervisor()), - ({"LITELLM_RUST": " 1 "}, Supervisor()), - ({"LITELLM_RUST": "enabled"}, Disabled()), - ( - { - "LITELLM_RUST": "true", - "_LITELLM_GATEWAY_FD": "17", - "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", - "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", - }, - Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"), - ), - ], -) -def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None: - assert rust_gateway_mode(environ) == expected - - -@pytest.mark.parametrize( - ("environ", "expected"), - [ - ({}, Supervised(17, "0.0.0.0", True, "127.0.0.1")), - ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, Supervised(17, "0.0.0.0", True, "10.0.0.5")), - ({"FORWARDED_ALLOW_IPS": "*"}, Supervised(17, "0.0.0.0", True, "*")), - ({"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}, Supervised(17, "0.0.0.0", False, "127.0.0.1")), - ], -) -def test_python_child_keeps_the_trust_python_alone_would_use(environ: dict[str, str], expected: Supervised) -> None: - spec: Final = python_spec(("python",), 17, 45555, "0.0.0.0", environ) - assert rust_gateway_mode(spec.env) == expected - - -def test_python_child_listens_privately_and_keeps_options() -> None: - arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") - spec: Final = python_spec(arguments, 17, 45555, "0.0.0.0", {}) - params: Final = run_server.make_context("litellm", list(spec.argv)).params - assert params["host"] == "127.0.0.1" - assert params["port"] == 45555 - assert params["num_workers"] == 2 - assert params["reload"] is True - assert spec.pass_fds == (17,) - assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli") - - -def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None: - spec: Final = rust_spec("gateway", 45555, "0.0.0.0", 4000, {"RUST_LOG": "info"}) - assert spec.argv == ("gateway",) - assert spec.pass_fds == () - assert dict(spec.env) == { - "RUST_LOG": "info", - "HOST": "0.0.0.0", - "PORT": "4000", - "LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555", - } - - -def test_missing_binary_gives_install_instruction() -> None: - assert startup_error(None, {}) == ( - "LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment" - ) - - -def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - scripts: Final = tmp_path / "scripts" - elsewhere: Final = tmp_path / "elsewhere" - scripts.mkdir() - elsewhere.mkdir() - binary: Final = elsewhere / "litellm-gateway" - binary.write_text("#!/bin/sh\nexit 0\n") - binary.chmod(0o755) - monkeypatch.setenv("PATH", str(elsewhere)) - assert gateway_binary(str(scripts)) is None - assert gateway_binary(str(elsewhere)) == str(binary) - - -@pytest.mark.parametrize("listener_fd", [None, 17]) -def test_inherited_socket_reaches_uvicorn(listener_fd: int | None) -> None: - options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( - "127.0.0.1", 45555, listener_fd=listener_fd - ) - assert options.get("fd") == listener_fd - assert options.get("proxy_headers", True) is (listener_fd is None) - - -def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]: - def advance(acc: tuple[State, tuple[Signal, ...]], event: Event) -> tuple[State, tuple[Signal, ...]]: - state, sent = acc - following, action = step(state, event) - return following, sent if action is None else (*sent, action) - - return functools.reduce(advance, events, (Running(), ())) - - -RUST: Final = Child.RUST -PYTHON: Final = Child.PYTHON - - -@pytest.mark.parametrize( - ("events", "signals", "code"), - [ - ( - (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 0)), - (Signal(RUST, TERM), Signal(PYTHON, TERM)), - ExitCode.CLEAN, - ), - ((Exited(PYTHON, 0),), (), ExitCode.CHILD_FAILED), - ((Exited(RUST, 1),), (), ExitCode.CHILD_FAILED), - ((Stop(TERM), Exited(PYTHON, 0)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), - ((Stop(TERM), Exited(RUST, 1)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED), - ( - (Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 1)), - (Signal(RUST, TERM), Signal(PYTHON, TERM)), - ExitCode.CHILD_FAILED, - ), - ( - (Stop(INT), Stop(INT), Exited(RUST, 0), Exited(PYTHON, 0)), - (Signal(RUST, INT), Signal(PYTHON, INT)), - ExitCode.CLEAN, - ), - ( - (Exited(RUST, 1), Stop(TERM), Exited(PYTHON, 0)), - (), - ExitCode.CHILD_FAILED, - ), - ], -) -def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit( - events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode -) -> None: - assert _fold(events) == (Finished(code), signals) - - -Headers = tuple[tuple[bytes, bytes], ...] - - -async def _receive() -> "HTTPDisconnectEvent": - return {"type": "http.disconnect"} - - -async def _send(message: "ASGISendEvent") -> None: - return None - - -async def _seen_by_app( - behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised -) -> tuple[tuple[str, int] | None, str, Headers]: - seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = ( - asyncio.get_running_loop().create_future() - ) - - async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: - assert scope["type"] == "http" - seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"]))) - - direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe - app: Final = RustPeerMiddleware(probe, mode) if behind_rust else direct - scope: Final[HTTPScope] = { - "type": "http", - "asgi": {"version": "3.0", "spec_version": "2.3"}, - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": "/", - "raw_path": b"/", - "query_string": b"", - "root_path": "", - "headers": headers, - "client": client, - "server": ("127.0.0.1", 4000), - } - await app(scope, _receive, _send) - return seen.result() - - -PEER: Final = ("203.0.113.7", 51234) -TRUSTED_PEER: Final = ("10.0.12.34", 40000) -FORWARDED: Final[Headers] = ( - (b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"), - (b"x-forwarded-proto", b"https"), -) - - -@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)]) -@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))]) -@pytest.mark.parametrize( - "mode", - [ - Supervised(17, "0.0.0.0", True, "127.0.0.1"), - Supervised(17, "0.0.0.0", True, "10.0.0.0/8"), - Supervised(17, "0.0.0.0", True, "*"), - Supervised(17, "0.0.0.0", False, "10.0.0.0/8"), - ], -) -async def test_python_behind_rust_sees_what_a_direct_connection_would( - peer: tuple[str, int], headers: Headers, mode: Supervised -) -> None: - rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}" - behind_rust: Final = await _seen_by_app( - True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode - ) - direct: Final = await _seen_by_app(False, peer, headers, mode) - assert behind_rust == direct - - -async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None: - client, scheme, _ = await _seen_by_app( - True, - ("127.0.0.1", 60000), - ((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")), - Supervised(17, "0.0.0.0", True, "127.0.0.1"), - ) - assert (client, scheme) == (("203.0.113.7", 51234), "http") - - -async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None: - client, _, headers = await _seen_by_app( - True, - ("192.0.2.9", 1000), - ((b"x-litellm-rust-peer", b"10.0.12.34:1"),), - Supervised(17, "0.0.0.0", False, "127.0.0.1"), - ) - assert (client, headers) == (("192.0.2.9", 1000), ())