diff --git a/gateway/launch.py b/gateway/launch.py index d5d798f262b..5dec5de46fa 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -46,9 +46,7 @@ def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str], supervised: "S ( "--fd", str(supervised.listener_fd), - "--proxy-headers", - "--forwarded-allow-ips", - ",".join(supervised.trusted_proxies), + "--no-proxy-headers", ) if supervised is not None else () @@ -93,7 +91,7 @@ def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, """Validate the uvicorn flags for Rust forwarding and return the binary, public address and child environment.""" from pydantic import TypeAdapter - from litellm.proxy.rust_gateway import TRUSTED_PROXIES, startup_error + from litellm.proxy.rust_gateway import PROXY_HEADERS, TRUSTED_PROXIES, startup_error params: Final[Mapping[str, object]] = uvicorn_main.make_context( "uvicorn", list(uvicorn_argv(argv, os.environ, None)) @@ -113,7 +111,14 @@ def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, assert binary is not None host, port = TypeAdapter(tuple[str, int]).validate_python((params["host"], params["port"])) operator_trust: Final = TypeAdapter(str | None).validate_python(params["forwarded_allow_ips"]) - return binary, host, port, {} if operator_trust is None else {TRUSTED_PROXIES: operator_trust} + proxy_headers: Final = TypeAdapter(bool).validate_python(params["proxy_headers"]) + return ( + binary, + host, + port, + {PROXY_HEADERS: str(proxy_headers).lower()} + | ({} if operator_trust is None else {TRUSTED_PROXIES: operator_trust}), + ) def configure_database() -> None: diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 68db6d3f316..d1520fd56d9 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -429,7 +429,7 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), - **({"fd": listener_fd} if listener_fd is not None else {}), + **({"fd": listener_fd, "proxy_headers": False} if listener_fd is not None else {}), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -758,7 +758,9 @@ class ProxyInitializationHelpers: gunicorn_options: Final = { "bind": f"fd://{listener_fd}" if listener_fd is not None else f"{host}:{port}", "workers": num_workers, # default is 1 - "worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout), + "worker_class": build_graceful_shutdown_uvicorn_worker( + graceful_shutdown_timeout, proxy_headers=listener_fd is None + ), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, "preload": True, # Add the preload flag, "accesslog": "-", # Log to stdout diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c5fcbfb2564..8a50c116125 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -835,6 +835,7 @@ from litellm.proxy.plugin_routes import ( from litellm.proxy.plugin_routes import ( router as plugin_router, ) +from litellm.proxy.rust_gateway import RustPeerMiddleware, Supervised, rust_gateway_mode from litellm.proxy.spend_tracking.spend_event_producer import ( CollectorSettings, SpendEventProducer, @@ -21924,6 +21925,10 @@ if _server_root_paths: ) app.add_middleware(PerRequestRootPathMiddleware, root_paths=_server_root_paths) +_rust_gateway_mode: Final = rust_gateway_mode(os.environ) +if isinstance(_rust_gateway_mode, Supervised): + app.add_middleware(RustPeerMiddleware, mode=_rust_gateway_mode) + async def _stream_mcp_asgi_response(handle_fn, scope: dict, receive) -> "StreamingResponse": """ diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 65237eb00e0..0dbf3a4de3d 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -9,16 +9,22 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from enum import Enum, IntEnum from types import MappingProxyType -from typing import Final, TypeAlias +from typing import TYPE_CHECKING, Final, TypeAlias from pydantic import TypeAdapter from typing_extensions import assert_never +if TYPE_CHECKING: + from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope + ENABLE: Final = "LITELLM_RUST_GATEWAY" GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" +PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" +DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1" RUST_HOP: Final = "127.0.0.1" +PEER_HEADER: Final = b"x-litellm-rust-peer" _FD: Final = TypeAdapter(int) _ENABLED: Final = TypeAdapter(bool) @@ -39,26 +45,62 @@ class Supervisor: class Supervised: listener_fd: int metrics_host: str - trusted_proxies: tuple[str, ...] + proxy_headers: bool + trusted_proxies: str RustGatewayMode: TypeAlias = Disabled | Supervisor | Supervised -def trusted_proxies(operator_value: str | None) -> tuple[str, ...]: - configured: Final = tuple(host.strip() for host in (operator_value or "").split(",") if host.strip()) - return ("*",) if "*" in configured else (RUST_HOP, *(host for host in configured if host != RUST_HOP)) - - def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode: fd: Final = environ.get(GATEWAY_FD) if fd is not None: - return Supervised(_FD.validate_python(fd), environ[METRICS_HOST], trusted_proxies(environ.get(TRUSTED_PROXIES))) + return Supervised( + _FD.validate_python(fd), + environ[METRICS_HOST], + _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), + environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), + ) if _ENABLED.validate_python(environ.get(ENABLE, "false")): return Supervisor() return Disabled() +def _restore_peer(scope: "WWWScope") -> None: + headers: Final = tuple(scope["headers"]) + peer: Final = next((value for name, value in headers if name == PEER_HEADER), None) + if peer is None: + return + remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER) + scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop + client: Final = scope["client"] + if client is None or client[0] != RUST_HOP: + return + host, _, port = peer.decode("latin-1").rpartition(":") + scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope + + +class RustPeerMiddleware: + """Make Python see each request as if the client had connected to it directly. + + Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real + TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the + real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust + settings Python would have used on the public port. The server's proxy-header handling is off in + this mode, since it would judge Rust's loopback address instead of the real peer. + """ + + def __init__(self, app: "ASGI3Application", mode: Supervised) -> None: + from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + + self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app + + async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + if scope["type"] != "lifespan": + _restore_peer(scope) + await self.app(scope, receive, send) + + def gateway_binary(scripts_dir: str) -> str | None: name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway" return shutil.which(name, path=scripts_dir) @@ -175,7 +217,6 @@ def python_spec( **environ, GATEWAY_FD: str(listener_fd), METRICS_HOST: public_host, - TRUSTED_PROXIES: ",".join(trusted_proxies(environ.get(TRUSTED_PROXIES))), } ), pass_fds=(listener_fd,), diff --git a/litellm/proxy/shutdown/graceful_uvicorn_worker.py b/litellm/proxy/shutdown/graceful_uvicorn_worker.py index 6ae5ae60c3f..8bbd64190f8 100644 --- a/litellm/proxy/shutdown/graceful_uvicorn_worker.py +++ b/litellm/proxy/shutdown/graceful_uvicorn_worker.py @@ -1,10 +1,16 @@ from uvicorn.workers import UvicornWorker -def build_graceful_shutdown_uvicorn_worker(timeout_graceful_shutdown: float) -> type[UvicornWorker]: +def build_graceful_shutdown_uvicorn_worker( + timeout_graceful_shutdown: float, *, proxy_headers: bool = True +) -> type[UvicornWorker]: class GracefulShutdownUvicornWorker(UvicornWorker): def __init__(self, *args: object) -> None: - self.CONFIG_KWARGS = {**UvicornWorker.CONFIG_KWARGS, "timeout_graceful_shutdown": timeout_graceful_shutdown} + self.CONFIG_KWARGS = { + **UvicornWorker.CONFIG_KWARGS, + "timeout_graceful_shutdown": timeout_graceful_shutdown, + "proxy_headers": proxy_headers, + } super().__init__(*args) return GracefulShutdownUvicornWorker diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index 729c0b847e3..06d77d6f3d0 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -22,7 +22,6 @@ def ready(client: httpx.Client) -> bool: return False -@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) def client_ip(base_url: str, local_address: str, key: str, forwarded_for: str) -> httpx.Response: with httpx.Client( base_url=base_url, @@ -36,6 +35,7 @@ def client_ip(base_url: str, local_address: str, key: str, forwarded_for: str) - ) +@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) def test_gateway_preserves_metrics_bind_and_mcp_client_ip( gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...] ) -> None: diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index 606e760ab3f..789067653b7 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -127,12 +127,16 @@ class TestUvicornArgv: @pytest.mark.parametrize( ("argv", "trust"), [ - ((), {}), - (("--forwarded-allow-ips", "10.0.0.5"), {"FORWARDED_ALLOW_IPS": "10.0.0.5"}), - (("--forwarded-allow-ips=*",), {"FORWARDED_ALLOW_IPS": "*"}), + ((), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true"}), + ( + ("--forwarded-allow-ips", "10.0.0.5"), + {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "10.0.0.5"}, + ), + (("--forwarded-allow-ips=*",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "*"}), + (("--no-proxy-headers",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}), ], ) - def test_forwarding_carries_operator_proxy_trust_to_the_child( + def test_forwarding_carries_operator_proxy_header_options_to_the_child( self, argv: tuple[str, ...], trust: dict[str, str] ) -> None: assert public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") == ( @@ -143,20 +147,15 @@ class TestUvicornArgv: ) def test_supervised_listener_uses_the_inherited_loopback_socket(self): - supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1",)) + supervised: Final = Supervised(17, "0.0.0.0", True, "127.0.0.1") params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised)) assert params["fd"] == 17 assert params["workers"] == 2 - @pytest.mark.parametrize( - "argv", - [(), ("--no-proxy-headers",), ("--forwarded-allow-ips", "192.0.2.1"), ("--forwarded-allow-ips=*",)], - ) - def test_private_hop_always_processes_headers_with_the_resolved_trust(self, argv: tuple[str, ...]): - supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5")) - params: Final = _uvicorn_params(uvicorn_argv(argv, {}, supervised)) - assert params["proxy_headers"] is True - assert params["forwarded_allow_ips"] == "127.0.0.1,10.0.0.5" + @pytest.mark.parametrize("argv", [(), ("--proxy-headers",), ("--forwarded-allow-ips=*",)]) + def test_private_hop_leaves_proxy_headers_to_the_app(self, argv: tuple[str, ...]): + params: Final = _uvicorn_params(uvicorn_argv(argv, {}, Supervised(17, "0.0.0.0", True, "*"))) + assert params["proxy_headers"] is False def test_direct_listener_keeps_explicit_proxy_header_options(self): params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {}, None)) diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index 278dc4621d3..b71a098ddb3 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -1,12 +1,24 @@ +import asyncio import functools import signal import sys from pathlib import Path -from typing import Final +from typing import TYPE_CHECKING, Final import pytest from click.testing import CliRunner from pydantic import ValidationError +from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + +if TYPE_CHECKING: + from uvicorn._types import ( + ASGIReceiveCallable, + ASGISendCallable, + ASGISendEvent, + HTTPDisconnectEvent, + HTTPScope, + Scope, + ) from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server from litellm.proxy.rust_gateway import ( @@ -17,6 +29,7 @@ from litellm.proxy.rust_gateway import ( Exited, Finished, Running, + RustPeerMiddleware, Signal, State, Stop, @@ -29,7 +42,6 @@ from litellm.proxy.rust_gateway import ( rust_spec, startup_error, step, - trusted_proxies, ) TERM: Final = signal.SIGTERM @@ -66,7 +78,7 @@ def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", }, - Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5")), + Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"), ), ], ) @@ -80,27 +92,17 @@ def test_unrecognized_switch_value_is_rejected() -> None: @pytest.mark.parametrize( - ("operator_value", "expected"), + ("environ", "expected"), [ - (None, ("127.0.0.1",)), - ("", ("127.0.0.1",)), - ("10.0.0.5", ("127.0.0.1", "10.0.0.5")), - ("127.0.0.1, 10.0.0.5", ("127.0.0.1", "10.0.0.5")), - ("10.0.0.5,127.0.0.1", ("127.0.0.1", "10.0.0.5")), - ("*", ("*",)), - ("10.0.0.5,*", ("*",)), + ({}, Supervised(17, "0.0.0.0", True, "127.0.0.1")), + ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, Supervised(17, "0.0.0.0", True, "10.0.0.5")), + ({"FORWARDED_ALLOW_IPS": "*"}, Supervised(17, "0.0.0.0", True, "*")), + ({"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}, Supervised(17, "0.0.0.0", False, "127.0.0.1")), ], ) -def test_operator_trust_is_kept_and_rust_hop_added(operator_value: str | None, expected: tuple[str, ...]) -> None: - assert trusted_proxies(operator_value) == expected - - -@pytest.mark.parametrize("operator_value", [None, "", "10.0.0.5", "10.0.0.5, 192.0.2.0/24", "127.0.0.1"]) -def test_python_child_always_trusts_the_rust_hop(operator_value: str | None) -> None: - environ: Final = {} if operator_value is None else {"FORWARDED_ALLOW_IPS": operator_value} +def test_python_child_keeps_the_trust_python_alone_would_use(environ: dict[str, str], expected: Supervised) -> None: spec: Final = python_spec(("python",), 17, 45555, "0.0.0.0", environ) - assert spec.env["FORWARDED_ALLOW_IPS"].split(",")[0] == "127.0.0.1" - assert rust_gateway_mode(spec.env) == Supervised(17, "0.0.0.0", trusted_proxies(operator_value)) + assert rust_gateway_mode(spec.env) == expected def test_python_child_listens_privately_and_keeps_options() -> None: @@ -152,6 +154,7 @@ def test_inherited_socket_reaches_uvicorn(listener_fd: int | None) -> None: "127.0.0.1", 45555, listener_fd=listener_fd ) assert options.get("fd") == listener_fd + assert options.get("proxy_headers", True) is (listener_fd is None) def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]: @@ -200,3 +203,95 @@ def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit( events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode ) -> None: assert _fold(events) == (Finished(code), signals) + + +Headers = tuple[tuple[bytes, bytes], ...] + + +async def _receive() -> "HTTPDisconnectEvent": + return {"type": "http.disconnect"} + + +async def _send(message: "ASGISendEvent") -> None: + return None + + +async def _seen_by_app( + behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised +) -> tuple[tuple[str, int] | None, str, Headers]: + seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = ( + asyncio.get_running_loop().create_future() + ) + + async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + assert scope["type"] == "http" + seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"]))) + + direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe + app: Final = RustPeerMiddleware(probe, mode) if behind_rust else direct + scope: Final[HTTPScope] = { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": "/", + "raw_path": b"/", + "query_string": b"", + "root_path": "", + "headers": headers, + "client": client, + "server": ("127.0.0.1", 4000), + } + await app(scope, _receive, _send) + return seen.result() + + +PEER: Final = ("203.0.113.7", 51234) +TRUSTED_PEER: Final = ("10.0.12.34", 40000) +FORWARDED: Final[Headers] = ( + (b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"), + (b"x-forwarded-proto", b"https"), +) + + +@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)]) +@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))]) +@pytest.mark.parametrize( + "mode", + [ + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + Supervised(17, "0.0.0.0", True, "10.0.0.0/8"), + Supervised(17, "0.0.0.0", True, "*"), + Supervised(17, "0.0.0.0", False, "10.0.0.0/8"), + ], +) +async def test_python_behind_rust_sees_what_a_direct_connection_would( + peer: tuple[str, int], headers: Headers, mode: Supervised +) -> None: + rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}" + behind_rust: Final = await _seen_by_app( + True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode + ) + direct: Final = await _seen_by_app(False, peer, headers, mode) + assert behind_rust == direct + + +async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None: + client, scheme, _ = await _seen_by_app( + True, + ("127.0.0.1", 60000), + ((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")), + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + ) + assert (client, scheme) == (("203.0.113.7", 51234), "http") + + +async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None: + client, _, headers = await _seen_by_app( + True, + ("192.0.2.9", 1000), + ((b"x-litellm-rust-peer", b"10.0.12.34:1"),), + Supervised(17, "0.0.0.0", False, "127.0.0.1"), + ) + assert (client, headers) == (("192.0.2.9", 1000), ())