From 543c01530530f30eb0397f4eca1f1318f43df871 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 09:07:08 +0900 Subject: [PATCH] fix(proxy): restore the client's peer behind Rust instead of trusting the Rust hop Python now sees each request as if the client had connected to it directly. The Rust hop sends the real TCP peer in x-litellm-rust-peer, an app middleware puts it back into the ASGI scope, and uvicorn's ProxyHeadersMiddleware then runs with the same trust settings Python alone would use. Server-level proxy headers are off in supervised mode for uvicorn and gunicorn, since they would judge the loopback hop. The trust-list merge that always trusted 127.0.0.1 is gone, so operators keep the exact FORWARDED_ALLOW_IPS behavior they had. Co-Authored-By: Claude Opus 5.5 --- gateway/launch.py | 15 +- litellm/proxy/proxy_cli.py | 6 +- litellm/proxy/proxy_server.py | 5 + litellm/proxy/rust_gateway.py | 59 ++++++-- .../proxy/shutdown/graceful_uvicorn_worker.py | 10 +- .../configuration/test_rust_gateway.py | 2 +- tests/unit/gateway/test_launch.py | 27 ++-- tests/unit/proxy/test_rust_gateway.py | 135 +++++++++++++++--- 8 files changed, 206 insertions(+), 53 deletions(-) diff --git a/gateway/launch.py b/gateway/launch.py index d5d798f262b..5dec5de46fa 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -46,9 +46,7 @@ def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str], supervised: "S ( "--fd", str(supervised.listener_fd), - "--proxy-headers", - "--forwarded-allow-ips", - ",".join(supervised.trusted_proxies), + "--no-proxy-headers", ) if supervised is not None else () @@ -93,7 +91,7 @@ def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, """Validate the uvicorn flags for Rust forwarding and return the binary, public address and child environment.""" from pydantic import TypeAdapter - from litellm.proxy.rust_gateway import TRUSTED_PROXIES, startup_error + from litellm.proxy.rust_gateway import PROXY_HEADERS, TRUSTED_PROXIES, startup_error params: Final[Mapping[str, object]] = uvicorn_main.make_context( "uvicorn", list(uvicorn_argv(argv, os.environ, None)) @@ -113,7 +111,14 @@ def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, assert binary is not None host, port = TypeAdapter(tuple[str, int]).validate_python((params["host"], params["port"])) operator_trust: Final = TypeAdapter(str | None).validate_python(params["forwarded_allow_ips"]) - return binary, host, port, {} if operator_trust is None else {TRUSTED_PROXIES: operator_trust} + proxy_headers: Final = TypeAdapter(bool).validate_python(params["proxy_headers"]) + return ( + binary, + host, + port, + {PROXY_HEADERS: str(proxy_headers).lower()} + | ({} if operator_trust is None else {TRUSTED_PROXIES: operator_trust}), + ) def configure_database() -> None: diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 68db6d3f316..d1520fd56d9 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -429,7 +429,7 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), - **({"fd": listener_fd} if listener_fd is not None else {}), + **({"fd": listener_fd, "proxy_headers": False} if listener_fd is not None else {}), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -758,7 +758,9 @@ class ProxyInitializationHelpers: gunicorn_options: Final = { "bind": f"fd://{listener_fd}" if listener_fd is not None else f"{host}:{port}", "workers": num_workers, # default is 1 - "worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout), + "worker_class": build_graceful_shutdown_uvicorn_worker( + graceful_shutdown_timeout, proxy_headers=listener_fd is None + ), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, "preload": True, # Add the preload flag, "accesslog": "-", # Log to stdout diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c5fcbfb2564..8a50c116125 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -835,6 +835,7 @@ from litellm.proxy.plugin_routes import ( from litellm.proxy.plugin_routes import ( router as plugin_router, ) +from litellm.proxy.rust_gateway import RustPeerMiddleware, Supervised, rust_gateway_mode from litellm.proxy.spend_tracking.spend_event_producer import ( CollectorSettings, SpendEventProducer, @@ -21924,6 +21925,10 @@ if _server_root_paths: ) app.add_middleware(PerRequestRootPathMiddleware, root_paths=_server_root_paths) +_rust_gateway_mode: Final = rust_gateway_mode(os.environ) +if isinstance(_rust_gateway_mode, Supervised): + app.add_middleware(RustPeerMiddleware, mode=_rust_gateway_mode) + async def _stream_mcp_asgi_response(handle_fn, scope: dict, receive) -> "StreamingResponse": """ diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 65237eb00e0..0dbf3a4de3d 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -9,16 +9,22 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from enum import Enum, IntEnum from types import MappingProxyType -from typing import Final, TypeAlias +from typing import TYPE_CHECKING, Final, TypeAlias from pydantic import TypeAdapter from typing_extensions import assert_never +if TYPE_CHECKING: + from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope + ENABLE: Final = "LITELLM_RUST_GATEWAY" GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" +PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS" +DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1" RUST_HOP: Final = "127.0.0.1" +PEER_HEADER: Final = b"x-litellm-rust-peer" _FD: Final = TypeAdapter(int) _ENABLED: Final = TypeAdapter(bool) @@ -39,26 +45,62 @@ class Supervisor: class Supervised: listener_fd: int metrics_host: str - trusted_proxies: tuple[str, ...] + proxy_headers: bool + trusted_proxies: str RustGatewayMode: TypeAlias = Disabled | Supervisor | Supervised -def trusted_proxies(operator_value: str | None) -> tuple[str, ...]: - configured: Final = tuple(host.strip() for host in (operator_value or "").split(",") if host.strip()) - return ("*",) if "*" in configured else (RUST_HOP, *(host for host in configured if host != RUST_HOP)) - - def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode: fd: Final = environ.get(GATEWAY_FD) if fd is not None: - return Supervised(_FD.validate_python(fd), environ[METRICS_HOST], trusted_proxies(environ.get(TRUSTED_PROXIES))) + return Supervised( + _FD.validate_python(fd), + environ[METRICS_HOST], + _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), + environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), + ) if _ENABLED.validate_python(environ.get(ENABLE, "false")): return Supervisor() return Disabled() +def _restore_peer(scope: "WWWScope") -> None: + headers: Final = tuple(scope["headers"]) + peer: Final = next((value for name, value in headers if name == PEER_HEADER), None) + if peer is None: + return + remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER) + scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop + client: Final = scope["client"] + if client is None or client[0] != RUST_HOP: + return + host, _, port = peer.decode("latin-1").rpartition(":") + scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope + + +class RustPeerMiddleware: + """Make Python see each request as if the client had connected to it directly. + + Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real + TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the + real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust + settings Python would have used on the public port. The server's proxy-header handling is off in + this mode, since it would judge Rust's loopback address instead of the real peer. + """ + + def __init__(self, app: "ASGI3Application", mode: Supervised) -> None: + from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + + self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app + + async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + if scope["type"] != "lifespan": + _restore_peer(scope) + await self.app(scope, receive, send) + + def gateway_binary(scripts_dir: str) -> str | None: name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway" return shutil.which(name, path=scripts_dir) @@ -175,7 +217,6 @@ def python_spec( **environ, GATEWAY_FD: str(listener_fd), METRICS_HOST: public_host, - TRUSTED_PROXIES: ",".join(trusted_proxies(environ.get(TRUSTED_PROXIES))), } ), pass_fds=(listener_fd,), diff --git a/litellm/proxy/shutdown/graceful_uvicorn_worker.py b/litellm/proxy/shutdown/graceful_uvicorn_worker.py index 6ae5ae60c3f..8bbd64190f8 100644 --- a/litellm/proxy/shutdown/graceful_uvicorn_worker.py +++ b/litellm/proxy/shutdown/graceful_uvicorn_worker.py @@ -1,10 +1,16 @@ from uvicorn.workers import UvicornWorker -def build_graceful_shutdown_uvicorn_worker(timeout_graceful_shutdown: float) -> type[UvicornWorker]: +def build_graceful_shutdown_uvicorn_worker( + timeout_graceful_shutdown: float, *, proxy_headers: bool = True +) -> type[UvicornWorker]: class GracefulShutdownUvicornWorker(UvicornWorker): def __init__(self, *args: object) -> None: - self.CONFIG_KWARGS = {**UvicornWorker.CONFIG_KWARGS, "timeout_graceful_shutdown": timeout_graceful_shutdown} + self.CONFIG_KWARGS = { + **UvicornWorker.CONFIG_KWARGS, + "timeout_graceful_shutdown": timeout_graceful_shutdown, + "proxy_headers": proxy_headers, + } super().__init__(*args) return GracefulShutdownUvicornWorker diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index 729c0b847e3..06d77d6f3d0 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -22,7 +22,6 @@ def ready(client: httpx.Client) -> bool: return False -@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) def client_ip(base_url: str, local_address: str, key: str, forwarded_for: str) -> httpx.Response: with httpx.Client( base_url=base_url, @@ -36,6 +35,7 @@ def client_ip(base_url: str, local_address: str, key: str, forwarded_for: str) - ) +@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) def test_gateway_preserves_metrics_bind_and_mcp_client_ip( gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...] ) -> None: diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index 606e760ab3f..789067653b7 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -127,12 +127,16 @@ class TestUvicornArgv: @pytest.mark.parametrize( ("argv", "trust"), [ - ((), {}), - (("--forwarded-allow-ips", "10.0.0.5"), {"FORWARDED_ALLOW_IPS": "10.0.0.5"}), - (("--forwarded-allow-ips=*",), {"FORWARDED_ALLOW_IPS": "*"}), + ((), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true"}), + ( + ("--forwarded-allow-ips", "10.0.0.5"), + {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "10.0.0.5"}, + ), + (("--forwarded-allow-ips=*",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "*"}), + (("--no-proxy-headers",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}), ], ) - def test_forwarding_carries_operator_proxy_trust_to_the_child( + def test_forwarding_carries_operator_proxy_header_options_to_the_child( self, argv: tuple[str, ...], trust: dict[str, str] ) -> None: assert public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") == ( @@ -143,20 +147,15 @@ class TestUvicornArgv: ) def test_supervised_listener_uses_the_inherited_loopback_socket(self): - supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1",)) + supervised: Final = Supervised(17, "0.0.0.0", True, "127.0.0.1") params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised)) assert params["fd"] == 17 assert params["workers"] == 2 - @pytest.mark.parametrize( - "argv", - [(), ("--no-proxy-headers",), ("--forwarded-allow-ips", "192.0.2.1"), ("--forwarded-allow-ips=*",)], - ) - def test_private_hop_always_processes_headers_with_the_resolved_trust(self, argv: tuple[str, ...]): - supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5")) - params: Final = _uvicorn_params(uvicorn_argv(argv, {}, supervised)) - assert params["proxy_headers"] is True - assert params["forwarded_allow_ips"] == "127.0.0.1,10.0.0.5" + @pytest.mark.parametrize("argv", [(), ("--proxy-headers",), ("--forwarded-allow-ips=*",)]) + def test_private_hop_leaves_proxy_headers_to_the_app(self, argv: tuple[str, ...]): + params: Final = _uvicorn_params(uvicorn_argv(argv, {}, Supervised(17, "0.0.0.0", True, "*"))) + assert params["proxy_headers"] is False def test_direct_listener_keeps_explicit_proxy_header_options(self): params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {}, None)) diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index 278dc4621d3..b71a098ddb3 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -1,12 +1,24 @@ +import asyncio import functools import signal import sys from pathlib import Path -from typing import Final +from typing import TYPE_CHECKING, Final import pytest from click.testing import CliRunner from pydantic import ValidationError +from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + +if TYPE_CHECKING: + from uvicorn._types import ( + ASGIReceiveCallable, + ASGISendCallable, + ASGISendEvent, + HTTPDisconnectEvent, + HTTPScope, + Scope, + ) from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server from litellm.proxy.rust_gateway import ( @@ -17,6 +29,7 @@ from litellm.proxy.rust_gateway import ( Exited, Finished, Running, + RustPeerMiddleware, Signal, State, Stop, @@ -29,7 +42,6 @@ from litellm.proxy.rust_gateway import ( rust_spec, startup_error, step, - trusted_proxies, ) TERM: Final = signal.SIGTERM @@ -66,7 +78,7 @@ def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", }, - Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5")), + Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"), ), ], ) @@ -80,27 +92,17 @@ def test_unrecognized_switch_value_is_rejected() -> None: @pytest.mark.parametrize( - ("operator_value", "expected"), + ("environ", "expected"), [ - (None, ("127.0.0.1",)), - ("", ("127.0.0.1",)), - ("10.0.0.5", ("127.0.0.1", "10.0.0.5")), - ("127.0.0.1, 10.0.0.5", ("127.0.0.1", "10.0.0.5")), - ("10.0.0.5,127.0.0.1", ("127.0.0.1", "10.0.0.5")), - ("*", ("*",)), - ("10.0.0.5,*", ("*",)), + ({}, Supervised(17, "0.0.0.0", True, "127.0.0.1")), + ({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, Supervised(17, "0.0.0.0", True, "10.0.0.5")), + ({"FORWARDED_ALLOW_IPS": "*"}, Supervised(17, "0.0.0.0", True, "*")), + ({"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}, Supervised(17, "0.0.0.0", False, "127.0.0.1")), ], ) -def test_operator_trust_is_kept_and_rust_hop_added(operator_value: str | None, expected: tuple[str, ...]) -> None: - assert trusted_proxies(operator_value) == expected - - -@pytest.mark.parametrize("operator_value", [None, "", "10.0.0.5", "10.0.0.5, 192.0.2.0/24", "127.0.0.1"]) -def test_python_child_always_trusts_the_rust_hop(operator_value: str | None) -> None: - environ: Final = {} if operator_value is None else {"FORWARDED_ALLOW_IPS": operator_value} +def test_python_child_keeps_the_trust_python_alone_would_use(environ: dict[str, str], expected: Supervised) -> None: spec: Final = python_spec(("python",), 17, 45555, "0.0.0.0", environ) - assert spec.env["FORWARDED_ALLOW_IPS"].split(",")[0] == "127.0.0.1" - assert rust_gateway_mode(spec.env) == Supervised(17, "0.0.0.0", trusted_proxies(operator_value)) + assert rust_gateway_mode(spec.env) == expected def test_python_child_listens_privately_and_keeps_options() -> None: @@ -152,6 +154,7 @@ def test_inherited_socket_reaches_uvicorn(listener_fd: int | None) -> None: "127.0.0.1", 45555, listener_fd=listener_fd ) assert options.get("fd") == listener_fd + assert options.get("proxy_headers", True) is (listener_fd is None) def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]: @@ -200,3 +203,95 @@ def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit( events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode ) -> None: assert _fold(events) == (Finished(code), signals) + + +Headers = tuple[tuple[bytes, bytes], ...] + + +async def _receive() -> "HTTPDisconnectEvent": + return {"type": "http.disconnect"} + + +async def _send(message: "ASGISendEvent") -> None: + return None + + +async def _seen_by_app( + behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised +) -> tuple[tuple[str, int] | None, str, Headers]: + seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = ( + asyncio.get_running_loop().create_future() + ) + + async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + assert scope["type"] == "http" + seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"]))) + + direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe + app: Final = RustPeerMiddleware(probe, mode) if behind_rust else direct + scope: Final[HTTPScope] = { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": "/", + "raw_path": b"/", + "query_string": b"", + "root_path": "", + "headers": headers, + "client": client, + "server": ("127.0.0.1", 4000), + } + await app(scope, _receive, _send) + return seen.result() + + +PEER: Final = ("203.0.113.7", 51234) +TRUSTED_PEER: Final = ("10.0.12.34", 40000) +FORWARDED: Final[Headers] = ( + (b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"), + (b"x-forwarded-proto", b"https"), +) + + +@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)]) +@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))]) +@pytest.mark.parametrize( + "mode", + [ + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + Supervised(17, "0.0.0.0", True, "10.0.0.0/8"), + Supervised(17, "0.0.0.0", True, "*"), + Supervised(17, "0.0.0.0", False, "10.0.0.0/8"), + ], +) +async def test_python_behind_rust_sees_what_a_direct_connection_would( + peer: tuple[str, int], headers: Headers, mode: Supervised +) -> None: + rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}" + behind_rust: Final = await _seen_by_app( + True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode + ) + direct: Final = await _seen_by_app(False, peer, headers, mode) + assert behind_rust == direct + + +async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None: + client, scheme, _ = await _seen_by_app( + True, + ("127.0.0.1", 60000), + ((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")), + Supervised(17, "0.0.0.0", True, "127.0.0.1"), + ) + assert (client, scheme) == (("203.0.113.7", 51234), "http") + + +async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None: + client, _, headers = await _seen_by_app( + True, + ("192.0.2.9", 1000), + ((b"x-litellm-rust-peer", b"10.0.12.34:1"),), + Supervised(17, "0.0.0.0", False, "127.0.0.1"), + ) + assert (client, headers) == (("192.0.2.9", 1000), ())