fix(proxy): restore the client's peer behind Rust instead of trusting the Rust hop

Python now sees each request as if the client had connected to it directly.
The Rust hop sends the real TCP peer in x-litellm-rust-peer, an app middleware
puts it back into the ASGI scope, and uvicorn's ProxyHeadersMiddleware then
runs with the same trust settings Python alone would use. Server-level proxy
headers are off in supervised mode for uvicorn and gunicorn, since they would
judge the loopback hop. The trust-list merge that always trusted 127.0.0.1 is
gone, so operators keep the exact FORWARDED_ALLOW_IPS behavior they had.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-11-03 09:07:08 +09:00
parent 5be0eb7666
commit 543c015305
8 changed files with 206 additions and 53 deletions

View file

@ -46,9 +46,7 @@ def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str], supervised: "S
(
"--fd",
str(supervised.listener_fd),
"--proxy-headers",
"--forwarded-allow-ips",
",".join(supervised.trusted_proxies),
"--no-proxy-headers",
)
if supervised is not None
else ()
@ -93,7 +91,7 @@ def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str,
"""Validate the uvicorn flags for Rust forwarding and return the binary, public address and child environment."""
from pydantic import TypeAdapter
from litellm.proxy.rust_gateway import TRUSTED_PROXIES, startup_error
from litellm.proxy.rust_gateway import PROXY_HEADERS, TRUSTED_PROXIES, startup_error
params: Final[Mapping[str, object]] = uvicorn_main.make_context(
"uvicorn", list(uvicorn_argv(argv, os.environ, None))
@ -113,7 +111,14 @@ def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str,
assert binary is not None
host, port = TypeAdapter(tuple[str, int]).validate_python((params["host"], params["port"]))
operator_trust: Final = TypeAdapter(str | None).validate_python(params["forwarded_allow_ips"])
return binary, host, port, {} if operator_trust is None else {TRUSTED_PROXIES: operator_trust}
proxy_headers: Final = TypeAdapter(bool).validate_python(params["proxy_headers"])
return (
binary,
host,
port,
{PROXY_HEADERS: str(proxy_headers).lower()}
| ({} if operator_trust is None else {TRUSTED_PROXIES: operator_trust}),
)
def configure_database() -> None:

View file

@ -429,7 +429,7 @@ class ProxyInitializationHelpers:
"port": port,
"server_header": False,
"timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(),
**({"fd": listener_fd} if listener_fd is not None else {}),
**({"fd": listener_fd, "proxy_headers": False} if listener_fd is not None else {}),
}
if log_config is not None:
print_startup_message(f"Using log_config: {log_config}")
@ -758,7 +758,9 @@ class ProxyInitializationHelpers:
gunicorn_options: Final = {
"bind": f"fd://{listener_fd}" if listener_fd is not None else f"{host}:{port}",
"workers": num_workers, # default is 1
"worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout),
"worker_class": build_graceful_shutdown_uvicorn_worker(
graceful_shutdown_timeout, proxy_headers=listener_fd is None
),
"graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS,
"preload": True, # Add the preload flag,
"accesslog": "-", # Log to stdout

View file

@ -835,6 +835,7 @@ from litellm.proxy.plugin_routes import (
from litellm.proxy.plugin_routes import (
router as plugin_router,
)
from litellm.proxy.rust_gateway import RustPeerMiddleware, Supervised, rust_gateway_mode
from litellm.proxy.spend_tracking.spend_event_producer import (
CollectorSettings,
SpendEventProducer,
@ -21924,6 +21925,10 @@ if _server_root_paths:
)
app.add_middleware(PerRequestRootPathMiddleware, root_paths=_server_root_paths)
_rust_gateway_mode: Final = rust_gateway_mode(os.environ)
if isinstance(_rust_gateway_mode, Supervised):
app.add_middleware(RustPeerMiddleware, mode=_rust_gateway_mode)
async def _stream_mcp_asgi_response(handle_fn, scope: dict, receive) -> "StreamingResponse":
"""

View file

@ -9,16 +9,22 @@ from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from enum import Enum, IntEnum
from types import MappingProxyType
from typing import Final, TypeAlias
from typing import TYPE_CHECKING, Final, TypeAlias
from pydantic import TypeAdapter
from typing_extensions import assert_never
if TYPE_CHECKING:
from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope
ENABLE: Final = "LITELLM_RUST_GATEWAY"
GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD"
METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST"
PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS"
TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS"
DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1"
RUST_HOP: Final = "127.0.0.1"
PEER_HEADER: Final = b"x-litellm-rust-peer"
_FD: Final = TypeAdapter(int)
_ENABLED: Final = TypeAdapter(bool)
@ -39,26 +45,62 @@ class Supervisor:
class Supervised:
listener_fd: int
metrics_host: str
trusted_proxies: tuple[str, ...]
proxy_headers: bool
trusted_proxies: str
RustGatewayMode: TypeAlias = Disabled | Supervisor | Supervised
def trusted_proxies(operator_value: str | None) -> tuple[str, ...]:
configured: Final = tuple(host.strip() for host in (operator_value or "").split(",") if host.strip())
return ("*",) if "*" in configured else (RUST_HOP, *(host for host in configured if host != RUST_HOP))
def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode:
fd: Final = environ.get(GATEWAY_FD)
if fd is not None:
return Supervised(_FD.validate_python(fd), environ[METRICS_HOST], trusted_proxies(environ.get(TRUSTED_PROXIES)))
return Supervised(
_FD.validate_python(fd),
environ[METRICS_HOST],
_ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")),
environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES),
)
if _ENABLED.validate_python(environ.get(ENABLE, "false")):
return Supervisor()
return Disabled()
def _restore_peer(scope: "WWWScope") -> None:
headers: Final = tuple(scope["headers"])
peer: Final = next((value for name, value in headers if name == PEER_HEADER), None)
if peer is None:
return
remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER)
scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop
client: Final = scope["client"]
if client is None or client[0] != RUST_HOP:
return
host, _, port = peer.decode("latin-1").rpartition(":")
scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope
class RustPeerMiddleware:
"""Make Python see each request as if the client had connected to it directly.
Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real
TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the
real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust
settings Python would have used on the public port. The server's proxy-header handling is off in
this mode, since it would judge Rust's loopback address instead of the real peer.
"""
def __init__(self, app: "ASGI3Application", mode: Supervised) -> None:
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app
async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None:
if scope["type"] != "lifespan":
_restore_peer(scope)
await self.app(scope, receive, send)
def gateway_binary(scripts_dir: str) -> str | None:
name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway"
return shutil.which(name, path=scripts_dir)
@ -175,7 +217,6 @@ def python_spec(
**environ,
GATEWAY_FD: str(listener_fd),
METRICS_HOST: public_host,
TRUSTED_PROXIES: ",".join(trusted_proxies(environ.get(TRUSTED_PROXIES))),
}
),
pass_fds=(listener_fd,),

View file

@ -1,10 +1,16 @@
from uvicorn.workers import UvicornWorker
def build_graceful_shutdown_uvicorn_worker(timeout_graceful_shutdown: float) -> type[UvicornWorker]:
def build_graceful_shutdown_uvicorn_worker(
timeout_graceful_shutdown: float, *, proxy_headers: bool = True
) -> type[UvicornWorker]:
class GracefulShutdownUvicornWorker(UvicornWorker):
def __init__(self, *args: object) -> None:
self.CONFIG_KWARGS = {**UvicornWorker.CONFIG_KWARGS, "timeout_graceful_shutdown": timeout_graceful_shutdown}
self.CONFIG_KWARGS = {
**UvicornWorker.CONFIG_KWARGS,
"timeout_graceful_shutdown": timeout_graceful_shutdown,
"proxy_headers": proxy_headers,
}
super().__init__(*args)
return GracefulShutdownUvicornWorker

View file

@ -22,7 +22,6 @@ def ready(client: httpx.Client) -> bool:
return False
@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)])
def client_ip(base_url: str, local_address: str, key: str, forwarded_for: str) -> httpx.Response:
with httpx.Client(
base_url=base_url,
@ -36,6 +35,7 @@ def client_ip(base_url: str, local_address: str, key: str, forwarded_for: str) -
)
@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)])
def test_gateway_preserves_metrics_bind_and_mcp_client_ip(
gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...]
) -> None:

View file

@ -127,12 +127,16 @@ class TestUvicornArgv:
@pytest.mark.parametrize(
("argv", "trust"),
[
((), {}),
(("--forwarded-allow-ips", "10.0.0.5"), {"FORWARDED_ALLOW_IPS": "10.0.0.5"}),
(("--forwarded-allow-ips=*",), {"FORWARDED_ALLOW_IPS": "*"}),
((), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true"}),
(
("--forwarded-allow-ips", "10.0.0.5"),
{"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "10.0.0.5"},
),
(("--forwarded-allow-ips=*",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "true", "FORWARDED_ALLOW_IPS": "*"}),
(("--no-proxy-headers",), {"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}),
],
)
def test_forwarding_carries_operator_proxy_trust_to_the_child(
def test_forwarding_carries_operator_proxy_header_options_to_the_child(
self, argv: tuple[str, ...], trust: dict[str, str]
) -> None:
assert public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") == (
@ -143,20 +147,15 @@ class TestUvicornArgv:
)
def test_supervised_listener_uses_the_inherited_loopback_socket(self):
supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1",))
supervised: Final = Supervised(17, "0.0.0.0", True, "127.0.0.1")
params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised))
assert params["fd"] == 17
assert params["workers"] == 2
@pytest.mark.parametrize(
"argv",
[(), ("--no-proxy-headers",), ("--forwarded-allow-ips", "192.0.2.1"), ("--forwarded-allow-ips=*",)],
)
def test_private_hop_always_processes_headers_with_the_resolved_trust(self, argv: tuple[str, ...]):
supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5"))
params: Final = _uvicorn_params(uvicorn_argv(argv, {}, supervised))
assert params["proxy_headers"] is True
assert params["forwarded_allow_ips"] == "127.0.0.1,10.0.0.5"
@pytest.mark.parametrize("argv", [(), ("--proxy-headers",), ("--forwarded-allow-ips=*",)])
def test_private_hop_leaves_proxy_headers_to_the_app(self, argv: tuple[str, ...]):
params: Final = _uvicorn_params(uvicorn_argv(argv, {}, Supervised(17, "0.0.0.0", True, "*")))
assert params["proxy_headers"] is False
def test_direct_listener_keeps_explicit_proxy_header_options(self):
params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {}, None))

View file

@ -1,12 +1,24 @@
import asyncio
import functools
import signal
import sys
from pathlib import Path
from typing import Final
from typing import TYPE_CHECKING, Final
import pytest
from click.testing import CliRunner
from pydantic import ValidationError
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
if TYPE_CHECKING:
from uvicorn._types import (
ASGIReceiveCallable,
ASGISendCallable,
ASGISendEvent,
HTTPDisconnectEvent,
HTTPScope,
Scope,
)
from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server
from litellm.proxy.rust_gateway import (
@ -17,6 +29,7 @@ from litellm.proxy.rust_gateway import (
Exited,
Finished,
Running,
RustPeerMiddleware,
Signal,
State,
Stop,
@ -29,7 +42,6 @@ from litellm.proxy.rust_gateway import (
rust_spec,
startup_error,
step,
trusted_proxies,
)
TERM: Final = signal.SIGTERM
@ -66,7 +78,7 @@ def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch:
"_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0",
"FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5",
},
Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5")),
Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"),
),
],
)
@ -80,27 +92,17 @@ def test_unrecognized_switch_value_is_rejected() -> None:
@pytest.mark.parametrize(
("operator_value", "expected"),
("environ", "expected"),
[
(None, ("127.0.0.1",)),
("", ("127.0.0.1",)),
("10.0.0.5", ("127.0.0.1", "10.0.0.5")),
("127.0.0.1, 10.0.0.5", ("127.0.0.1", "10.0.0.5")),
("10.0.0.5,127.0.0.1", ("127.0.0.1", "10.0.0.5")),
("*", ("*",)),
("10.0.0.5,*", ("*",)),
({}, Supervised(17, "0.0.0.0", True, "127.0.0.1")),
({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, Supervised(17, "0.0.0.0", True, "10.0.0.5")),
({"FORWARDED_ALLOW_IPS": "*"}, Supervised(17, "0.0.0.0", True, "*")),
({"_LITELLM_GATEWAY_PROXY_HEADERS": "false"}, Supervised(17, "0.0.0.0", False, "127.0.0.1")),
],
)
def test_operator_trust_is_kept_and_rust_hop_added(operator_value: str | None, expected: tuple[str, ...]) -> None:
assert trusted_proxies(operator_value) == expected
@pytest.mark.parametrize("operator_value", [None, "", "10.0.0.5", "10.0.0.5, 192.0.2.0/24", "127.0.0.1"])
def test_python_child_always_trusts_the_rust_hop(operator_value: str | None) -> None:
environ: Final = {} if operator_value is None else {"FORWARDED_ALLOW_IPS": operator_value}
def test_python_child_keeps_the_trust_python_alone_would_use(environ: dict[str, str], expected: Supervised) -> None:
spec: Final = python_spec(("python",), 17, 45555, "0.0.0.0", environ)
assert spec.env["FORWARDED_ALLOW_IPS"].split(",")[0] == "127.0.0.1"
assert rust_gateway_mode(spec.env) == Supervised(17, "0.0.0.0", trusted_proxies(operator_value))
assert rust_gateway_mode(spec.env) == expected
def test_python_child_listens_privately_and_keeps_options() -> None:
@ -152,6 +154,7 @@ def test_inherited_socket_reaches_uvicorn(listener_fd: int | None) -> None:
"127.0.0.1", 45555, listener_fd=listener_fd
)
assert options.get("fd") == listener_fd
assert options.get("proxy_headers", True) is (listener_fd is None)
def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]:
@ -200,3 +203,95 @@ def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit(
events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode
) -> None:
assert _fold(events) == (Finished(code), signals)
Headers = tuple[tuple[bytes, bytes], ...]
async def _receive() -> "HTTPDisconnectEvent":
return {"type": "http.disconnect"}
async def _send(message: "ASGISendEvent") -> None:
return None
async def _seen_by_app(
behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised
) -> tuple[tuple[str, int] | None, str, Headers]:
seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = (
asyncio.get_running_loop().create_future()
)
async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None:
assert scope["type"] == "http"
seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"])))
direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe
app: Final = RustPeerMiddleware(probe, mode) if behind_rust else direct
scope: Final[HTTPScope] = {
"type": "http",
"asgi": {"version": "3.0", "spec_version": "2.3"},
"http_version": "1.1",
"method": "GET",
"scheme": "http",
"path": "/",
"raw_path": b"/",
"query_string": b"",
"root_path": "",
"headers": headers,
"client": client,
"server": ("127.0.0.1", 4000),
}
await app(scope, _receive, _send)
return seen.result()
PEER: Final = ("203.0.113.7", 51234)
TRUSTED_PEER: Final = ("10.0.12.34", 40000)
FORWARDED: Final[Headers] = (
(b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"),
(b"x-forwarded-proto", b"https"),
)
@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)])
@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))])
@pytest.mark.parametrize(
"mode",
[
Supervised(17, "0.0.0.0", True, "127.0.0.1"),
Supervised(17, "0.0.0.0", True, "10.0.0.0/8"),
Supervised(17, "0.0.0.0", True, "*"),
Supervised(17, "0.0.0.0", False, "10.0.0.0/8"),
],
)
async def test_python_behind_rust_sees_what_a_direct_connection_would(
peer: tuple[str, int], headers: Headers, mode: Supervised
) -> None:
rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}"
behind_rust: Final = await _seen_by_app(
True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode
)
direct: Final = await _seen_by_app(False, peer, headers, mode)
assert behind_rust == direct
async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None:
client, scheme, _ = await _seen_by_app(
True,
("127.0.0.1", 60000),
((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")),
Supervised(17, "0.0.0.0", True, "127.0.0.1"),
)
assert (client, scheme) == (("203.0.113.7", 51234), "http")
async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None:
client, _, headers = await _seen_by_app(
True,
("192.0.2.9", 1000),
((b"x-litellm-rust-peer", b"10.0.12.34:1"),),
Supervised(17, "0.0.0.0", False, "127.0.0.1"),
)
assert (client, headers) == (("192.0.2.9", 1000), ())