feat(proxy): supervise the Rust forwarding server from gateway.launch

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-11-03 08:50:07 +09:00
parent 3442588fa0
commit 3399254dd1
3 changed files with 206 additions and 10 deletions

View file

@ -19,9 +19,11 @@ Run with:
import os
import sys
import sysconfig
from collections.abc import Callable, Mapping, Sequence
from typing import TYPE_CHECKING, Final
from typing_extensions import assert_never
from uvicorn.main import main as uvicorn_main
from litellm_cli.main import without_gc_pauses
@ -31,17 +33,29 @@ if TYPE_CHECKING:
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings
from litellm.proxy.rust_gateway import Supervised
GATEWAY_APP: Final = "gateway.launch:worker_app"
KEEPALIVE_FLAG: Final = "--timeout-keep-alive"
def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str, ...]:
def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str], supervised: "Supervised | None") -> tuple[str, ...]:
"""Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly."""
keepalive: Final = environ.get("KEEPALIVE_TIMEOUT")
inherited_socket: Final = (
(
"--fd",
str(supervised.listener_fd),
"--proxy-headers",
"--forwarded-allow-ips",
",".join(supervised.trusted_proxies),
)
if supervised is not None
else ()
)
if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv):
return (GATEWAY_APP, "--factory", *argv)
return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive)
return (GATEWAY_APP, "--factory", *argv, *inherited_socket)
return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive, *inherited_socket)
def pool_database_url(
@ -75,7 +89,34 @@ def _serve(argv: Sequence[str]) -> None:
uvicorn_main(tuple(argv), prog_name="uvicorn")
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, int, Mapping[str, str]]:
"""Validate the uvicorn flags for Rust forwarding and return the binary, public address and child environment."""
from pydantic import TypeAdapter
from litellm.proxy.rust_gateway import TRUSTED_PROXIES, startup_error
params: Final[Mapping[str, object]] = uvicorn_main.make_context(
"uvicorn", list(uvicorn_argv(argv, os.environ, None))
).params
error: Final = startup_error(
binary,
{
"--ssl-keyfile": params["ssl_keyfile"],
"--ssl-certfile": params["ssl_certfile"],
"--ssl-ciphers": any(arg.split("=")[0] == "--ssl-ciphers" for arg in argv),
"--uds": params["uds"] is not None,
"--fd": params["fd"] is not None,
},
)
if error is not None:
sys.exit(error)
assert binary is not None
host, port = TypeAdapter(tuple[str, int]).validate_python((params["host"], params["port"]))
operator_trust: Final = TypeAdapter(str | None).validate_python(params["forwarded_allow_ips"])
return binary, host, port, {} if operator_trust is None else {TRUSTED_PROXIES: operator_trust}
def configure_database() -> None:
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url
@ -86,7 +127,32 @@ def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -
sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}")
if pooled_url is not None:
export_pooled_database_url(pooled_url)
serve(uvicorn_argv(argv, os.environ))
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
from litellm.proxy.rust_gateway import (
Disabled,
Supervised,
Supervisor,
gateway_binary,
rust_gateway_mode,
supervise,
)
mode: Final = rust_gateway_mode(os.environ)
match mode:
case Disabled():
configure_database()
serve(uvicorn_argv(argv, os.environ, None))
case Supervisor():
binary, host, port, trust = public_listener(argv, gateway_binary(sysconfig.get_path("scripts")))
configure_database()
command: Final = (sys.executable, "-m", "gateway.launch", *argv)
sys.exit(supervise(binary, command, host, port, {**os.environ, **trust}))
case Supervised():
serve(uvicorn_argv(argv, os.environ, mode))
case _:
assert_never(mode)
if __name__ == "__main__":

View file

@ -107,3 +107,74 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip(
finally:
assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text()
assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text()
@pytest.mark.parametrize(
"server_arguments",
[("--no-proxy-headers",), ("--forwarded-allow-ips", "127.0.0.1")],
)
def test_gateway_launcher_private_hop_preserves_client_address(
tmp_path: Path, server_arguments: tuple[str, ...]
) -> None:
config: Final = tmp_path / "config.yaml"
config.write_text(
json.dumps({"model_list": [], "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY"}})
)
public_port: Final = allocate_port()
environment: Final = {
**{key: value for key, value in os.environ.items() if not key.startswith("DATABASE_") and key != "DIRECT_URL"},
"LITELLM_MASTER_KEY": "sk-rust-forwarding-integration-key-not-a-secret",
"LITELLM_RUST_GATEWAY": "true",
"LITELLM_PGBOUNCER_ENABLED": "false",
"CONFIG_FILE_PATH": str(config),
"STORE_MODEL_IN_DB": "False",
"PYTHON_DOTENV_DISABLED": "1",
}
log: Final = tmp_path / "gateway.log"
with log.open("wb") as output:
with subprocess.Popen(
(
sys.executable,
"-I",
"-m",
"gateway.launch",
"--host",
"0.0.0.0",
"--port",
str(public_port),
"--workers",
"2",
"--timeout-worker-healthcheck",
"120",
*server_arguments,
),
env=environment,
stdout=output,
stderr=output,
) as supervisor:
try:
with httpx.Client(
base_url=f"http://127.0.0.1:{public_port}",
transport=httpx.HTTPTransport(local_address="127.0.0.2"),
trust_env=False,
timeout=5,
) as external:
assert eventually(lambda: ready(external), bool, seconds=180), log.read_text()
response: Final = external.get(
"/health/liveliness?client-ip-proof", headers={"X-Forwarded-For": "198.51.100.10"}
)
assert response.status_code == 200, response.text
events: Final = eventually(
lambda: tuple(
line
for line in log.read_text().splitlines()
if "GET /health/liveliness?client-ip-proof " in line
),
bool,
seconds=10,
)
assert len(events) == 1, log.read_text()
assert "127.0.0.2:" in events[0], events
finally:
assert stop_root_process(supervisor), log.read_text()
assert supervisor.returncode == 0, log.read_text()

View file

@ -15,10 +15,11 @@ import pytest
from uvicorn.importer import import_from_string
from uvicorn.main import main as uvicorn_main
from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv
from gateway.launch import GATEWAY_APP, main, pool_database_url, public_listener, uvicorn_argv
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings
from litellm.proxy.proxy_server import app as proxy_app
from litellm.proxy.rust_gateway import Supervised
DB_ENV: Final = {
"DATABASE_HOST": "db.internal",
@ -114,17 +115,65 @@ def _uvicorn_params(argv: tuple[str, ...]) -> dict[str, object]:
class TestUvicornArgv:
@pytest.mark.parametrize(
"arguments", [("--uds", "/tmp/gateway.sock"), ("--fd", "0"), ("--fd=17",), ("--ssl-keyfile", "unused")]
)
def test_forwarding_rejects_unsupported_listeners_before_starting_processes(
self, arguments: tuple[str, ...]
) -> None:
with pytest.raises(SystemExit, match=f"does not support {arguments[0].split('=')[0]}"):
public_listener(arguments, "gateway")
@pytest.mark.parametrize(
("argv", "trust"),
[
((), {}),
(("--forwarded-allow-ips", "10.0.0.5"), {"FORWARDED_ALLOW_IPS": "10.0.0.5"}),
(("--forwarded-allow-ips=*",), {"FORWARDED_ALLOW_IPS": "*"}),
],
)
def test_forwarding_carries_operator_proxy_trust_to_the_child(
self, argv: tuple[str, ...], trust: dict[str, str]
) -> None:
assert public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") == (
"gateway",
"0.0.0.0",
4000,
trust,
)
def test_supervised_listener_uses_the_inherited_loopback_socket(self):
supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1",))
params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised))
assert params["fd"] == 17
assert params["workers"] == 2
@pytest.mark.parametrize(
"argv",
[(), ("--no-proxy-headers",), ("--forwarded-allow-ips", "192.0.2.1"), ("--forwarded-allow-ips=*",)],
)
def test_private_hop_always_processes_headers_with_the_resolved_trust(self, argv: tuple[str, ...]):
supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5"))
params: Final = _uvicorn_params(uvicorn_argv(argv, {}, supervised))
assert params["proxy_headers"] is True
assert params["forwarded_allow_ips"] == "127.0.0.1,10.0.0.5"
def test_direct_listener_keeps_explicit_proxy_header_options(self):
params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {}, None))
assert params["proxy_headers"] is False
assert params["forwarded_allow_ips"] == "*"
def test_keepalive_env_reaches_uvicorn(self):
params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"}))
params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"}, None))
assert params["app"] == GATEWAY_APP
assert params["workers"] == 4
assert params["timeout_keep_alive"] == 75
def test_unset_env_keeps_the_uvicorn_default(self):
assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}))["timeout_keep_alive"] == 5
assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}, None))["timeout_keep_alive"] == 5
def test_an_explicit_flag_wins_over_the_env(self):
argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"})
argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}, None)
assert _uvicorn_params(argv)["timeout_keep_alive"] == 30
def test_the_app_uvicorn_is_told_to_serve_is_the_trimmed_gateway(self, monkeypatch: pytest.MonkeyPatch) -> None:
@ -141,7 +190,7 @@ class TestUvicornArgv:
):
monkeypatch.delenv(key, raising=False)
params: Final = _uvicorn_params(uvicorn_argv((), {}))
params: Final = _uvicorn_params(uvicorn_argv((), {}, None))
assert params["factory"] is True
try:
served: Final = import_from_string(cast(str, params["app"]))()
@ -229,6 +278,16 @@ class TestMain:
DatabaseURLSettings.from_env().apply_to_env()
assert os.environ["DATABASE_URL"] == pooled
def test_forwarding_disabled_keeps_uvicorn_socket_options(
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("LITELLM_RUST_GATEWAY", "false")
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False)
served: Final[list[tuple[str, ...]]] = []
main(("--uds", "/tmp/gateway.sock"), serve=lambda argv: served.append(tuple(argv)))
assert _uvicorn_params(served[0])["uds"] == "/tmp/gateway.sock"
def test_a_pooler_that_cannot_start_stops_the_gateway_before_uvicorn(
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):