diff --git a/gateway/launch.py b/gateway/launch.py index 59f7121e513..d5d798f262b 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -19,9 +19,11 @@ Run with: import os import sys +import sysconfig from collections.abc import Callable, Mapping, Sequence from typing import TYPE_CHECKING, Final +from typing_extensions import assert_never from uvicorn.main import main as uvicorn_main from litellm_cli.main import without_gc_pauses @@ -31,17 +33,29 @@ if TYPE_CHECKING: from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings + from litellm.proxy.rust_gateway import Supervised GATEWAY_APP: Final = "gateway.launch:worker_app" KEEPALIVE_FLAG: Final = "--timeout-keep-alive" -def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str, ...]: +def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str], supervised: "Supervised | None") -> tuple[str, ...]: """Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly.""" keepalive: Final = environ.get("KEEPALIVE_TIMEOUT") + inherited_socket: Final = ( + ( + "--fd", + str(supervised.listener_fd), + "--proxy-headers", + "--forwarded-allow-ips", + ",".join(supervised.trusted_proxies), + ) + if supervised is not None + else () + ) if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv): - return (GATEWAY_APP, "--factory", *argv) - return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive) + return (GATEWAY_APP, "--factory", *argv, *inherited_socket) + return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive, *inherited_socket) def pool_database_url( @@ -75,7 +89,34 @@ def _serve(argv: Sequence[str]) -> None: uvicorn_main(tuple(argv), prog_name="uvicorn") -def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: +def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, int, Mapping[str, str]]: + """Validate the uvicorn flags for Rust forwarding and return the binary, public address and child environment.""" + from pydantic import TypeAdapter + + from litellm.proxy.rust_gateway import TRUSTED_PROXIES, startup_error + + params: Final[Mapping[str, object]] = uvicorn_main.make_context( + "uvicorn", list(uvicorn_argv(argv, os.environ, None)) + ).params + error: Final = startup_error( + binary, + { + "--ssl-keyfile": params["ssl_keyfile"], + "--ssl-certfile": params["ssl_certfile"], + "--ssl-ciphers": any(arg.split("=")[0] == "--ssl-ciphers" for arg in argv), + "--uds": params["uds"] is not None, + "--fd": params["fd"] is not None, + }, + ) + if error is not None: + sys.exit(error) + assert binary is not None + host, port = TypeAdapter(tuple[str, int]).validate_python((params["host"], params["port"])) + operator_trust: Final = TypeAdapter(str | None).validate_python(params["forwarded_allow_ips"]) + return binary, host, port, {} if operator_trust is None else {TRUSTED_PROXIES: operator_trust} + + +def configure_database() -> None: from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url @@ -86,7 +127,32 @@ def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) - sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}") if pooled_url is not None: export_pooled_database_url(pooled_url) - serve(uvicorn_argv(argv, os.environ)) + + +def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: + from litellm.proxy.rust_gateway import ( + Disabled, + Supervised, + Supervisor, + gateway_binary, + rust_gateway_mode, + supervise, + ) + + mode: Final = rust_gateway_mode(os.environ) + match mode: + case Disabled(): + configure_database() + serve(uvicorn_argv(argv, os.environ, None)) + case Supervisor(): + binary, host, port, trust = public_listener(argv, gateway_binary(sysconfig.get_path("scripts"))) + configure_database() + command: Final = (sys.executable, "-m", "gateway.launch", *argv) + sys.exit(supervise(binary, command, host, port, {**os.environ, **trust})) + case Supervised(): + serve(uvicorn_argv(argv, os.environ, mode)) + case _: + assert_never(mode) if __name__ == "__main__": diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index bbcb20cfa8c..729c0b847e3 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -107,3 +107,74 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( finally: assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text() assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text() + + +@pytest.mark.parametrize( + "server_arguments", + [("--no-proxy-headers",), ("--forwarded-allow-ips", "127.0.0.1")], +) +def test_gateway_launcher_private_hop_preserves_client_address( + tmp_path: Path, server_arguments: tuple[str, ...] +) -> None: + config: Final = tmp_path / "config.yaml" + config.write_text( + json.dumps({"model_list": [], "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY"}}) + ) + public_port: Final = allocate_port() + environment: Final = { + **{key: value for key, value in os.environ.items() if not key.startswith("DATABASE_") and key != "DIRECT_URL"}, + "LITELLM_MASTER_KEY": "sk-rust-forwarding-integration-key-not-a-secret", + "LITELLM_RUST_GATEWAY": "true", + "LITELLM_PGBOUNCER_ENABLED": "false", + "CONFIG_FILE_PATH": str(config), + "STORE_MODEL_IN_DB": "False", + "PYTHON_DOTENV_DISABLED": "1", + } + log: Final = tmp_path / "gateway.log" + with log.open("wb") as output: + with subprocess.Popen( + ( + sys.executable, + "-I", + "-m", + "gateway.launch", + "--host", + "0.0.0.0", + "--port", + str(public_port), + "--workers", + "2", + "--timeout-worker-healthcheck", + "120", + *server_arguments, + ), + env=environment, + stdout=output, + stderr=output, + ) as supervisor: + try: + with httpx.Client( + base_url=f"http://127.0.0.1:{public_port}", + transport=httpx.HTTPTransport(local_address="127.0.0.2"), + trust_env=False, + timeout=5, + ) as external: + assert eventually(lambda: ready(external), bool, seconds=180), log.read_text() + response: Final = external.get( + "/health/liveliness?client-ip-proof", headers={"X-Forwarded-For": "198.51.100.10"} + ) + assert response.status_code == 200, response.text + events: Final = eventually( + lambda: tuple( + line + for line in log.read_text().splitlines() + if "GET /health/liveliness?client-ip-proof " in line + ), + bool, + seconds=10, + ) + assert len(events) == 1, log.read_text() + assert "127.0.0.2:" in events[0], events + finally: + assert stop_root_process(supervisor), log.read_text() + assert supervisor.returncode == 0, log.read_text() diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index d682fcbad2d..606e760ab3f 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -15,10 +15,11 @@ import pytest from uvicorn.importer import import_from_string from uvicorn.main import main as uvicorn_main -from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv +from gateway.launch import GATEWAY_APP, main, pool_database_url, public_listener, uvicorn_argv from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings from litellm.proxy.proxy_server import app as proxy_app +from litellm.proxy.rust_gateway import Supervised DB_ENV: Final = { "DATABASE_HOST": "db.internal", @@ -114,17 +115,65 @@ def _uvicorn_params(argv: tuple[str, ...]) -> dict[str, object]: class TestUvicornArgv: + @pytest.mark.parametrize( + "arguments", [("--uds", "/tmp/gateway.sock"), ("--fd", "0"), ("--fd=17",), ("--ssl-keyfile", "unused")] + ) + def test_forwarding_rejects_unsupported_listeners_before_starting_processes( + self, arguments: tuple[str, ...] + ) -> None: + with pytest.raises(SystemExit, match=f"does not support {arguments[0].split('=')[0]}"): + public_listener(arguments, "gateway") + + @pytest.mark.parametrize( + ("argv", "trust"), + [ + ((), {}), + (("--forwarded-allow-ips", "10.0.0.5"), {"FORWARDED_ALLOW_IPS": "10.0.0.5"}), + (("--forwarded-allow-ips=*",), {"FORWARDED_ALLOW_IPS": "*"}), + ], + ) + def test_forwarding_carries_operator_proxy_trust_to_the_child( + self, argv: tuple[str, ...], trust: dict[str, str] + ) -> None: + assert public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") == ( + "gateway", + "0.0.0.0", + 4000, + trust, + ) + + def test_supervised_listener_uses_the_inherited_loopback_socket(self): + supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1",)) + params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised)) + assert params["fd"] == 17 + assert params["workers"] == 2 + + @pytest.mark.parametrize( + "argv", + [(), ("--no-proxy-headers",), ("--forwarded-allow-ips", "192.0.2.1"), ("--forwarded-allow-ips=*",)], + ) + def test_private_hop_always_processes_headers_with_the_resolved_trust(self, argv: tuple[str, ...]): + supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5")) + params: Final = _uvicorn_params(uvicorn_argv(argv, {}, supervised)) + assert params["proxy_headers"] is True + assert params["forwarded_allow_ips"] == "127.0.0.1,10.0.0.5" + + def test_direct_listener_keeps_explicit_proxy_header_options(self): + params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {}, None)) + assert params["proxy_headers"] is False + assert params["forwarded_allow_ips"] == "*" + def test_keepalive_env_reaches_uvicorn(self): - params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"})) + params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"}, None)) assert params["app"] == GATEWAY_APP assert params["workers"] == 4 assert params["timeout_keep_alive"] == 75 def test_unset_env_keeps_the_uvicorn_default(self): - assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}))["timeout_keep_alive"] == 5 + assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}, None))["timeout_keep_alive"] == 5 def test_an_explicit_flag_wins_over_the_env(self): - argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}) + argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}, None) assert _uvicorn_params(argv)["timeout_keep_alive"] == 30 def test_the_app_uvicorn_is_told_to_serve_is_the_trimmed_gateway(self, monkeypatch: pytest.MonkeyPatch) -> None: @@ -141,7 +190,7 @@ class TestUvicornArgv: ): monkeypatch.delenv(key, raising=False) - params: Final = _uvicorn_params(uvicorn_argv((), {})) + params: Final = _uvicorn_params(uvicorn_argv((), {}, None)) assert params["factory"] is True try: served: Final = import_from_string(cast(str, params["app"]))() @@ -229,6 +278,16 @@ class TestMain: DatabaseURLSettings.from_env().apply_to_env() assert os.environ["DATABASE_URL"] == pooled + def test_forwarding_disabled_keeps_uvicorn_socket_options( + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LITELLM_RUST_GATEWAY", "false") + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False) + served: Final[list[tuple[str, ...]]] = [] + main(("--uds", "/tmp/gateway.sock"), serve=lambda argv: served.append(tuple(argv))) + assert _uvicorn_params(served[0])["uds"] == "/tmp/gateway.sock" + def test_a_pooler_that_cannot_start_stops_the_gateway_before_uvicorn( self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path ):