diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 94878bd1af2..203358cba97 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -819,7 +819,7 @@ class ProxyInitializationHelpers: _run_ollama_serve = run_ollama_serve @staticmethod - def is_port_in_use(port): + def is_port_in_use(port: int): import socket with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: @@ -1221,8 +1221,8 @@ class ProxyInitializationHelpers: ) def run_server( cli_args, - host, - port, + host: str, + port: int, api_base, api_version, model, @@ -1252,7 +1252,7 @@ def run_server( setup, version, run_gunicorn, - rust_gateway, + rust_gateway: bool, run_hypercorn, run_granian, ssl_keyfile_path, @@ -1298,7 +1298,7 @@ def run_server( from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise binary: Final = gateway_binary() - error: Final = startup_error( + gateway_error: Final = startup_error( binary, { "--ssl_keyfile_path": ssl_keyfile_path, @@ -1308,10 +1308,13 @@ def run_server( "--run_granian": run_granian, }, ) - if error is not None: - raise click.ClickException(error) + if gateway_error is not None: + raise click.ClickException(gateway_error) assert binary is not None - raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, port)) + public_port: Final = ( + random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port + ) + raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port)) args: Final = locals() try: @@ -1761,15 +1764,17 @@ def run_server( if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None: from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process + from litellm.proxy.rust_gateway import metrics_host + metrics_bind_host: Final = metrics_host(host) try: metrics_process: Final = start_metrics_server_process( - host=host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir + host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir ) except MetricsServerStartupError as error: raise click.ClickException(str(error)) from error print_startup_message( - f"\033[1;32mLiteLLM: Serving Prometheus metrics on {host}:{prometheus_metrics_port}/metrics " + f"\033[1;32mLiteLLM: Serving Prometheus metrics on {metrics_bind_host}:{prometheus_metrics_port}/metrics " f"(pid {metrics_process.pid})\033[0m" ) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 830a4ab3d3d..9b7ba98c161 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -38,6 +38,10 @@ def inherited_fd() -> int | None: return int(value) if value is not None else None +def metrics_host(host: str) -> str: + return os.environ.get("_LITELLM_GATEWAY_METRICS_HOST", host) if inherited_fd() is not None else host + + def terminate_group(process: asyncio.subprocess.Process) -> None: try: os.killpg(process.pid, signal.SIGTERM) @@ -65,6 +69,7 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int) **os.environ, "LITELLM_RUST_GATEWAY": "false", GATEWAY_FD: str(listener.fileno()), + "_LITELLM_GATEWAY_METRICS_HOST": host, "FORWARDED_ALLOW_IPS": "127.0.0.1", }, pass_fds=(listener.fileno(),), diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py new file mode 100644 index 00000000000..bb0119f9398 --- /dev/null +++ b/tests/integration/configuration/test_rust_gateway.py @@ -0,0 +1,89 @@ +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Final + +import httpx +import pytest + +from tests.integration._support.client import Gateway, eventually +from tests.integration._support.ports import allocate_port +from tests.integration._support.process import stop_root_process + + +def ready(client: httpx.Client) -> bool: + try: + return client.get("/health/liveliness").status_code == 200 + except httpx.TransportError: + return False + + +@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) +def test_gateway_preserves_metrics_bind_and_mcp_client_ip( + gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...] +) -> None: + config: Final = tmp_path / "config.yaml" + config.write_text( + json.dumps( + { + "model_list": [], + "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY", "use_x_forwarded_for": False}, + } + ) + ) + metrics_port: Final = allocate_port() + public_port: Final = allocate_port() + environment: Final = { + **{key: value for key, value in os.environ.items() if key not in ("DATABASE_URL", "DATABASE_URL_READ_REPLICA")}, + "LITELLM_MASTER_KEY": gateway.key, + "STORE_MODEL_IN_DB": "False", + "PYTHON_DOTENV_DISABLED": "1", + } + with (tmp_path / "proxy.log").open("wb") as output: + with subprocess.Popen( + ( + sys.executable, + "-I", + "-m", + "litellm.proxy.proxy_cli", + "--rust_gateway", + "--config", + str(config), + "--host", + "0.0.0.0", + "--port", + str(public_port), + "--num_workers", + "2", + "--timeout_worker_healthcheck", + "120", + "--prometheus_metrics_port", + str(metrics_port), + *server_arguments, + ), + env=environment, + stdout=output, + stderr=output, + ) as supervisor: + try: + with httpx.Client( + base_url=f"http://127.0.0.1:{public_port}", + transport=httpx.HTTPTransport(local_address="127.0.0.2"), + trust_env=False, + follow_redirects=True, + timeout=5, + ) as external: + assert eventually(lambda: ready(external), bool, seconds=180), (tmp_path / "proxy.log").read_text() + response: Final = external.get(f"http://127.0.0.2:{metrics_port}/metrics") + assert response.status_code == 200, response.text + client_ip: Final = external.get( + "/v1/mcp/network/client-ip", + headers={"Authorization": f"Bearer {gateway.key}", "X-Forwarded-For": "1.2.3.4"}, + ) + assert client_ip.status_code == 200, client_ip.text + assert client_ip.json() == {"ip": "127.0.0.2"} + finally: + assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text() + assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text() diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index c9992d07d14..c1d03d696b9 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -6,7 +6,14 @@ import pytest from click.testing import CliRunner from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server -from litellm.proxy.rust_gateway import child_argv, cli_command, gateway_binary, inherited_fd, startup_error +from litellm.proxy.rust_gateway import ( + child_argv, + cli_command, + gateway_binary, + inherited_fd, + metrics_host, + startup_error, +) @pytest.mark.parametrize( @@ -41,6 +48,19 @@ def test_missing_binary_gives_install_instruction() -> None: ) +@pytest.mark.parametrize("supervised", [False, True]) +@pytest.mark.parametrize("host", ["0.0.0.0", "192.0.2.1", "::"]) +def test_metrics_keep_configured_host_separate_from_private_listener( + supervised: bool, host: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", host) + if supervised: + monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17") + else: + monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) + assert metrics_host("127.0.0.1") == (host if supervised else "127.0.0.1") + + def test_current_environment_binary_wins_over_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: binary: Final = tmp_path / "litellm-gateway" binary.write_text("#!/bin/sh\nexit 0\n")