fix(proxy): preserve gateway metrics and default-port fallback

This commit is contained in:
Devin AI 2026-11-01 23:39:14 +00:00 • committed by Yujong Lee
parent 608a5c3e57
commit c0ec348583
4 changed files with 130 additions and 11 deletions

View file

@ -819,7 +819,7 @@ class ProxyInitializationHelpers:
_run_ollama_serve = run_ollama_serve
@staticmethod
def is_port_in_use(port):
def is_port_in_use(port: int):
import socket
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
@ -1221,8 +1221,8 @@ class ProxyInitializationHelpers:
)
def run_server(
cli_args,
host,
port,
host: str,
port: int,
api_base,
api_version,
model,
@ -1252,7 +1252,7 @@ def run_server(
setup,
version,
run_gunicorn,
rust_gateway,
rust_gateway: bool,
run_hypercorn,
run_granian,
ssl_keyfile_path,
@ -1298,7 +1298,7 @@ def run_server(
from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise
binary: Final = gateway_binary()
error: Final = startup_error(
gateway_error: Final = startup_error(
binary,
{
"--ssl_keyfile_path": ssl_keyfile_path,
@ -1308,10 +1308,13 @@ def run_server(
"--run_granian": run_granian,
},
)
if error is not None:
raise click.ClickException(error)
if gateway_error is not None:
raise click.ClickException(gateway_error)
assert binary is not None
raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, port))
public_port: Final = (
random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port
)
raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port))
args: Final = locals()
try:
@ -1761,15 +1764,17 @@ def run_server(
if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None:
from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process
from litellm.proxy.rust_gateway import metrics_host
metrics_bind_host: Final = metrics_host(host)
try:
metrics_process: Final = start_metrics_server_process(
host=host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir
host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir
)
except MetricsServerStartupError as error:
raise click.ClickException(str(error)) from error
print_startup_message(
f"\033[1;32mLiteLLM: Serving Prometheus metrics on {host}:{prometheus_metrics_port}/metrics "
f"\033[1;32mLiteLLM: Serving Prometheus metrics on {metrics_bind_host}:{prometheus_metrics_port}/metrics "
f"(pid {metrics_process.pid})\033[0m"
)

View file

@ -38,6 +38,10 @@ def inherited_fd() -> int | None:
return int(value) if value is not None else None
def metrics_host(host: str) -> str:
return os.environ.get("_LITELLM_GATEWAY_METRICS_HOST", host) if inherited_fd() is not None else host
def terminate_group(process: asyncio.subprocess.Process) -> None:
try:
os.killpg(process.pid, signal.SIGTERM)
@ -65,6 +69,7 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int)
**os.environ,
"LITELLM_RUST_GATEWAY": "false",
GATEWAY_FD: str(listener.fileno()),
"_LITELLM_GATEWAY_METRICS_HOST": host,
"FORWARDED_ALLOW_IPS": "127.0.0.1",
},
pass_fds=(listener.fileno(),),

View file

@ -0,0 +1,89 @@
import json
import os
import subprocess
import sys
from pathlib import Path
from typing import Final
import httpx
import pytest
from tests.integration._support.client import Gateway, eventually
from tests.integration._support.ports import allocate_port
from tests.integration._support.process import stop_root_process
def ready(client: httpx.Client) -> bool:
try:
return client.get("/health/liveliness").status_code == 200
except httpx.TransportError:
return False
@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)])
def test_gateway_preserves_metrics_bind_and_mcp_client_ip(
gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...]
) -> None:
config: Final = tmp_path / "config.yaml"
config.write_text(
json.dumps(
{
"model_list": [],
"general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY", "use_x_forwarded_for": False},
}
)
)
metrics_port: Final = allocate_port()
public_port: Final = allocate_port()
environment: Final = {
**{key: value for key, value in os.environ.items() if key not in ("DATABASE_URL", "DATABASE_URL_READ_REPLICA")},
"LITELLM_MASTER_KEY": gateway.key,
"STORE_MODEL_IN_DB": "False",
"PYTHON_DOTENV_DISABLED": "1",
}
with (tmp_path / "proxy.log").open("wb") as output:
with subprocess.Popen(
(
sys.executable,
"-I",
"-m",
"litellm.proxy.proxy_cli",
"--rust_gateway",
"--config",
str(config),
"--host",
"0.0.0.0",
"--port",
str(public_port),
"--num_workers",
"2",
"--timeout_worker_healthcheck",
"120",
"--prometheus_metrics_port",
str(metrics_port),
*server_arguments,
),
env=environment,
stdout=output,
stderr=output,
) as supervisor:
try:
with httpx.Client(
base_url=f"http://127.0.0.1:{public_port}",
transport=httpx.HTTPTransport(local_address="127.0.0.2"),
trust_env=False,
follow_redirects=True,
timeout=5,
) as external:
assert eventually(lambda: ready(external), bool, seconds=180), (tmp_path / "proxy.log").read_text()
response: Final = external.get(f"http://127.0.0.2:{metrics_port}/metrics")
assert response.status_code == 200, response.text
client_ip: Final = external.get(
"/v1/mcp/network/client-ip",
headers={"Authorization": f"Bearer {gateway.key}", "X-Forwarded-For": "1.2.3.4"},
)
assert client_ip.status_code == 200, client_ip.text
assert client_ip.json() == {"ip": "127.0.0.2"}
finally:
assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text()
assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text()

View file

@ -6,7 +6,14 @@ import pytest
from click.testing import CliRunner
from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server
from litellm.proxy.rust_gateway import child_argv, cli_command, gateway_binary, inherited_fd, startup_error
from litellm.proxy.rust_gateway import (
child_argv,
cli_command,
gateway_binary,
inherited_fd,
metrics_host,
startup_error,
)
@pytest.mark.parametrize(
@ -41,6 +48,19 @@ def test_missing_binary_gives_install_instruction() -> None:
)
@pytest.mark.parametrize("supervised", [False, True])
@pytest.mark.parametrize("host", ["0.0.0.0", "192.0.2.1", "::"])
def test_metrics_keep_configured_host_separate_from_private_listener(
supervised: bool, host: str, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", host)
if supervised:
monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17")
else:
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
assert metrics_host("127.0.0.1") == (host if supervised else "127.0.0.1")
def test_current_environment_binary_wins_over_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
binary: Final = tmp_path / "litellm-gateway"
binary.write_text("#!/bin/sh\nexit 0\n")