mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(proxy): preserve gateway metrics and default-port fallback
This commit is contained in:
parent
608a5c3e57
commit
c0ec348583
4 changed files with 130 additions and 11 deletions
|
|
@ -819,7 +819,7 @@ class ProxyInitializationHelpers:
|
|||
_run_ollama_serve = run_ollama_serve
|
||||
|
||||
@staticmethod
|
||||
def is_port_in_use(port):
|
||||
def is_port_in_use(port: int):
|
||||
import socket
|
||||
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
|
|
@ -1221,8 +1221,8 @@ class ProxyInitializationHelpers:
|
|||
)
|
||||
def run_server(
|
||||
cli_args,
|
||||
host,
|
||||
port,
|
||||
host: str,
|
||||
port: int,
|
||||
api_base,
|
||||
api_version,
|
||||
model,
|
||||
|
|
@ -1252,7 +1252,7 @@ def run_server(
|
|||
setup,
|
||||
version,
|
||||
run_gunicorn,
|
||||
rust_gateway,
|
||||
rust_gateway: bool,
|
||||
run_hypercorn,
|
||||
run_granian,
|
||||
ssl_keyfile_path,
|
||||
|
|
@ -1298,7 +1298,7 @@ def run_server(
|
|||
from litellm.proxy.rust_gateway import cli_command, gateway_binary, startup_error, supervise
|
||||
|
||||
binary: Final = gateway_binary()
|
||||
error: Final = startup_error(
|
||||
gateway_error: Final = startup_error(
|
||||
binary,
|
||||
{
|
||||
"--ssl_keyfile_path": ssl_keyfile_path,
|
||||
|
|
@ -1308,10 +1308,13 @@ def run_server(
|
|||
"--run_granian": run_granian,
|
||||
},
|
||||
)
|
||||
if error is not None:
|
||||
raise click.ClickException(error)
|
||||
if gateway_error is not None:
|
||||
raise click.ClickException(gateway_error)
|
||||
assert binary is not None
|
||||
raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, port))
|
||||
public_port: Final = (
|
||||
random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port
|
||||
)
|
||||
raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), host, public_port))
|
||||
|
||||
args: Final = locals()
|
||||
try:
|
||||
|
|
@ -1761,15 +1764,17 @@ def run_server(
|
|||
|
||||
if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None:
|
||||
from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process
|
||||
from litellm.proxy.rust_gateway import metrics_host
|
||||
|
||||
metrics_bind_host: Final = metrics_host(host)
|
||||
try:
|
||||
metrics_process: Final = start_metrics_server_process(
|
||||
host=host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir
|
||||
host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir
|
||||
)
|
||||
except MetricsServerStartupError as error:
|
||||
raise click.ClickException(str(error)) from error
|
||||
print_startup_message(
|
||||
f"\033[1;32mLiteLLM: Serving Prometheus metrics on {host}:{prometheus_metrics_port}/metrics "
|
||||
f"\033[1;32mLiteLLM: Serving Prometheus metrics on {metrics_bind_host}:{prometheus_metrics_port}/metrics "
|
||||
f"(pid {metrics_process.pid})\033[0m"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -38,6 +38,10 @@ def inherited_fd() -> int | None:
|
|||
return int(value) if value is not None else None
|
||||
|
||||
|
||||
def metrics_host(host: str) -> str:
|
||||
return os.environ.get("_LITELLM_GATEWAY_METRICS_HOST", host) if inherited_fd() is not None else host
|
||||
|
||||
|
||||
def terminate_group(process: asyncio.subprocess.Process) -> None:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
|
|
@ -65,6 +69,7 @@ async def _supervise(binary: str, command: Sequence[str], host: str, port: int)
|
|||
**os.environ,
|
||||
"LITELLM_RUST_GATEWAY": "false",
|
||||
GATEWAY_FD: str(listener.fileno()),
|
||||
"_LITELLM_GATEWAY_METRICS_HOST": host,
|
||||
"FORWARDED_ALLOW_IPS": "127.0.0.1",
|
||||
},
|
||||
pass_fds=(listener.fileno(),),
|
||||
|
|
|
|||
89
tests/integration/configuration/test_rust_gateway.py
Normal file
89
tests/integration/configuration/test_rust_gateway.py
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests.integration._support.client import Gateway, eventually
|
||||
from tests.integration._support.ports import allocate_port
|
||||
from tests.integration._support.process import stop_root_process
|
||||
|
||||
|
||||
def ready(client: httpx.Client) -> bool:
|
||||
try:
|
||||
return client.get("/health/liveliness").status_code == 200
|
||||
except httpx.TransportError:
|
||||
return False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)])
|
||||
def test_gateway_preserves_metrics_bind_and_mcp_client_ip(
|
||||
gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...]
|
||||
) -> None:
|
||||
config: Final = tmp_path / "config.yaml"
|
||||
config.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_list": [],
|
||||
"general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY", "use_x_forwarded_for": False},
|
||||
}
|
||||
)
|
||||
)
|
||||
metrics_port: Final = allocate_port()
|
||||
public_port: Final = allocate_port()
|
||||
environment: Final = {
|
||||
**{key: value for key, value in os.environ.items() if key not in ("DATABASE_URL", "DATABASE_URL_READ_REPLICA")},
|
||||
"LITELLM_MASTER_KEY": gateway.key,
|
||||
"STORE_MODEL_IN_DB": "False",
|
||||
"PYTHON_DOTENV_DISABLED": "1",
|
||||
}
|
||||
with (tmp_path / "proxy.log").open("wb") as output:
|
||||
with subprocess.Popen(
|
||||
(
|
||||
sys.executable,
|
||||
"-I",
|
||||
"-m",
|
||||
"litellm.proxy.proxy_cli",
|
||||
"--rust_gateway",
|
||||
"--config",
|
||||
str(config),
|
||||
"--host",
|
||||
"0.0.0.0",
|
||||
"--port",
|
||||
str(public_port),
|
||||
"--num_workers",
|
||||
"2",
|
||||
"--timeout_worker_healthcheck",
|
||||
"120",
|
||||
"--prometheus_metrics_port",
|
||||
str(metrics_port),
|
||||
*server_arguments,
|
||||
),
|
||||
env=environment,
|
||||
stdout=output,
|
||||
stderr=output,
|
||||
) as supervisor:
|
||||
try:
|
||||
with httpx.Client(
|
||||
base_url=f"http://127.0.0.1:{public_port}",
|
||||
transport=httpx.HTTPTransport(local_address="127.0.0.2"),
|
||||
trust_env=False,
|
||||
follow_redirects=True,
|
||||
timeout=5,
|
||||
) as external:
|
||||
assert eventually(lambda: ready(external), bool, seconds=180), (tmp_path / "proxy.log").read_text()
|
||||
response: Final = external.get(f"http://127.0.0.2:{metrics_port}/metrics")
|
||||
assert response.status_code == 200, response.text
|
||||
client_ip: Final = external.get(
|
||||
"/v1/mcp/network/client-ip",
|
||||
headers={"Authorization": f"Bearer {gateway.key}", "X-Forwarded-For": "1.2.3.4"},
|
||||
)
|
||||
assert client_ip.status_code == 200, client_ip.text
|
||||
assert client_ip.json() == {"ip": "127.0.0.2"}
|
||||
finally:
|
||||
assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text()
|
||||
assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text()
|
||||
|
|
@ -6,7 +6,14 @@ import pytest
|
|||
from click.testing import CliRunner
|
||||
|
||||
from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server
|
||||
from litellm.proxy.rust_gateway import child_argv, cli_command, gateway_binary, inherited_fd, startup_error
|
||||
from litellm.proxy.rust_gateway import (
|
||||
child_argv,
|
||||
cli_command,
|
||||
gateway_binary,
|
||||
inherited_fd,
|
||||
metrics_host,
|
||||
startup_error,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -41,6 +48,19 @@ def test_missing_binary_gives_install_instruction() -> None:
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("supervised", [False, True])
|
||||
@pytest.mark.parametrize("host", ["0.0.0.0", "192.0.2.1", "::"])
|
||||
def test_metrics_keep_configured_host_separate_from_private_listener(
|
||||
supervised: bool, host: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", host)
|
||||
if supervised:
|
||||
monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17")
|
||||
else:
|
||||
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
|
||||
assert metrics_host("127.0.0.1") == (host if supervised else "127.0.0.1")
|
||||
|
||||
|
||||
def test_current_environment_binary_wins_over_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
binary: Final = tmp_path / "litellm-gateway"
|
||||
binary.write_text("#!/bin/sh\nexit 0\n")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue