mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
feat(proxy): supervise the Rust forwarding server from gateway.launch
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
3442588fa0
commit
3399254dd1
3 changed files with 206 additions and 10 deletions
|
|
@ -19,9 +19,11 @@ Run with:
|
|||
|
||||
import os
|
||||
import sys
|
||||
import sysconfig
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from typing_extensions import assert_never
|
||||
from uvicorn.main import main as uvicorn_main
|
||||
|
||||
from litellm_cli.main import without_gc_pauses
|
||||
|
|
@ -31,17 +33,29 @@ if TYPE_CHECKING:
|
|||
|
||||
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
||||
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings
|
||||
from litellm.proxy.rust_gateway import Supervised
|
||||
|
||||
GATEWAY_APP: Final = "gateway.launch:worker_app"
|
||||
KEEPALIVE_FLAG: Final = "--timeout-keep-alive"
|
||||
|
||||
|
||||
def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str, ...]:
|
||||
def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str], supervised: "Supervised | None") -> tuple[str, ...]:
|
||||
"""Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly."""
|
||||
keepalive: Final = environ.get("KEEPALIVE_TIMEOUT")
|
||||
inherited_socket: Final = (
|
||||
(
|
||||
"--fd",
|
||||
str(supervised.listener_fd),
|
||||
"--proxy-headers",
|
||||
"--forwarded-allow-ips",
|
||||
",".join(supervised.trusted_proxies),
|
||||
)
|
||||
if supervised is not None
|
||||
else ()
|
||||
)
|
||||
if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv):
|
||||
return (GATEWAY_APP, "--factory", *argv)
|
||||
return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive)
|
||||
return (GATEWAY_APP, "--factory", *argv, *inherited_socket)
|
||||
return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive, *inherited_socket)
|
||||
|
||||
|
||||
def pool_database_url(
|
||||
|
|
@ -75,7 +89,34 @@ def _serve(argv: Sequence[str]) -> None:
|
|||
uvicorn_main(tuple(argv), prog_name="uvicorn")
|
||||
|
||||
|
||||
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
|
||||
def public_listener(argv: Sequence[str], binary: str | None) -> tuple[str, str, int, Mapping[str, str]]:
|
||||
"""Validate the uvicorn flags for Rust forwarding and return the binary, public address and child environment."""
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.proxy.rust_gateway import TRUSTED_PROXIES, startup_error
|
||||
|
||||
params: Final[Mapping[str, object]] = uvicorn_main.make_context(
|
||||
"uvicorn", list(uvicorn_argv(argv, os.environ, None))
|
||||
).params
|
||||
error: Final = startup_error(
|
||||
binary,
|
||||
{
|
||||
"--ssl-keyfile": params["ssl_keyfile"],
|
||||
"--ssl-certfile": params["ssl_certfile"],
|
||||
"--ssl-ciphers": any(arg.split("=")[0] == "--ssl-ciphers" for arg in argv),
|
||||
"--uds": params["uds"] is not None,
|
||||
"--fd": params["fd"] is not None,
|
||||
},
|
||||
)
|
||||
if error is not None:
|
||||
sys.exit(error)
|
||||
assert binary is not None
|
||||
host, port = TypeAdapter(tuple[str, int]).validate_python((params["host"], params["port"]))
|
||||
operator_trust: Final = TypeAdapter(str | None).validate_python(params["forwarded_allow_ips"])
|
||||
return binary, host, port, {} if operator_trust is None else {TRUSTED_PROXIES: operator_trust}
|
||||
|
||||
|
||||
def configure_database() -> None:
|
||||
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
||||
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url
|
||||
|
||||
|
|
@ -86,7 +127,32 @@ def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -
|
|||
sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}")
|
||||
if pooled_url is not None:
|
||||
export_pooled_database_url(pooled_url)
|
||||
serve(uvicorn_argv(argv, os.environ))
|
||||
|
||||
|
||||
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
|
||||
from litellm.proxy.rust_gateway import (
|
||||
Disabled,
|
||||
Supervised,
|
||||
Supervisor,
|
||||
gateway_binary,
|
||||
rust_gateway_mode,
|
||||
supervise,
|
||||
)
|
||||
|
||||
mode: Final = rust_gateway_mode(os.environ)
|
||||
match mode:
|
||||
case Disabled():
|
||||
configure_database()
|
||||
serve(uvicorn_argv(argv, os.environ, None))
|
||||
case Supervisor():
|
||||
binary, host, port, trust = public_listener(argv, gateway_binary(sysconfig.get_path("scripts")))
|
||||
configure_database()
|
||||
command: Final = (sys.executable, "-m", "gateway.launch", *argv)
|
||||
sys.exit(supervise(binary, command, host, port, {**os.environ, **trust}))
|
||||
case Supervised():
|
||||
serve(uvicorn_argv(argv, os.environ, mode))
|
||||
case _:
|
||||
assert_never(mode)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -107,3 +107,74 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip(
|
|||
finally:
|
||||
assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text()
|
||||
assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"server_arguments",
|
||||
[("--no-proxy-headers",), ("--forwarded-allow-ips", "127.0.0.1")],
|
||||
)
|
||||
def test_gateway_launcher_private_hop_preserves_client_address(
|
||||
tmp_path: Path, server_arguments: tuple[str, ...]
|
||||
) -> None:
|
||||
config: Final = tmp_path / "config.yaml"
|
||||
config.write_text(
|
||||
json.dumps({"model_list": [], "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY"}})
|
||||
)
|
||||
public_port: Final = allocate_port()
|
||||
environment: Final = {
|
||||
**{key: value for key, value in os.environ.items() if not key.startswith("DATABASE_") and key != "DIRECT_URL"},
|
||||
"LITELLM_MASTER_KEY": "sk-rust-forwarding-integration-key-not-a-secret",
|
||||
"LITELLM_RUST_GATEWAY": "true",
|
||||
"LITELLM_PGBOUNCER_ENABLED": "false",
|
||||
"CONFIG_FILE_PATH": str(config),
|
||||
"STORE_MODEL_IN_DB": "False",
|
||||
"PYTHON_DOTENV_DISABLED": "1",
|
||||
}
|
||||
log: Final = tmp_path / "gateway.log"
|
||||
with log.open("wb") as output:
|
||||
with subprocess.Popen(
|
||||
(
|
||||
sys.executable,
|
||||
"-I",
|
||||
"-m",
|
||||
"gateway.launch",
|
||||
"--host",
|
||||
"0.0.0.0",
|
||||
"--port",
|
||||
str(public_port),
|
||||
"--workers",
|
||||
"2",
|
||||
"--timeout-worker-healthcheck",
|
||||
"120",
|
||||
*server_arguments,
|
||||
),
|
||||
env=environment,
|
||||
stdout=output,
|
||||
stderr=output,
|
||||
) as supervisor:
|
||||
try:
|
||||
with httpx.Client(
|
||||
base_url=f"http://127.0.0.1:{public_port}",
|
||||
transport=httpx.HTTPTransport(local_address="127.0.0.2"),
|
||||
trust_env=False,
|
||||
timeout=5,
|
||||
) as external:
|
||||
assert eventually(lambda: ready(external), bool, seconds=180), log.read_text()
|
||||
response: Final = external.get(
|
||||
"/health/liveliness?client-ip-proof", headers={"X-Forwarded-For": "198.51.100.10"}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
events: Final = eventually(
|
||||
lambda: tuple(
|
||||
line
|
||||
for line in log.read_text().splitlines()
|
||||
if "GET /health/liveliness?client-ip-proof " in line
|
||||
),
|
||||
bool,
|
||||
seconds=10,
|
||||
)
|
||||
assert len(events) == 1, log.read_text()
|
||||
assert "127.0.0.2:" in events[0], events
|
||||
finally:
|
||||
assert stop_root_process(supervisor), log.read_text()
|
||||
assert supervisor.returncode == 0, log.read_text()
|
||||
|
|
|
|||
|
|
@ -15,10 +15,11 @@ import pytest
|
|||
from uvicorn.importer import import_from_string
|
||||
from uvicorn.main import main as uvicorn_main
|
||||
|
||||
from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv
|
||||
from gateway.launch import GATEWAY_APP, main, pool_database_url, public_listener, uvicorn_argv
|
||||
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
||||
from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings
|
||||
from litellm.proxy.proxy_server import app as proxy_app
|
||||
from litellm.proxy.rust_gateway import Supervised
|
||||
|
||||
DB_ENV: Final = {
|
||||
"DATABASE_HOST": "db.internal",
|
||||
|
|
@ -114,17 +115,65 @@ def _uvicorn_params(argv: tuple[str, ...]) -> dict[str, object]:
|
|||
|
||||
|
||||
class TestUvicornArgv:
|
||||
@pytest.mark.parametrize(
|
||||
"arguments", [("--uds", "/tmp/gateway.sock"), ("--fd", "0"), ("--fd=17",), ("--ssl-keyfile", "unused")]
|
||||
)
|
||||
def test_forwarding_rejects_unsupported_listeners_before_starting_processes(
|
||||
self, arguments: tuple[str, ...]
|
||||
) -> None:
|
||||
with pytest.raises(SystemExit, match=f"does not support {arguments[0].split('=')[0]}"):
|
||||
public_listener(arguments, "gateway")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("argv", "trust"),
|
||||
[
|
||||
((), {}),
|
||||
(("--forwarded-allow-ips", "10.0.0.5"), {"FORWARDED_ALLOW_IPS": "10.0.0.5"}),
|
||||
(("--forwarded-allow-ips=*",), {"FORWARDED_ALLOW_IPS": "*"}),
|
||||
],
|
||||
)
|
||||
def test_forwarding_carries_operator_proxy_trust_to_the_child(
|
||||
self, argv: tuple[str, ...], trust: dict[str, str]
|
||||
) -> None:
|
||||
assert public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway") == (
|
||||
"gateway",
|
||||
"0.0.0.0",
|
||||
4000,
|
||||
trust,
|
||||
)
|
||||
|
||||
def test_supervised_listener_uses_the_inherited_loopback_socket(self):
|
||||
supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1",))
|
||||
params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised))
|
||||
assert params["fd"] == 17
|
||||
assert params["workers"] == 2
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"argv",
|
||||
[(), ("--no-proxy-headers",), ("--forwarded-allow-ips", "192.0.2.1"), ("--forwarded-allow-ips=*",)],
|
||||
)
|
||||
def test_private_hop_always_processes_headers_with_the_resolved_trust(self, argv: tuple[str, ...]):
|
||||
supervised: Final = Supervised(17, "0.0.0.0", ("127.0.0.1", "10.0.0.5"))
|
||||
params: Final = _uvicorn_params(uvicorn_argv(argv, {}, supervised))
|
||||
assert params["proxy_headers"] is True
|
||||
assert params["forwarded_allow_ips"] == "127.0.0.1,10.0.0.5"
|
||||
|
||||
def test_direct_listener_keeps_explicit_proxy_header_options(self):
|
||||
params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {}, None))
|
||||
assert params["proxy_headers"] is False
|
||||
assert params["forwarded_allow_ips"] == "*"
|
||||
|
||||
def test_keepalive_env_reaches_uvicorn(self):
|
||||
params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"}))
|
||||
params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"}, None))
|
||||
assert params["app"] == GATEWAY_APP
|
||||
assert params["workers"] == 4
|
||||
assert params["timeout_keep_alive"] == 75
|
||||
|
||||
def test_unset_env_keeps_the_uvicorn_default(self):
|
||||
assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}))["timeout_keep_alive"] == 5
|
||||
assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}, None))["timeout_keep_alive"] == 5
|
||||
|
||||
def test_an_explicit_flag_wins_over_the_env(self):
|
||||
argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"})
|
||||
argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}, None)
|
||||
assert _uvicorn_params(argv)["timeout_keep_alive"] == 30
|
||||
|
||||
def test_the_app_uvicorn_is_told_to_serve_is_the_trimmed_gateway(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
|
@ -141,7 +190,7 @@ class TestUvicornArgv:
|
|||
):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
params: Final = _uvicorn_params(uvicorn_argv((), {}))
|
||||
params: Final = _uvicorn_params(uvicorn_argv((), {}, None))
|
||||
assert params["factory"] is True
|
||||
try:
|
||||
served: Final = import_from_string(cast(str, params["app"]))()
|
||||
|
|
@ -229,6 +278,16 @@ class TestMain:
|
|||
DatabaseURLSettings.from_env().apply_to_env()
|
||||
assert os.environ["DATABASE_URL"] == pooled
|
||||
|
||||
def test_forwarding_disabled_keeps_uvicorn_socket_options(
|
||||
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("LITELLM_RUST_GATEWAY", "false")
|
||||
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
|
||||
monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False)
|
||||
served: Final[list[tuple[str, ...]]] = []
|
||||
main(("--uds", "/tmp/gateway.sock"), serve=lambda argv: served.append(tuple(argv)))
|
||||
assert _uvicorn_params(served[0])["uds"] == "/tmp/gateway.sock"
|
||||
|
||||
def test_a_pooler_that_cannot_start_stops_the_gateway_before_uvicorn(
|
||||
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue