This commit is contained in:
yujonglee 2026-10-26 12:12:04 -04:00 • committed by GitHub
commit 14e341b710
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 2290 additions and 11 deletions

17
gateway/AGENTS.md Normal file
View file

@ -0,0 +1,17 @@
# Gateway
## Rust gateway
- `LITELLM_RUST_GATEWAY=true` enables the public Rust listener. Unset or exact `false` preserves Python-only startup
- Keep `LITELLM_RUST` independent. It controls the in-process Rust bridge
- Rust forwards every request to Python, which owns authentication, routing, policy and provider calls. Do not add native request selection or database initialization here
- `launch.py` owns startup and pooling. `supervisor.py` coordinates container processes, not application requests. A Python or Rust child exit fails the container rather than restarting that child
- Start Python on loopback `127.0.0.1:4002` before Rust on public port 4000. Reserve 4001 for metrics and never expose the private Python port in container or Service ports
- With a writer `DATABASE_URL`, start one managed PgBouncer even when `LITELLM_PGBOUNCER_ENABLED=false`. Both children receive the same pooled URL and marker, and Python must not start another pooler
- Keep database-free mode database-free. Read replicas remain outside the shared writer connection cap
- Drain Rust, stop Python, then stop PgBouncer and reap remaining children within the shutdown deadline. Defaults allow 15 seconds for Rust, 5 for Python and a 10-second cleanup margin
- Docker stop timeout and Helm termination grace must cover `LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS`, which defaults to 30
- Configure `LITELLM_RUST_GATEWAY_TRUSTED_PROXIES` with explicit ingress IPs or CIDRs. Empty trusts no external proxy, wildcard trust is rejected, and `Host` and `Origin` are preserved
- Terminate TLS at the edge. Enabled mode rejects Uvicorn TLS, reload, Unix-socket and file-descriptor listeners, env-file overrides and disabled proxy headers
- The image packages `/usr/local/bin/litellm-gateway`. Local runs may override `LITELLM_RUST_GATEWAY_BINARY` and the private port through `LITELLM_RUST_GATEWAY_PYTHON_PORT`
- Roll back by removing the Rust gateway opt-in or setting it to `false` and redeploying

View file

@ -47,6 +47,20 @@ RUN --mount=type=cache,target=/root/.cache/uv \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')"
RUN --mount=type=cache,target=/root/.cargo/registry \
--mount=type=cache,target=/root/.cargo/git \
--mount=type=cache,target=/app/litellm-rust/target \
CARGO_BIN="$(find /root/.cache /root/.cargo /root/.rustup /usr/local \( -type f -o -type l \) -name cargo -print -quit)" && \
test -x "$CARGO_BIN" || { echo "Rust cargo executable was not found" >&2; exit 1; } && \
cd litellm-rust && \
PATH="$(dirname "$CARGO_BIN"):${PATH}" "$CARGO_BIN" build --locked --release -p litellm-gateway --bin litellm-gateway && \
mkdir -p /rust-gateway && \
cp target/release/litellm-gateway /rust-gateway/litellm-gateway && \
readelf -l /rust-gateway/litellm-gateway > /tmp/litellm-gateway-linkage && \
grep -q "Requesting program interpreter:" /tmp/litellm-gateway-linkage && \
readelf -d /rust-gateway/litellm-gateway >> /tmp/litellm-gateway-linkage && \
grep -q "(NEEDED)" /tmp/litellm-gateway-linkage
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
@ -148,12 +162,13 @@ COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
COPY --from=builder /opt/prisma /opt/prisma
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
COPY --from=rust-native /rust-gateway/litellm-gateway /usr/local/bin/litellm-gateway
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete && \
RUN find /app/.venv -type d -path "*/tornado/test" -prune -exec rm -r {} \; && \
chmod -R a+rX /opt/prisma && \
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \
python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()"
python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()" && \
if LITELLM_RUST_GATEWAY=true /usr/local/bin/litellm-gateway 2>&1 | grep -Fq "Error: PythonUpstreamConfig"; then :; else exit 1; fi
USER nonroot

View file

@ -14,7 +14,10 @@ Run with:
import os
import sys
import time
from collections.abc import Callable, Mapping, Sequence
from contextlib import ExitStack
from types import MappingProxyType
from typing import Final
from uvicorn.main import main as uvicorn_main
@ -22,11 +25,13 @@ from uvicorn.main import main as uvicorn_main
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import (
PGBOUNCER_MANAGED_SHUTDOWN_SECONDS,
ManagedPgBouncerRuntime,
PgBouncerError,
PgBouncerSettings,
export_pooled_database_url,
pgbouncer_startup_signals,
start_in_container_pgbouncer,
start_managed_pgbouncer,
)
GATEWAY_APP: Final = "gateway.main:app"
@ -65,11 +70,92 @@ def pool_database_url(
)
def managed_pool_database_url(
settings: DatabaseURLSettings,
pgbouncer: PgBouncerSettings,
environ: Mapping[str, str],
) -> ManagedPgBouncerRuntime | PgBouncerError | None:
upstream_url: Final = environ.get("DATABASE_URL")
if not upstream_url:
return None
effective_settings: Final = pgbouncer if pgbouncer.enabled else pgbouncer.model_copy(update={"enabled": True})
return start_managed_pgbouncer(effective_settings, upstream_url, token_auth=settings.token_auth())
def _pooled_child_environments(
python_environment: Mapping[str, str],
rust_environment: Mapping[str, str],
pooler: ManagedPgBouncerRuntime | None,
) -> tuple[Mapping[str, str], Mapping[str, str]]:
if pooler is None:
return (
MappingProxyType({**python_environment, "LITELLM_PGBOUNCER_ENABLED": "false"}),
rust_environment,
)
pooled_values: Final = {
"DATABASE_URL": pooler.pooled_url,
"LITELLM_PGBOUNCER_POOLED_DATABASE_URL": "true",
}
return (
MappingProxyType(
{
**python_environment,
**pooled_values,
"LITELLM_PGBOUNCER_ENABLED": "false",
}
),
MappingProxyType({**rust_environment, **pooled_values}),
)
def _serve(argv: Sequence[str]) -> None:
uvicorn_main(tuple(argv), prog_name="uvicorn")
def _stop_starting_pooler(pooler: ManagedPgBouncerRuntime, timeout: int) -> None:
pooler.stop(time.monotonic() + timeout)
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
gateway_mode: Final = os.environ.get("LITELLM_RUST_GATEWAY")
if gateway_mode not in (None, "true", "false"):
sys.exit("LiteLLM gateway: LITELLM_RUST_GATEWAY must be exactly 'true' or 'false'")
if gateway_mode == "true":
from dataclasses import replace
from gateway.supervisor import ConfigurationFailure, build_configuration, startup_scope, supervise
with ExitStack() as startup_cleanup, startup_scope():
rust_settings: Final = DatabaseURLSettings.from_env()
rust_settings.apply_to_env()
preflight_configuration: Final = build_configuration(argv, os.environ, sys.executable)
if isinstance(preflight_configuration, ConfigurationFailure):
sys.exit(f"LiteLLM gateway supervisor: {preflight_configuration.reason}")
pooler: Final = managed_pool_database_url(rust_settings, PgBouncerSettings(), os.environ)
if isinstance(pooler, PgBouncerError):
sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooler.reason}")
if pooler is not None:
startup_cleanup.callback(
_stop_starting_pooler, pooler, preflight_configuration.termination_grace_timeout
)
export_pooled_database_url(pooler.pooled_url)
python_environment, rust_environment = _pooled_child_environments(
preflight_configuration.python_environment,
preflight_configuration.rust_environment,
pooler,
)
final_configuration: Final = replace(
preflight_configuration,
python_environment=python_environment,
rust_environment=rust_environment,
managed_pooler=pooler,
)
failure: Final = supervise(final_configuration)
startup_cleanup.pop_all()
if failure is not None:
sys.exit(f"LiteLLM gateway supervisor: {failure.reason}")
return
pgbouncer: Final = PgBouncerSettings()
with pgbouncer_startup_signals(pgbouncer.enabled):
settings: Final = DatabaseURLSettings.from_env()

725
gateway/supervisor.py Normal file
View file

@ -0,0 +1,725 @@
from __future__ import annotations
import ctypes
import http.client
import ipaddress
import os
import signal
import socket
import subprocess
import sys
import time
from collections.abc import Callable, Generator, Mapping, Sequence
from contextlib import ExitStack, contextmanager
from dataclasses import dataclass
from pathlib import Path
from threading import Event
from types import FrameType, MappingProxyType
from typing import Final, Protocol, cast
from urllib.parse import urlsplit
import click
from pydantic import TypeAdapter
from uvicorn.main import main as uvicorn_main
from gateway.launch import uvicorn_argv
from litellm.proxy.db.pgbouncer import ManagedPgBouncerRuntime, pgbouncer_startup_signals
RUST_GATEWAY_FLAG: Final = "LITELLM_RUST_GATEWAY"
RUST_GATEWAY_BINARY: Final = "LITELLM_RUST_GATEWAY_BINARY"
PYTHON_PORT_ENV: Final = "LITELLM_RUST_GATEWAY_PYTHON_PORT"
PYTHON_UPSTREAM_ENV: Final = "LITELLM_PYTHON_UPSTREAM"
TRUSTED_PROXIES_ENV: Final = "LITELLM_RUST_GATEWAY_TRUSTED_PROXIES"
METRICS_PORT_ENV: Final = "LITELLM_GATEWAY_METRICS_PORT"
TERMINATION_GRACE_ENV: Final = "LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS"
DEFAULT_RUST_GATEWAY_BINARY: Final = "/usr/local/bin/litellm-gateway"
DEFAULT_PRIVATE_PORT: Final = 4002
DEFAULT_PYTHON_GRACE_SECONDS: Final = 5
DEFAULT_RUST_DRAIN_SECONDS: Final = 15
DEFAULT_STARTUP_TIMEOUT_SECONDS: Final = 120
DEFAULT_TERMINATION_GRACE_SECONDS: Final = 30
TERMINATION_CLEANUP_MARGIN_SECONDS: Final = 10
LIVENESS_PATH: Final = "/health/liveliness"
@dataclass(frozen=True, slots=True)
class ConfigurationFailure:
reason: str
@dataclass(frozen=True, slots=True)
class SupervisorFailure:
reason: str
@dataclass(frozen=True, slots=True)
class SupervisorConfig:
python_command: tuple[str, ...]
rust_command: tuple[str, ...]
python_environment: Mapping[str, str]
rust_environment: Mapping[str, str]
public_probe_host: str
public_port: int
private_port: int
startup_timeout: int
rust_drain_timeout: int
python_grace_timeout: int
termination_grace_timeout: int = DEFAULT_TERMINATION_GRACE_SECONDS
managed_pooler: ManagedPgBouncerRuntime | None = None
class ChildProcess(Protocol):
@property
def pid(self) -> int: ...
def poll(self) -> int | None: ...
def wait(self, timeout: float | None = None) -> int: ...
class ProcessRunner(Protocol):
def start(self, command: tuple[str, ...], environment: Mapping[str, str]) -> ChildProcess: ...
class SubprocessRunner:
def start(self, command: tuple[str, ...], environment: Mapping[str, str]) -> ChildProcess:
return subprocess.Popen(command, env=environment, start_new_session=True)
def gateway_enabled(environ: Mapping[str, str]) -> bool | ConfigurationFailure:
value: Final = environ.get(RUST_GATEWAY_FLAG)
if value is None or value == "false":
return False
if value == "true":
return True
return ConfigurationFailure(f"{RUST_GATEWAY_FLAG} must be exactly 'true' or 'false'")
def _port_setting(name: str, default: int, environ: Mapping[str, str]) -> int | ConfigurationFailure:
value: Final = environ.get(name)
if value is None:
return default
try:
port: Final = int(value)
except ValueError:
return ConfigurationFailure(f"{name} must be an integer port")
if not 1 <= port <= 65535:
return ConfigurationFailure(f"{name} must be between 1 and 65535")
return port
def _timeout_setting(name: str, default: int, environ: Mapping[str, str]) -> int | ConfigurationFailure:
value: Final = environ.get(name)
if value is None:
return default
try:
timeout: Final = int(value)
except ValueError:
return ConfigurationFailure(f"{name} must be a positive integer")
if timeout < 1:
return ConfigurationFailure(f"{name} must be a positive integer")
return timeout
def _boolean_setting(name: str, environ: Mapping[str, str]) -> bool | ConfigurationFailure:
value: Final = environ.get(name)
if value is None:
return False
normalized: Final = value.casefold()
if normalized in ("true", "1", "yes", "y", "on", "t"):
return True
if normalized in ("false", "0", "no", "n", "off", "f"):
return False
return ConfigurationFailure(f"{name} must be a boolean")
def _collector_port(environ: Mapping[str, str]) -> int | ConfigurationFailure | None:
enabled: Final = _boolean_setting("LITELLM_COLLECTOR_ENABLED", environ)
if isinstance(enabled, ConfigurationFailure) or not enabled:
return enabled if isinstance(enabled, ConfigurationFailure) else None
address: Final = environ.get("LITELLM_COLLECTOR_ADDRESS")
if address is None or not address.startswith("tcp://"):
return None
try:
parsed: Final = urlsplit(address)
port: Final = parsed.port
except ValueError:
return ConfigurationFailure("LITELLM_COLLECTOR_ADDRESS must contain a valid TCP port")
if port is None:
return ConfigurationFailure("LITELLM_COLLECTOR_ADDRESS must contain a valid TCP port")
if not 1 <= port <= 65535:
return ConfigurationFailure("LITELLM_COLLECTOR_ADDRESS must contain a valid TCP port")
return port
def _trusted_proxies(environ: Mapping[str, str]) -> str | ConfigurationFailure:
value: Final = environ.get(TRUSTED_PROXIES_ENV, "")
if not value:
return value
try:
networks: Final = tuple(ipaddress.ip_network(entry.strip(), strict=False) for entry in value.split(","))
except ValueError:
return ConfigurationFailure(f"{TRUSTED_PROXIES_ENV} must contain IP addresses or CIDRs")
if any(network.prefixlen == 0 for network in networks):
return ConfigurationFailure(f"{TRUSTED_PROXIES_ENV} must not trust every address")
if any(not entry.strip() for entry in value.split(",")):
return ConfigurationFailure(f"{TRUSTED_PROXIES_ENV} contains an empty entry")
return value
def _worker_arguments(arguments: tuple[str, ...], private_port: int) -> tuple[str, ...]:
value_options: Final = frozenset(("--host", "--port", "--forwarded-allow-ips"))
boolean_options: Final = frozenset(("--proxy-headers",))
consumed_values: Final = frozenset(
index + 1 for index, argument in enumerate(arguments[:-1]) if argument in value_options
)
without_overrides: Final = tuple(
argument
for index, argument in enumerate(arguments)
if index not in consumed_values
and not any(
argument == option or argument.startswith(f"{option}=") for option in value_options | boolean_options
)
)
return (
"--host",
"127.0.0.1",
"--port",
str(private_port),
"--proxy-headers",
"--forwarded-allow-ips",
"127.0.0.1",
*without_overrides[1:],
)
def _context_value(context: click.Context, name: str) -> object:
parameters: Final = TypeAdapter(dict[str, object]).validate_python(context.params)
value: Final = parameters[name]
return value
def _first_port_collision(ports: tuple[tuple[str, int], ...]) -> tuple[str, str] | None:
for index, current in enumerate(ports):
for other in ports[index + 1 :]:
if current[1] == other[1]:
return current[0], other[0]
return None
def build_configuration(
argv: Sequence[str],
environ: Mapping[str, str],
python_executable: str,
) -> SupervisorConfig | ConfigurationFailure:
arguments: Final = uvicorn_argv(argv, environ)
try:
context: Final = uvicorn_main.make_context(
"uvicorn",
list(arguments),
allow_extra_args=True,
ignore_unknown_options=True,
)
except click.ClickException as error:
return ConfigurationFailure(f"invalid Uvicorn options: {error.format_message()}")
if any(argument == "--no-proxy-headers" for argument in arguments):
return ConfigurationFailure("--no-proxy-headers is unsupported with the Rust gateway")
if any(argument.startswith("--ssl-") for argument in arguments):
return ConfigurationFailure("Uvicorn TLS options are unsupported with the Rust gateway")
if any(name.startswith("UVICORN_SSL_") for name in environ):
return ConfigurationFailure("UVICORN_SSL_* settings are unsupported with the Rust gateway")
if _context_value(context, "env_file") is not None:
return ConfigurationFailure("--env-file is unsupported with the Rust gateway")
if _context_value(context, "uds") is not None or _context_value(context, "fd") is not None:
return ConfigurationFailure("Uvicorn Unix socket and file descriptor listeners are unsupported")
if _context_value(context, "reload") is True:
return ConfigurationFailure("Uvicorn reload is unsupported with the Rust gateway")
if _context_value(context, "proxy_headers") is not True:
return ConfigurationFailure("Uvicorn proxy headers must be enabled with the Rust gateway")
protocol: Final = _context_value(context, "http")
if not isinstance(protocol, str) or protocol not in ("auto", "h11", "httptools"):
return ConfigurationFailure("the Rust gateway supports only Uvicorn HTTP/1.1 protocols")
host: Final = _context_value(context, "host")
port: Final = _context_value(context, "port")
if (
not isinstance(host, str)
or not host
or any(character.isspace() or character in "/[]\r\n" for character in host)
):
return ConfigurationFailure("Uvicorn must have a non-empty public host")
if not isinstance(port, int) or isinstance(port, bool) or not 1 <= port <= 65535:
return ConfigurationFailure("Uvicorn public port must be between 1 and 65535")
if "HOST" in environ and environ["HOST"] != host:
return ConfigurationFailure("HOST conflicts with the effective Uvicorn host")
if "PORT" in environ and environ["PORT"] != str(port):
return ConfigurationFailure("PORT conflicts with the effective Uvicorn port")
private_port: Final = _port_setting(PYTHON_PORT_ENV, DEFAULT_PRIVATE_PORT, environ)
if isinstance(private_port, ConfigurationFailure):
return private_port
metrics_port: Final = _port_setting(METRICS_PORT_ENV, 0, environ) if METRICS_PORT_ENV in environ else None
if isinstance(metrics_port, ConfigurationFailure):
return metrics_port
pgbouncer_port: Final = (
_port_setting("LITELLM_PGBOUNCER_PORT", 6432, environ) if bool(environ.get("DATABASE_URL")) else None
)
if isinstance(pgbouncer_port, ConfigurationFailure):
return pgbouncer_port
collector_port: Final = _collector_port(environ)
if isinstance(collector_port, ConfigurationFailure):
return collector_port
startup_timeout: Final = _timeout_setting(
"LITELLM_RUST_GATEWAY_STARTUP_TIMEOUT_SECONDS",
DEFAULT_STARTUP_TIMEOUT_SECONDS,
environ,
)
if isinstance(startup_timeout, ConfigurationFailure):
return startup_timeout
rust_drain_timeout: Final = _timeout_setting(
"LITELLM_RUST_GATEWAY_DRAIN_TIMEOUT_SECONDS",
DEFAULT_RUST_DRAIN_SECONDS,
environ,
)
if isinstance(rust_drain_timeout, ConfigurationFailure):
return rust_drain_timeout
python_grace_timeout: Final = _timeout_setting(
"LITELLM_RUST_GATEWAY_PYTHON_GRACE_SECONDS",
DEFAULT_PYTHON_GRACE_SECONDS,
environ,
)
if isinstance(python_grace_timeout, ConfigurationFailure):
return python_grace_timeout
termination_grace_timeout: Final = _timeout_setting(
TERMINATION_GRACE_ENV,
DEFAULT_TERMINATION_GRACE_SECONDS,
environ,
)
if isinstance(termination_grace_timeout, ConfigurationFailure):
return termination_grace_timeout
if rust_drain_timeout + python_grace_timeout + TERMINATION_CLEANUP_MARGIN_SECONDS > termination_grace_timeout:
return ConfigurationFailure(f"{TERMINATION_GRACE_ENV} must allow Rust drain, Python grace, and cleanup margin")
trusted_proxies: Final = _trusted_proxies(environ)
if isinstance(trusted_proxies, ConfigurationFailure):
return trusted_proxies
collisions: Final = _first_port_collision(
(
("public gateway", port),
("private Python upstream", private_port),
*(() if metrics_port is None else (("metrics server", metrics_port),)),
*(() if pgbouncer_port is None else (("PgBouncer", pgbouncer_port),)),
*(() if collector_port is None else (("collector", collector_port),)),
)
)
if collisions is not None:
return ConfigurationFailure(f"{collisions[0]} and {collisions[1]} use the same TCP port")
binary: Final = environ.get(RUST_GATEWAY_BINARY, DEFAULT_RUST_GATEWAY_BINARY)
binary_path: Final = Path(binary)
if not binary_path.is_file() or not os.access(binary_path, os.X_OK):
return ConfigurationFailure(f"Rust gateway binary is not executable: {binary}")
upstream: Final = f"http://127.0.0.1:{private_port}"
configured_upstream: Final = environ.get(PYTHON_UPSTREAM_ENV)
if configured_upstream is not None and configured_upstream != upstream:
return ConfigurationFailure(f"{PYTHON_UPSTREAM_ENV} must equal {upstream}")
python_environment: Final = MappingProxyType(
{
**environ,
RUST_GATEWAY_FLAG: "false",
}
)
rust_environment: Final = MappingProxyType(
{
**environ,
RUST_GATEWAY_FLAG: "true",
PYTHON_UPSTREAM_ENV: upstream,
TRUSTED_PROXIES_ENV: trusted_proxies,
"HOST": host,
"PORT": str(port),
}
)
probe_host: Final = "127.0.0.1" if host in ("0.0.0.0", "*") else "::1" if host == "::" else host
python_command: Final = (
python_executable,
"-m",
"gateway.launch",
*_worker_arguments(arguments, private_port),
)
return SupervisorConfig(
python_command=python_command,
rust_command=(binary,),
python_environment=python_environment,
rust_environment=rust_environment,
public_probe_host=probe_host,
public_port=port,
private_port=private_port,
startup_timeout=startup_timeout,
rust_drain_timeout=rust_drain_timeout,
python_grace_timeout=python_grace_timeout,
termination_grace_timeout=termination_grace_timeout,
)
def _enable_subreaper() -> SupervisorFailure | None:
if not sys.platform.startswith("linux"):
return SupervisorFailure("the Rust gateway supervisor requires Linux")
prctl: Final = ctypes.CDLL(None, use_errno=True).prctl
if prctl(36, 1, 0, 0, 0) != 0:
return SupervisorFailure("could not enable Linux child subreaper mode")
return None
def _install_signal_handlers(
requested: Event,
) -> tuple[tuple[signal.Signals, object], ...]:
signals: Final = (signal.SIGTERM, signal.SIGINT)
previous: Final = tuple((handled, signal.getsignal(handled)) for handled in signals)
def request_shutdown(_signum: int, _frame: FrameType | None) -> None:
requested.set()
for handled in signals:
signal.signal(handled, request_shutdown)
return previous
def _restore_signal_handlers(
previous: tuple[tuple[signal.Signals, object], ...],
) -> None:
for handled, handler in previous:
signal.signal(handled, cast(signal.Handlers | Callable[[int, FrameType | None], object] | None, handler))
@contextmanager
def startup_scope() -> Generator[None, None, None]:
with pgbouncer_startup_signals(True):
subreaper_failure: Final = _enable_subreaper()
if subreaper_failure is not None:
raise SystemExit(subreaper_failure.reason)
yield
def _tcp_ready(host: str, port: int) -> bool:
try:
with socket.create_connection((host, port), timeout=0.2):
return True
except OSError:
return False
def _http_ready(host: str, port: int) -> bool:
connection: Final = http.client.HTTPConnection(host, port, timeout=0.3)
try:
connection.request("GET", LIVENESS_PATH)
response: Final = connection.getresponse()
response.read()
return response.status == 200
except (http.client.HTTPException, OSError):
return False
finally:
connection.close()
def _group_exists(process_group: int) -> bool:
try:
os.killpg(process_group, 0)
return True
except ProcessLookupError:
return False
except PermissionError:
return True
def _signal_group(process_group: int, requested: signal.Signals) -> None:
try:
os.killpg(process_group, requested)
except ProcessLookupError:
pass
def _reap_adopted_children(excluded: frozenset[int]) -> None:
children_path: Final = Path(f"/proc/self/task/{os.getpid()}/children")
try:
children: Final = tuple(int(pid) for pid in children_path.read_text().split() if int(pid) not in excluded)
except (OSError, ValueError):
return
for child in children:
try:
os.waitpid(child, os.WNOHANG)
except ChildProcessError:
pass
def _adopted_children(excluded: frozenset[int]) -> tuple[int, ...]:
children_path: Final = Path(f"/proc/self/task/{os.getpid()}/children")
try:
return tuple(int(pid) for pid in children_path.read_text().split() if int(pid) not in excluded)
except (OSError, ValueError):
return ()
def _signal_adopted_child(child: int, requested: signal.Signals) -> None:
process_group: Final = os.getpgid(child)
if process_group == os.getpgrp():
os.kill(child, requested)
else:
os.killpg(process_group, requested)
def _reap_supervised_children(
excluded: frozenset[int],
managed_pooler: ManagedPgBouncerRuntime | None,
) -> None:
if managed_pooler is None:
_reap_adopted_children(excluded)
return
with managed_pooler.owned_pid_guard() as pooler_pid:
protected: Final = frozenset((*excluded, *((pooler_pid,) if pooler_pid is not None else ())))
_reap_adopted_children(protected)
def _current_adopted_children(
excluded: frozenset[int],
managed_pooler: ManagedPgBouncerRuntime | None,
) -> tuple[tuple[int, ...], frozenset[int]]:
if managed_pooler is None:
return _adopted_children(excluded), excluded
with managed_pooler.owned_pid_guard() as pooler_pid:
protected: Final = frozenset((*excluded, *((pooler_pid,) if pooler_pid is not None else ())))
return _adopted_children(protected), protected
def _terminate_adopted_children(
excluded: frozenset[int],
deadline: float | None = None,
managed_pooler: ManagedPgBouncerRuntime | None = None,
) -> None:
term_children: Final = _current_adopted_children(excluded, managed_pooler)[0]
for child in term_children:
try:
_signal_adopted_child(child, signal.SIGTERM)
except ProcessLookupError:
pass
term_deadline: Final = time.monotonic() + 1 if deadline is None else min(deadline, time.monotonic() + 1)
while time.monotonic() < term_deadline and _current_adopted_children(excluded, managed_pooler)[0]:
_reap_supervised_children(excluded, managed_pooler)
time.sleep(0.05)
kill_children: Final = _current_adopted_children(excluded, managed_pooler)[0]
for child in kill_children:
try:
_signal_adopted_child(child, signal.SIGKILL)
except ProcessLookupError:
pass
final_deadline: Final = time.monotonic() + 1 if deadline is None else min(deadline, time.monotonic() + 1)
while time.monotonic() < final_deadline and _current_adopted_children(excluded, managed_pooler)[0]:
_reap_supervised_children(excluded, managed_pooler)
time.sleep(0.05)
_reap_supervised_children(excluded, managed_pooler)
def _wait_for_group(
process: ChildProcess,
deadline: float,
protected_children: frozenset[int],
managed_pooler: ManagedPgBouncerRuntime | None,
) -> bool:
while time.monotonic() < deadline and _group_exists(process.pid):
process.poll()
_reap_supervised_children(protected_children, managed_pooler)
time.sleep(0.05)
process.poll()
_reap_supervised_children(protected_children, managed_pooler)
return not _group_exists(process.pid)
def _stop_process_group(
process: ChildProcess,
timeout: int,
protected_children: frozenset[int],
managed_pooler: ManagedPgBouncerRuntime | None,
deadline: float,
) -> None:
_signal_group(process.pid, signal.SIGTERM)
stop_deadline: Final = min(deadline, time.monotonic() + timeout)
if not _wait_for_group(process, stop_deadline, protected_children, managed_pooler):
_signal_group(process.pid, signal.SIGKILL)
_wait_for_group(process, min(deadline, time.monotonic() + 1), protected_children, managed_pooler)
try:
process.wait(timeout=max(0.0, min(1.0, deadline - time.monotonic())))
except subprocess.TimeoutExpired:
_signal_group(process.pid, signal.SIGKILL)
try:
process.wait(timeout=max(0.0, deadline - time.monotonic()))
except subprocess.TimeoutExpired:
pass
_reap_supervised_children(protected_children, managed_pooler)
def _wait_for_python(
process: ChildProcess,
port: int,
deadline: float,
requested: Event,
managed_pooler: ManagedPgBouncerRuntime | None,
) -> bool | SupervisorFailure:
protected_children: Final = frozenset((process.pid,))
while not requested.is_set() and time.monotonic() < deadline:
_reap_supervised_children(protected_children, managed_pooler)
if (status := process.poll()) is not None:
return SupervisorFailure(f"Python gateway exited during startup with status {status}")
if _tcp_ready("127.0.0.1", port):
return True
time.sleep(0.05)
_reap_supervised_children(protected_children, managed_pooler)
return False
def _wait_for_rust(
python: ChildProcess,
rust: ChildProcess,
host: str,
port: int,
deadline: float,
requested: Event,
managed_pooler: ManagedPgBouncerRuntime | None,
) -> bool | SupervisorFailure:
protected_children: Final = frozenset((python.pid, rust.pid))
while not requested.is_set() and time.monotonic() < deadline:
_reap_supervised_children(protected_children, managed_pooler)
if (python_status := python.poll()) is not None:
return SupervisorFailure(f"Python gateway exited during startup with status {python_status}")
if (rust_status := rust.poll()) is not None:
return SupervisorFailure(f"Rust gateway exited during startup with status {rust_status}")
if _http_ready(host, port):
return True
time.sleep(0.05)
_reap_supervised_children(protected_children, managed_pooler)
return False
def _monitor_children(
python: ChildProcess,
rust: ChildProcess,
requested: Event,
managed_pooler: ManagedPgBouncerRuntime | None,
) -> SupervisorFailure | None:
protected_children: Final = frozenset((python.pid, rust.pid))
while not requested.wait(0.1):
_reap_supervised_children(protected_children, managed_pooler)
if (python_status := python.poll()) is not None:
return SupervisorFailure(f"Python gateway exited unexpectedly with status {python_status}")
if (rust_status := rust.poll()) is not None:
return SupervisorFailure(f"Rust gateway exited unexpectedly with status {rust_status}")
_reap_supervised_children(protected_children, managed_pooler)
return None
def _shutdown(
python: ChildProcess | None,
rust: ChildProcess | None,
config: SupervisorConfig,
) -> None:
deadline: Final = time.monotonic() + config.termination_grace_timeout
protected: Final = frozenset(process.pid for process in (python, rust) if process is not None)
if rust is not None:
_stop_process_group(
rust,
config.rust_drain_timeout,
protected,
config.managed_pooler,
deadline,
)
if python is not None:
_stop_process_group(
python,
config.python_grace_timeout,
protected,
config.managed_pooler,
deadline,
)
_terminate_adopted_children(protected, deadline, config.managed_pooler)
if config.managed_pooler is not None:
config.managed_pooler.stop(deadline)
_terminate_adopted_children(frozenset(), deadline)
def _supervise_with_python(
config: SupervisorConfig,
process_runner: ProcessRunner,
requested: Event,
python: ChildProcess,
) -> SupervisorFailure | None:
with ExitStack() as python_cleanup:
python_cleanup.callback(_shutdown, python, None, config)
startup_deadline: Final = time.monotonic() + config.startup_timeout
python_ready: Final = _wait_for_python(
python,
config.private_port,
startup_deadline,
requested,
config.managed_pooler,
)
if isinstance(python_ready, SupervisorFailure):
return python_ready
if not python_ready:
return None if requested.is_set() else SupervisorFailure("Python gateway startup timed out")
if requested.is_set():
return None
try:
rust: Final = process_runner.start(config.rust_command, config.rust_environment)
except OSError as error:
return SupervisorFailure(f"could not start Rust gateway: {error}")
with ExitStack() as gateway_cleanup:
gateway_cleanup.callback(_shutdown, python, rust, config)
python_cleanup.pop_all()
rust_ready: Final = _wait_for_rust(
python,
rust,
config.public_probe_host,
config.public_port,
startup_deadline,
requested,
config.managed_pooler,
)
if isinstance(rust_ready, SupervisorFailure):
return rust_ready
if not rust_ready:
return None if requested.is_set() else SupervisorFailure("Rust gateway readiness timed out")
return _monitor_children(python, rust, requested, config.managed_pooler)
def _supervise(
config: SupervisorConfig,
process_runner: ProcessRunner,
requested: Event,
) -> SupervisorFailure | None:
try:
python: Final = process_runner.start(config.python_command, config.python_environment)
except OSError as error:
_shutdown(None, None, config)
return SupervisorFailure(f"could not start Python gateway: {error}")
return _supervise_with_python(config, process_runner, requested, python)
def supervise(config: SupervisorConfig, runner: ProcessRunner | None = None) -> SupervisorFailure | None:
subreaper_failure: Final = _enable_subreaper()
if subreaper_failure is not None:
if config.managed_pooler is not None:
config.managed_pooler.stop(time.monotonic() + config.termination_grace_timeout)
return subreaper_failure
process_runner: Final = runner if runner is not None else SubprocessRunner()
requested: Final = Event()
previous_handlers: Final = _install_signal_handlers(requested)
try:
return _supervise(config, process_runner, requested)
finally:
_restore_signal_handlers(previous_handlers)
if __name__ == "__main__":
sys.exit("run the Rust gateway through python -m gateway.launch")

View file

@ -599,6 +599,14 @@ ImplementationSpecific
{{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}}
{{- define "litellm.gateway.rustGatewayEnabled" -}}
{{- range .Values.gateway.extraEnv -}}
{{- if and (eq .name "LITELLM_RUST_GATEWAY") (eq (toString .value) "true") -}}
true
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Directory of the collector's unix socket, shared by the gateway and
collector containers through an emptyDir. Empty when the sidecar is off

View file

@ -1,4 +1,16 @@
{{- if .Values.gateway.enabled }}
{{- $rustGatewayEnabled := eq (include "litellm.gateway.rustGatewayEnabled" . | trim) "true" }}
{{- $pythonPort := 4002 }}
{{- $configuredGrace := .Values.gateway.terminationGracePeriodSeconds }}
{{- $gracePeriod := 30 }}
{{- if not (or (kindIs "invalid" $configuredGrace) (eq (printf "%v" $configuredGrace) "")) }}
{{- $gracePeriod = int $configuredGrace }}
{{- end }}
{{- range .Values.gateway.extraEnv }}
{{- if and (eq .name "LITELLM_RUST_GATEWAY_PYTHON_PORT") .value }}
{{- $pythonPort = int .value }}
{{- end }}
{{- end }}
apiVersion: apps/v1
kind: Deployment
metadata:
@ -79,9 +91,16 @@ spec:
{{- if eq (int .Values.gateway.metricsServer.port) 4000 }}
{{- fail "gateway.metricsServer.port must differ from the gateway port 4000" }}
{{- end }}
{{- if and $rustGatewayEnabled (eq (int .Values.gateway.metricsServer.port) $pythonPort) }}
{{- fail "gateway.metricsServer.port must differ from the private Python gateway port" }}
{{- end }}
- name: PROMETHEUS_MULTIPROC_DIR
value: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
- name: LITELLM_GATEWAY_METRICS_PORT
value: {{ .Values.gateway.metricsServer.port | quote }}
{{- end }}
- name: LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS
value: {{ $gracePeriod | quote }}
{{- if .Values.gateway.collector.enabled }}
{{- include "litellm.gateway.collectorEnv" . | nindent 12 }}
{{- end }}
@ -245,8 +264,5 @@ spec:
topologySpreadConstraints:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- $gracePeriod := .Values.gateway.terminationGracePeriodSeconds }}
{{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }}
terminationGracePeriodSeconds: {{ $gracePeriod }}
{{- end }}
{{- end }}

View file

@ -0,0 +1,141 @@
suite: test opt-in Rust gateway deployment
templates:
- gateway/deployment.yaml
- gateway/service.yaml
- gateway/configmap.yaml
values:
- ./values/required.yaml
tests:
- it: leaves HTTP probes and service ports unchanged when disabled
set:
gateway.metricsServer.enabled: true
asserts:
- equal:
path: spec.template.spec.containers[0].livenessProbe.httpGet.path
value: /health/liveliness
template: gateway/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_GATEWAY_METRICS_PORT
value: "4001"
any: true
template: gateway/deployment.yaml
- equal:
path: spec.template.spec.containers[0].ports[0].containerPort
value: 4000
template: gateway/deployment.yaml
- equal:
path: spec.ports[0].port
value: 4000
template: gateway/service.yaml
- it: passes enabled gateway settings and keeps the private listener off Services
set:
gateway.extraEnv:
- name: LITELLM_RUST_GATEWAY
value: "true"
gateway.envConfigMaps:
- gateway-runtime
gateway.metricsServer.enabled: true
gateway.metricsServer.port: 4001
gateway.collector.enabled: true
gateway.collector.address: tcp://127.0.0.1:4102
gateway.terminationGracePeriodSeconds: 30
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_RUST_GATEWAY
value: "true"
template: gateway/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_GATEWAY_METRICS_PORT
value: "4001"
template: gateway/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_COLLECTOR_ADDRESS
value: tcp://127.0.0.1:4102
template: gateway/deployment.yaml
- equal:
path: spec.template.spec.containers[0].envFrom[0].configMapRef.name
value: gateway-runtime
template: gateway/deployment.yaml
- equal:
path: spec.template.spec.containers[0].livenessProbe.httpGet.path
value: /health/liveliness
template: gateway/deployment.yaml
- equal:
path: spec.template.spec.containers[0].readinessProbe.httpGet.path
value: /health/readiness
template: gateway/deployment.yaml
- equal:
path: spec.template.spec.containers[0].ports[0].containerPort
value: 4000
template: gateway/deployment.yaml
- equal:
path: spec.template.spec.terminationGracePeriodSeconds
value: 30
template: gateway/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS
value: "30"
template: gateway/deployment.yaml
- equal:
path: spec.ports[0].port
value: 4000
template: gateway/service.yaml
- it: rejects a metrics port that collides with the default private Python listener
template: gateway/deployment.yaml
set:
gateway.extraEnv:
- name: LITELLM_RUST_GATEWAY
value: "true"
gateway.metricsServer.enabled: true
gateway.metricsServer.port: 4002
asserts:
- failedTemplate:
errorPattern: gateway.metricsServer.port must differ from the private Python gateway port
- it: exports metrics collision metadata when the flag comes from envFrom
set:
gateway.envConfigMaps:
- gateway-runtime
gateway.metricsServer.enabled: true
gateway.metricsServer.port: 4001
asserts:
- equal:
path: spec.template.spec.containers[0].envFrom[0].configMapRef.name
value: gateway-runtime
template: gateway/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_GATEWAY_METRICS_PORT
value: "4001"
template: gateway/deployment.yaml
- it: preserves an explicit zero termination grace for enabled-mode rejection
set:
gateway.extraEnv:
- name: LITELLM_RUST_GATEWAY
value: "true"
gateway.terminationGracePeriodSeconds: 0
asserts:
- equal:
path: spec.template.spec.terminationGracePeriodSeconds
value: 0
template: gateway/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS
value: "0"
template: gateway/deployment.yaml

View file

@ -3770,6 +3770,7 @@ dependencies = [
"http-body-util",
"hyper 1.10.1",
"hyper-util",
"ipnet",
"litellm-auth-types",
"litellm-config",
"litellm-core-utils",

View file

@ -12,6 +12,7 @@ envy = "0.4.2"
http-body-util = "0.1"
hyper = { version = "1", features = ["http1"] }
hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] }
ipnet = "2.12"
litellm-core-utils.workspace = true
litellm-inference.workspace = true
litellm-gateway-inference.workspace = true

View file

@ -53,6 +53,7 @@ pub struct InvalidUpstream {
pub struct Forward {
client: Arc<Client<HttpConnector, Body>>,
authority: Authority,
trusted_proxies: Option<crate::proxy_headers::TrustedProxies>,
}
impl Forward {
@ -72,9 +73,15 @@ impl Forward {
Ok(Self {
client: Arc::new(Client::builder(TokioExecutor::new()).build(HttpConnector::new())),
authority,
trusted_proxies: None,
})
}
pub fn with_trusted_proxies(mut self, proxies: &str) -> Result<Self, crate::Error> {
self.trusted_proxies = Some(crate::proxy_headers::TrustedProxies::parse(proxies)?);
Ok(self)
}
pub(super) async fn send(&self, mut request: Request, decline: Decline) -> Response {
let websocket = if request.method() == Method::GET
&& request.version() == Version::HTTP_11
@ -126,8 +133,14 @@ impl Forward {
if websocket.is_some() {
websocket_headers(&mut parts.headers);
}
if let Some(ConnectInfo(client)) = parts.extensions.get::<ConnectInfo<SocketAddr>>() {
append_forwarded_for(&mut parts.headers, *client);
let peer = parts
.extensions
.get::<ConnectInfo<SocketAddr>>()
.map(|info| info.0);
if let Some(policy) = &self.trusted_proxies {
policy.apply(&mut parts.headers, peer);
} else if let Some(client) = peer {
append_forwarded_for(&mut parts.headers, client);
}
parts
.headers

View file

@ -1,5 +1,9 @@
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error(
"LITELLM_RUST_GATEWAY_TRUSTED_PROXIES must contain comma-separated IP addresses or CIDRs, not wildcard trust"
)]
TrustedProxies,
#[error(
"LITELLM_RUST_GATEWAY=true requires LITELLM_PYTHON_UPSTREAM to be a valid http:// origin"
)]

View file

@ -1,6 +1,7 @@
mod auth;
mod dispatch;
mod error;
mod proxy_headers;
mod secrets;
mod upgrades;
pub use dispatch::{Fallback, Forward, InvalidUpstream};

View file

@ -0,0 +1,172 @@
use std::net::{IpAddr, SocketAddr};
use axum::http::{HeaderMap, HeaderValue};
use ipnet::IpNet;
#[derive(Clone)]
pub(crate) struct TrustedProxies(Vec<IpNet>);
impl TrustedProxies {
pub(crate) fn parse(value: &str) -> Result<Self, crate::Error> {
if value.trim().is_empty() {
return Ok(Self(Vec::new()));
}
value
.split(',')
.map(|value| {
let value = value.trim();
let network = value
.parse::<IpNet>()
.or_else(|_| value.parse::<IpAddr>().map(IpNet::from))
.map_err(|_| crate::Error::TrustedProxies)?;
if network.prefix_len() == 0 {
return Err(crate::Error::TrustedProxies);
}
Ok(network)
})
.collect::<Result<Vec<_>, _>>()
.map(Self)
}
fn contains(&self, address: IpAddr) -> bool {
self.0.iter().any(|network| network.contains(&address))
}
pub(crate) fn apply(&self, headers: &mut HeaderMap, peer: Option<SocketAddr>) {
let trusted = peer.is_some_and(|peer| self.contains(peer.ip()));
let chain = trusted
.then(|| {
headers
.get_all("x-forwarded-for")
.iter()
.map(|value| value.to_str().ok())
.collect::<Option<Vec<_>>>()
.and_then(|values| {
values
.iter()
.flat_map(|value| value.split(','))
.map(|value| value.trim().parse::<IpAddr>().ok())
.collect::<Option<Vec<_>>>()
})
})
.flatten();
let client = chain
.as_ref()
.and_then(|chain| {
chain
.iter()
.rev()
.find(|address| !self.contains(**address))
.or_else(|| chain.first())
.copied()
})
.or_else(|| peer.map(|peer| peer.ip()));
let scheme = if trusted && headers.get_all("x-forwarded-proto").iter().count() == 1 {
match headers
.get("x-forwarded-proto")
.and_then(|value| value.to_str().ok())
{
Some("https") => "https",
_ => "http",
}
} else {
"http"
};
for name in [
"forwarded",
"x-forwarded-for",
"x-forwarded-proto",
"x-forwarded-host",
"x-forwarded-port",
"x-forwarded-prefix",
"x-real-ip",
] {
headers.remove(name);
}
if let Some(client) = client
&& let Ok(value) = HeaderValue::from_str(&client.to_string())
{
headers.insert("x-forwarded-for", value);
}
headers.insert("x-forwarded-proto", HeaderValue::from_static(scheme));
}
}
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
#[rstest]
#[case::direct("", "127.0.0.2", "198.51.100.7", "https", "127.0.0.2", "http")]
#[case::trusted(
"127.0.0.2",
"127.0.0.2",
"198.51.100.7",
"https",
"198.51.100.7",
"https"
)]
#[case::chain(
"10.0.0.0/8",
"10.0.0.1",
"192.0.2.1, 198.51.100.7, 10.0.0.2",
"https",
"198.51.100.7",
"https"
)]
#[case::invalid_chain(
"10.0.0.0/8",
"10.0.0.1",
"198.51.100.7, unknown",
"https",
"10.0.0.1",
"https"
)]
#[case::invalid_scheme(
"127.0.0.2",
"127.0.0.2",
"198.51.100.7",
"https,http",
"198.51.100.7",
"http"
)]
#[case::ipv6("fd00::/8", "fd00::1", "2001:db8::7", "https", "2001:db8::7", "https")]
fn normalizes_proxy_identity(
#[case] trusted: &str,
#[case] peer: &str,
#[case] chain: &str,
#[case] proto: &str,
#[case] client: &str,
#[case] scheme: &str,
) {
let mut headers = HeaderMap::new();
headers.insert("host", HeaderValue::from_static("gateway.example"));
headers.insert("origin", HeaderValue::from_static("https://app.example"));
headers.insert("x-forwarded-for", HeaderValue::from_str(chain).unwrap());
headers.insert("x-forwarded-proto", HeaderValue::from_str(proto).unwrap());
headers.insert("forwarded", HeaderValue::from_static("proto=https"));
TrustedProxies::parse(trusted).unwrap().apply(
&mut headers,
Some(SocketAddr::new(peer.parse().unwrap(), 1234)),
);
assert_eq!(headers["x-forwarded-for"], client);
assert_eq!(headers["x-forwarded-proto"], scheme);
assert_eq!(headers["host"], "gateway.example");
assert_eq!(headers["origin"], "https://app.example");
assert!(!headers.contains_key("forwarded"));
}
#[rstest]
#[case("*")]
#[case("0.0.0.0/0")]
#[case("::/0")]
#[case("127.0.0.1,")]
#[case("proxy.example")]
fn invalid_policy_fails_closed(#[case] value: &str) {
assert!(matches!(
TrustedProxies::parse(value),
Err(crate::Error::TrustedProxies)
));
}
}

View file

@ -12,6 +12,7 @@ pub(super) struct Settings {
pub litellm_config: String,
pub litellm_python_upstream: Option<String>,
pub litellm_rust_gateway: bool,
pub litellm_rust_gateway_trusted_proxies: Option<String>,
rust_log: Option<String>,
}
@ -23,6 +24,7 @@ impl Default for Settings {
litellm_config: "config.yaml".into(),
litellm_python_upstream: None,
litellm_rust_gateway: false,
litellm_rust_gateway_trusted_proxies: None,
rust_log: None,
}
}
@ -51,7 +53,11 @@ impl Settings {
{
return Err(invalid());
}
litellm_gateway::Forward::new(upstream).map_err(|_| invalid())
let forward = litellm_gateway::Forward::new(upstream).map_err(|_| invalid())?;
match self.litellm_rust_gateway_trusted_proxies.as_deref() {
Some(proxies) => forward.with_trusted_proxies(proxies),
None => Ok(forward),
}
}
pub fn log_level(&self) -> Level {
@ -192,6 +198,43 @@ mod tests {
);
}
#[rstest]
#[case::absent(None)]
#[case::empty(Some(""))]
#[case::explicit_trust(Some("127.0.0.2"))]
fn trusted_proxy_configuration_keeps_absent_and_empty_distinct(#[case] value: Option<&str>) {
let settings: Settings = match value {
Some(trusted) => parse(&[
("LITELLM_PYTHON_UPSTREAM", "http://127.0.0.1:4002"),
("LITELLM_RUST_GATEWAY_TRUSTED_PROXIES", trusted),
])
.unwrap(),
None => parse(&[("LITELLM_PYTHON_UPSTREAM", "http://127.0.0.1:4002")]).unwrap(),
};
assert_eq!(
settings.litellm_rust_gateway_trusted_proxies.as_deref(),
value
);
assert!(settings.forwarder().is_ok());
}
#[rstest]
#[case::wildcard("*")]
#[case::default_route("0.0.0.0/0")]
fn invalid_trusted_proxy_configuration_fails_forwarder(#[case] value: &str) {
let settings: Settings = parse(&[
("LITELLM_PYTHON_UPSTREAM", "http://127.0.0.1:4002"),
("LITELLM_RUST_GATEWAY_TRUSTED_PROXIES", value),
])
.unwrap();
assert!(matches!(
settings.forwarder(),
Err(litellm_gateway::Error::TrustedProxies)
));
}
#[rstest]
fn environment_overrides_startup_settings() {
let settings: Settings = parse(&[

View file

@ -0,0 +1,647 @@
import http.client
import os
import signal
import subprocess
import sys
import textwrap
import time
from collections.abc import Iterator, Mapping
from contextlib import ExitStack
from dataclasses import dataclass
from pathlib import Path
from typing import Final
import pytest
from gateway.launch import (
_pooled_child_environments, # pyright: ignore[reportPrivateUsage] # exercise the child env policy
)
from gateway.supervisor import ChildProcess, SupervisorConfig, SupervisorFailure, supervise
from litellm.proxy.db.pgbouncer import (
ManagedPgBouncerRuntime,
PgBouncerSettings,
start_managed_pgbouncer,
)
from tests.integration._support.ports import allocate_port
ROOT: Final = Path(__file__).resolve().parents[3]
SERVER_SOURCE: Final = """\
import http.server
import signal
import subprocess
import sys
import threading
from pathlib import Path
role, port, events, mode, child_pid_file = sys.argv[1:]
if role == "python" and mode == "startup-failure":
raise SystemExit(17)
class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
if self.path == "/events":
self.send_header("Content-Type", "text/event-stream")
self.end_headers()
self.wfile.write(b"data: connected\\n\\n" if self.path == "/events" else b"ok")
self.wfile.flush()
if self.path == "/events":
threading.Event().wait()
if role == "rust" and mode == "exit":
threading.Timer(0.1, server.shutdown).start()
def log_message(self, *_args):
pass
server = http.server.ThreadingHTTPServer(("127.0.0.1", int(port)), Handler)
server.daemon_threads = True
if mode == "orphan":
child = subprocess.Popen(
(
sys.executable,
"-P",
"-c",
"import signal,time; signal.signal(signal.SIGTERM, signal.SIG_IGN); time.sleep(60)",
),
start_new_session=True,
)
Path(child_pid_file).write_text(str(child.pid))
elif mode == "orphan-exit":
middle_command = "import subprocess,sys,time; child=subprocess.Popen((sys.executable,'-P','-c','import time; time.sleep(0.35)'), start_new_session=True); open(sys.argv[1],'w').write(str(child.pid)); time.sleep(0.2)"
subprocess.Popen((sys.executable, "-P", "-c", middle_command, child_pid_file))
def stop(_signum, _frame):
with Path(events).open("a") as output:
output.write(role + "\\n")
if mode not in ("ignore", "orphan"):
threading.Thread(target=server.shutdown).start()
signal.signal(signal.SIGTERM, stop)
if role == "python" and mode == "exit":
threading.Timer(0.5, server.shutdown).start()
server.serve_forever()
"""
LAUNCH_MAIN_SOURCE: Final = """\
import os
from pathlib import Path
from gateway.launch import main
def serve(_argv):
Path(os.environ["TEST_SERVE_MARKER"]).write_text(
os.environ["LITELLM_PGBOUNCER_ENABLED"] + "|" + os.environ.get("DATABASE_URL", "")
)
main((), serve)
"""
SUPERVISOR_SOURCE: Final = """\
import os
import subprocess
import sys
import threading
import time
from dataclasses import replace
from types import MappingProxyType
from pathlib import Path
from gateway.supervisor import SupervisorConfig, supervise
server, private_port, public_port, events, python_mode, rust_mode, child_pid_file, rust_timeout = sys.argv[1:]
environment = MappingProxyType(dict(os.environ))
python_command = (sys.executable, server, "python", private_port, events, python_mode, child_pid_file)
rust_command = (
("/missing/rust-gateway",)
if rust_mode == "missing"
else (sys.executable, server, "rust", public_port, events, rust_mode, child_pid_file)
)
config = SupervisorConfig(
python_command=python_command,
rust_command=rust_command,
python_environment=environment,
rust_environment=environment,
public_probe_host="127.0.0.1",
public_port=int(public_port),
private_port=int(private_port),
startup_timeout=5,
rust_drain_timeout=int(rust_timeout),
python_grace_timeout=2,
)
if "TEST_MANAGED_PGBOUNCER_BINARY" in os.environ:
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, start_managed_pgbouncer
managed_pooler = start_managed_pgbouncer(
PgBouncerSettings(
enabled=True,
port=int(os.environ["TEST_MANAGED_PGBOUNCER_PORT"]),
binary=os.environ["TEST_MANAGED_PGBOUNCER_BINARY"],
),
"postgresql://litellm:password@127.0.0.1/litellm",
)
if isinstance(managed_pooler, PgBouncerError):
print(managed_pooler.reason, file=sys.stderr)
sys.exit(1)
pooler_pid_file = Path(os.environ["TEST_MANAGED_PGBOUNCER_PID_FILE"])
def write_pooler_pid():
temporary = pooler_pid_file.with_suffix(".tmp")
temporary.write_text(str(managed_pooler.pooler.pid))
temporary.replace(pooler_pid_file)
def record_pooler_pid():
while True:
write_pooler_pid()
time.sleep(0.05)
write_pooler_pid()
threading.Thread(target=record_pooler_pid, daemon=True).start()
config = replace(config, managed_pooler=managed_pooler)
failure = supervise(config)
if failure is not None:
print(failure.reason, file=sys.stderr)
sys.exit(1)
"""
@dataclass(frozen=True, slots=True)
class _FailAfterPythonStartRunner:
python_command: tuple[str, ...]
python_pid_file: Path
cleanup: ExitStack
def start(self, command: tuple[str, ...], environment: Mapping[str, str]) -> ChildProcess:
if command != self.python_command:
raise OSError("injected Rust gateway start failure")
process: Final = subprocess.Popen(
command,
env=environment,
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
start_new_session=True,
)
self.cleanup.callback(_terminate_if_running, process)
self.python_pid_file.write_text(str(process.pid))
return process
def _start_supervisor(
tmp_path: Path,
private_port: int,
public_port: int,
events: Path,
python_mode: str = "normal",
rust_mode: str = "normal",
rust_timeout: int = 2,
managed_pooler: tuple[Path, int, Path] | None = None,
) -> subprocess.Popen[bytes]:
server: Final = tmp_path / "server.py"
launcher: Final = tmp_path / "supervisor.py"
child_pid_file: Final = tmp_path / "orphan.pid"
server.write_text(SERVER_SOURCE)
launcher.write_text(SUPERVISOR_SOURCE)
environment: Final = {
**os.environ,
**(
{}
if managed_pooler is None
else {
"TEST_MANAGED_PGBOUNCER_BINARY": str(managed_pooler[0]),
"TEST_MANAGED_PGBOUNCER_PORT": str(managed_pooler[1]),
"TEST_MANAGED_PGBOUNCER_PID_FILE": str(managed_pooler[2]),
}
),
}
return subprocess.Popen(
(
sys.executable,
"-P",
str(launcher),
str(server),
str(private_port),
str(public_port),
str(events),
python_mode,
rust_mode,
str(child_pid_file),
str(rust_timeout),
),
cwd=ROOT,
env=environment,
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
start_new_session=True,
)
def _fake_pgbouncer(tmp_path: Path) -> Path:
binary: Final = tmp_path / "fake-pgbouncer"
binary.write_text(
textwrap.dedent(
f"""\
#!{sys.executable}
import configparser, select, socket, sys
if sys.argv[1:] == ["--version"]:
print("PgBouncer 1.25.2")
sys.exit(0)
configuration = configparser.ConfigParser()
configuration.read(sys.argv[1])
port = configuration.getint("pgbouncer", "listen_port")
tcp = socket.socket()
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcp.bind(("127.0.0.1", port))
tcp.listen()
unix = socket.socket(socket.AF_UNIX)
unix.bind(configuration.get("pgbouncer", "unix_socket_dir") + f"/.s.PGSQL.{{port}}")
unix.listen()
while True:
for listener in select.select((tcp, unix), [], [])[0]:
listener.accept()[0].close()
"""
)
)
binary.chmod(0o700)
return binary
def _slow_starting_pgbouncer(tmp_path: Path, pid_file: Path, config_file: Path) -> Path:
binary: Final = tmp_path / "slow-pgbouncer"
binary.write_text(
textwrap.dedent(
f"""\
#!{sys.executable}
import os, signal, sys, time
from pathlib import Path
if sys.argv[1:] == ["--version"]:
print("PgBouncer 1.26.0")
sys.exit(0)
Path({str(config_file)!r}).write_text(sys.argv[1])
Path({str(pid_file)!r}).write_text(str(os.getpid()))
signal.signal(signal.SIGTERM, signal.SIG_IGN)
time.sleep(60)
"""
)
)
binary.chmod(0o700)
return binary
def test_managed_pgbouncer_stop_is_bounded_and_guards_its_owned_pid(tmp_path: Path) -> None:
port: Final = allocate_port()
runtime: Final = start_managed_pgbouncer(
PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pgbouncer(tmp_path))),
"postgresql://litellm:password@127.0.0.1/litellm",
)
assert isinstance(runtime, ManagedPgBouncerRuntime), runtime
pid: Final = runtime.pooler.pid
deadline: Final = time.monotonic() + 2
try:
with runtime.owned_pid_guard() as guarded_pid:
assert guarded_pid == pid
finally:
runtime.stop(deadline)
assert pid is not None
assert not Path(f"/proc/{pid}").exists()
assert not runtime.runtime_dir.exists()
def test_launch_cleans_up_managed_pooler_when_startup_is_interrupted(tmp_path: Path) -> None:
pid_file: Final = tmp_path / "pooler.pid"
config_file: Final = tmp_path / "pooler-config"
process: Final = subprocess.Popen( # test-quality-ok: child must import the working-tree gateway.launch module
(
sys.executable,
"-m",
"gateway.launch",
"--host",
"127.0.0.1",
"--port",
str(allocate_port()),
),
cwd=ROOT,
env={
**os.environ,
"DATABASE_URL": "postgresql://litellm:password@127.0.0.1/litellm",
"LITELLM_PGBOUNCER_BINARY": str(_slow_starting_pgbouncer(tmp_path, pid_file, config_file)),
"LITELLM_PGBOUNCER_ENABLED": "false",
"LITELLM_PGBOUNCER_PORT": str(allocate_port()),
"LITELLM_RUST_GATEWAY": "true",
"LITELLM_RUST_GATEWAY_BINARY": "/bin/true",
},
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
start_new_session=True,
)
try:
pooler_pid: Final = _wait_for_pid(pid_file)
os.kill(process.pid, signal.SIGTERM) # test-quality-ok: Interrupt startup to exercise managed cleanup
assert process.wait(timeout=20) == 143
assert not Path(f"/proc/{pooler_pid}").exists()
runtime_dir: Final = Path(config_file.read_text()).parent
assert not runtime_dir.exists()
finally:
_terminate_if_running(process)
def _wait_for_pid(pid_file: Path) -> int:
deadline: Final = time.monotonic() + 5
pooler_pid: Final = next(
(pid for pid in _recorded_pooler_pids(pid_file, deadline) if Path(f"/proc/{pid}").exists()),
None,
)
if pooler_pid is None:
pytest.fail("managed PgBouncer did not record a live process")
return pooler_pid
def _recorded_pooler_pids(pid_file: Path, deadline: float) -> Iterator[int]:
while time.monotonic() < deadline:
try:
yield int(pid_file.read_text())
except (OSError, ValueError):
pass
time.sleep(0.05)
def _wait_for_replacement_pooler(pid_file: Path, original_pid: int) -> int | None:
deadline: Final = time.monotonic() + 5
return next(
(
pid
for pid in _recorded_pooler_pids(pid_file, deadline)
if pid != original_pid and Path(f"/proc/{pid}").exists()
),
None,
)
def test_database_free_python_child_runs_launch_with_pooling_disabled(tmp_path: Path) -> None:
serve_marker: Final = tmp_path / "serve"
inherited_environment: Final = {
name: value
for name, value in os.environ.items()
if not name.startswith("DATABASE_") and name not in ("DIRECT_URL", "LITELLM_PGBOUNCER_POOLED_DATABASE_URL")
}
rust_environment: Final = {
**inherited_environment,
"LITELLM_RUST_GATEWAY": "false",
"LITELLM_PGBOUNCER_ENABLED": "true",
"TEST_SERVE_MARKER": str(serve_marker),
}
python_environment, _ = _pooled_child_environments(rust_environment, {}, None)
child: Final = subprocess.run(
(sys.executable, "-P", "-c", LAUNCH_MAIN_SOURCE),
cwd=ROOT,
env=python_environment,
capture_output=True,
timeout=10,
)
assert child.returncode == 0, child.stderr.decode(errors="replace")
assert serve_marker.read_text() == "false|"
def _wait_for_gateway(process: subprocess.Popen[bytes], port: int) -> None:
deadline: Final = time.monotonic() + 10
while time.monotonic() < deadline:
if process.poll() is not None:
pytest.fail(f"supervisor exited early with status {process.returncode}")
if _gateway_readiness_status(port) == 200:
return
time.sleep(0.05)
pytest.fail("supervisor did not start both child listeners")
def _gateway_readiness_status(port: int) -> int | None:
connection: Final = http.client.HTTPConnection("127.0.0.1", port, timeout=0.2)
try:
connection.request("GET", "/health/readiness")
response: Final = connection.getresponse()
response.read()
return response.status
except OSError:
return None
finally:
connection.close()
def _terminate_if_running(process: subprocess.Popen[bytes]) -> None:
if process.poll() is not None:
return
os.killpg(process.pid, signal.SIGKILL) # test-quality-ok: Cleanup terminates only this test-owned supervisor group
process.wait(timeout=5)
def test_supervisor_drains_rust_before_python(tmp_path: Path) -> None:
private_port: Final = allocate_port()
public_port: Final = allocate_port()
events: Final = tmp_path / "signals"
process: Final = _start_supervisor(tmp_path, private_port, public_port, events)
try:
_wait_for_gateway(process, public_port)
os.kill(process.pid, signal.SIGTERM) # test-quality-ok: The signal is the ordered-shutdown event under test
assert process.wait(timeout=10) == 0
assert events.read_text().splitlines() == ["rust", "python"]
finally:
_terminate_if_running(process)
def test_supervisor_fails_when_python_cannot_start(tmp_path: Path) -> None:
process: Final = _start_supervisor(
tmp_path,
allocate_port(),
allocate_port(),
tmp_path / "signals",
python_mode="startup-failure",
)
try:
assert process.wait(timeout=10) == 1
finally:
_terminate_if_running(process)
def test_supervisor_cleans_up_python_when_rust_cannot_start(tmp_path: Path) -> None:
events: Final = tmp_path / "signals"
process: Final = _start_supervisor(tmp_path, allocate_port(), allocate_port(), events, rust_mode="missing")
try:
assert process.wait(timeout=10) == 1
assert events.read_text().splitlines() == ["python"]
finally:
_terminate_if_running(process)
def test_supervisor_cleans_children_and_managed_pooler_when_rust_start_raises(tmp_path: Path) -> None:
server: Final = tmp_path / "server.py"
server.write_text(SERVER_SOURCE)
python_pid_file: Final = tmp_path / "python.pid"
python_command: Final = (
sys.executable,
"-P",
str(server),
"python",
str(allocate_port()),
str(tmp_path / "events"),
"normal",
str(tmp_path / "orphan.pid"),
)
pooler: Final = start_managed_pgbouncer(
PgBouncerSettings(enabled=True, port=allocate_port(), binary=str(_fake_pgbouncer(tmp_path))),
"postgresql://litellm:password@127.0.0.1/litellm",
)
assert isinstance(pooler, ManagedPgBouncerRuntime), pooler
pooler_pid: Final = pooler.pooler.pid
config: Final = SupervisorConfig(
python_command=python_command,
rust_command=("/injected/rust-gateway",),
python_environment={},
rust_environment={},
public_probe_host="127.0.0.1",
public_port=allocate_port(),
private_port=int(python_command[4]),
startup_timeout=5,
rust_drain_timeout=1,
python_grace_timeout=1,
termination_grace_timeout=5,
managed_pooler=pooler,
)
try:
with ExitStack() as child_cleanup:
failure: Final = supervise(
config,
runner=_FailAfterPythonStartRunner(python_command, python_pid_file, child_cleanup),
)
assert isinstance(failure, SupervisorFailure)
assert "injected Rust gateway start failure" in failure.reason
python_pid: Final = int(python_pid_file.read_text())
assert not Path(f"/proc/{python_pid}").exists()
assert pooler_pid is not None
assert not Path(f"/proc/{pooler_pid}").exists()
assert not pooler.runtime_dir.exists()
finally:
pooler.stop(time.monotonic() + 2)
@pytest.mark.parametrize("exiting_child", ("python", "rust"))
def test_supervisor_fails_when_either_child_exits(tmp_path: Path, exiting_child: str) -> None:
events: Final = tmp_path / "signals"
process: Final = _start_supervisor(
tmp_path,
allocate_port(),
allocate_port(),
events,
python_mode="exit" if exiting_child == "python" else "normal",
rust_mode="exit" if exiting_child == "rust" else "normal",
)
try:
assert process.wait(timeout=10) == 1
finally:
_terminate_if_running(process)
def test_supervisor_kills_unresponsive_rust_group_and_reaps_orphan(tmp_path: Path) -> None:
private_port: Final = allocate_port()
public_port: Final = allocate_port()
events: Final = tmp_path / "signals"
child_pid_file: Final = tmp_path / "orphan.pid"
process: Final = _start_supervisor(
tmp_path,
private_port,
public_port,
events,
rust_mode="orphan",
rust_timeout=1,
)
try:
_wait_for_gateway(process, public_port)
deadline: Final = time.monotonic() + 5
while not child_pid_file.is_file() and time.monotonic() < deadline:
time.sleep(0.05)
assert child_pid_file.is_file()
child_pid: Final = int(child_pid_file.read_text())
os.kill(process.pid, signal.SIGTERM) # test-quality-ok: The signal is the orphan-cleanup event under test
assert process.wait(timeout=10) == 0
assert events.read_text().splitlines() == ["rust", "python"]
assert not Path(f"/proc/{child_pid}").exists()
finally:
_terminate_if_running(process)
def test_supervisor_reaps_adopted_child_while_gateway_is_ready(tmp_path: Path) -> None:
public_port: Final = allocate_port()
child_pid_file: Final = tmp_path / "orphan.pid"
pooler_pid_file: Final = tmp_path / "pooler.pid"
process: Final = _start_supervisor(
tmp_path,
allocate_port(),
public_port,
tmp_path / "signals",
rust_mode="orphan-exit",
managed_pooler=(_fake_pgbouncer(tmp_path), allocate_port(), pooler_pid_file),
)
try:
_wait_for_gateway(process, public_port)
pooler_pid: Final = _wait_for_pid(pooler_pid_file)
deadline: Final = time.monotonic() + 5
while not child_pid_file.is_file() and time.monotonic() < deadline:
time.sleep(0.05)
assert child_pid_file.is_file()
child_pid: Final = int(child_pid_file.read_text())
child_deadline: Final = time.monotonic() + 5
while Path(f"/proc/{child_pid}").exists() and time.monotonic() < child_deadline:
time.sleep(0.05)
assert not Path(f"/proc/{child_pid}").exists()
assert process.poll() is None
assert Path(f"/proc/{pooler_pid}").exists()
os.kill(pooler_pid, signal.SIGKILL) # test-quality-ok: The signal exercises supervised PgBouncer restart
replacement_pid: Final = _wait_for_replacement_pooler(pooler_pid_file, pooler_pid)
assert replacement_pid is not None
assert replacement_pid != pooler_pid
_wait_for_gateway(process, public_port)
os.kill(process.pid, signal.SIGTERM) # test-quality-ok: Signal shutdown after observing the adopted child reap
assert process.wait(timeout=10) == 0
assert not Path(f"/proc/{replacement_pid}").exists()
finally:
_terminate_if_running(process)
def test_supervisor_stops_managed_pooler_when_python_startup_fails(tmp_path: Path) -> None:
pooler_pid_file: Final = tmp_path / "pooler.pid"
process: Final = _start_supervisor(
tmp_path,
allocate_port(),
allocate_port(),
tmp_path / "signals",
python_mode="startup-failure",
managed_pooler=(_fake_pgbouncer(tmp_path), allocate_port(), pooler_pid_file),
)
try:
assert process.wait(timeout=10) == 1
pooler_pid: Final = int(pooler_pid_file.read_text())
assert not Path(f"/proc/{pooler_pid}").exists()
finally:
_terminate_if_running(process)
def test_supervisor_bounds_shutdown_with_an_open_sse_response(tmp_path: Path) -> None:
public_port: Final = allocate_port()
events: Final = tmp_path / "signals"
process: Final = _start_supervisor(
tmp_path,
allocate_port(),
public_port,
events,
rust_mode="ignore",
rust_timeout=1,
)
connection: Final = http.client.HTTPConnection("127.0.0.1", public_port, timeout=2)
try:
_wait_for_gateway(process, public_port)
connection.request("GET", "/events")
response: Final = connection.getresponse()
assert response.status == 200
assert response.getheader("Content-Type") == "text/event-stream"
assert response.readline() == b"data: connected\n"
os.kill(process.pid, signal.SIGTERM) # test-quality-ok: The signal starts the bounded SSE drain under test
assert process.wait(timeout=10) == 0
assert events.read_text().splitlines() == ["rust", "python"]
finally:
connection.close()
_terminate_if_running(process)

View file

@ -0,0 +1,183 @@
import http.client
import os
import signal
import subprocess
import sys
import time
from collections.abc import Mapping
from pathlib import Path
from typing import Final
import pytest
from pydantic import TypeAdapter
from tests.integration._support.ports import allocate_port
ROOT: Final = Path(__file__).resolve().parents[3]
DEFAULT_BINARY: Final = ROOT / "litellm-rust" / "target" / "debug" / "litellm-gateway"
HEADER_OBSERVATION: Final = TypeAdapter(dict[str, str | None])
APP_SOURCE: Final = """\
from fastapi import FastAPI, Request
app = FastAPI()
@app.get("/health/readiness")
async def readiness():
return {"status": "healthy"}
@app.get("/headers")
async def headers(request: Request):
return {
"client": request.client.host if request.client is not None else None,
"host": request.headers.get("host"),
"origin": request.headers.get("origin"),
"xff": request.headers.get("x-forwarded-for"),
"proto": request.headers.get("x-forwarded-proto"),
"scheme": request.url.scheme,
}
"""
def _request(address: str, port: int, source: str | None = None) -> dict[str, str | None]:
client: Final = http.client.HTTPConnection(
address,
port,
timeout=1,
source_address=None if source is None else (source, 0),
)
try:
client.request(
"GET",
"/headers",
headers={
"Host": "gateway.example",
"Origin": "https://ui.example",
"X-Forwarded-For": "198.51.100.7",
"X-Forwarded-Proto": "https",
"Connection": "close",
},
)
response: Final = client.getresponse()
body: Final = response.read()
assert response.status == 200, body
return HEADER_OBSERVATION.validate_json(body)
finally:
client.close()
def _wait_until_ready(address: str, port: int, process: subprocess.Popen[bytes]) -> None:
deadline: Final = time.monotonic() + 10
while time.monotonic() < deadline:
if process.poll() is not None:
_stdout, stderr = process.communicate()
pytest.fail(
f"Rust gateway exited during startup with status {process.returncode}: "
f"{(stderr or b'').decode(errors='replace')}"
)
connection: Final = http.client.HTTPConnection(address, port, timeout=0.2)
try:
connection.request("GET", "/health/readiness")
response: Final = connection.getresponse()
response.read()
if response.status == 200:
return
except OSError:
pass
finally:
connection.close()
time.sleep(0.05)
pytest.fail("Rust gateway did not forward readiness to the Uvicorn fixture")
@pytest.mark.parametrize(
("trusted_proxies", "expected_client", "expected_proto"),
(
("", "127.0.0.2", "http"),
("127.0.0.2", "198.51.100.7", "https"),
),
)
def test_rust_gateway_canonicalizes_headers_before_real_uvicorn(
tmp_path: Path,
trusted_proxies: str,
expected_client: str,
expected_proto: str,
) -> None:
binary: Final = Path(os.environ.get("LITELLM_RUST_GATEWAY_BINARY", str(DEFAULT_BINARY)))
if not binary.is_file():
pytest.skip("build litellm-rust/crates/gateway before running the real Uvicorn trust fixture")
python_port: Final = allocate_port()
rust_port: Final = allocate_port()
app_path: Final = tmp_path / "fixture.py"
app_path.write_text(APP_SOURCE)
python_environment: Final = {
**os.environ,
"PYTHONPATH": str(tmp_path),
"FORWARDED_ALLOW_IPS": "127.0.0.1",
}
python: Final = subprocess.Popen(
(
sys.executable,
"-P",
"-m",
"uvicorn",
"fixture:app",
"--host",
"127.0.0.1",
"--port",
str(python_port),
"--proxy-headers",
"--forwarded-allow-ips",
"127.0.0.1",
),
env=python_environment,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
start_new_session=True,
)
_wait_until_ready("127.0.0.1", python_port, python)
direct_python: Final = _request("127.0.0.1", python_port, "127.0.0.2")
assert direct_python == {
"client": "127.0.0.2",
"host": "gateway.example",
"origin": "https://ui.example",
"xff": "198.51.100.7",
"proto": "https",
"scheme": "http",
}
rust_environment: Final[Mapping[str, str]] = {
**os.environ,
"HOST": "127.0.0.1",
"PORT": str(rust_port),
"LITELLM_RUST_GATEWAY": "true",
"LITELLM_PYTHON_UPSTREAM": f"http://127.0.0.1:{python_port}",
"LITELLM_RUST_GATEWAY_TRUSTED_PROXIES": trusted_proxies,
}
rust: Final = subprocess.Popen(
(str(binary),),
env=rust_environment,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
start_new_session=True,
)
try:
_wait_until_ready("127.0.0.1", rust_port, rust)
response: Final = _request("127.0.0.1", rust_port, "127.0.0.2")
assert response == {
"client": expected_client,
"host": "gateway.example",
"origin": "https://ui.example",
"xff": expected_client,
"proto": expected_proto,
"scheme": expected_proto,
}
finally:
for process in (rust, python):
if process.poll() is None:
os.killpg(process.pid, signal.SIGTERM) # test-quality-ok: Signal only this fixture's process group
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL) # test-quality-ok: Kill only this fixture's process group
process.wait(timeout=5)

View file

@ -3,6 +3,7 @@ import os
import socket
import sys
import textwrap
import time
import urllib.parse
from collections.abc import Iterator
from pathlib import Path
@ -13,9 +14,27 @@ import pytest
from uvicorn.importer import import_from_string
from uvicorn.main import main as uvicorn_main
from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv
from gateway.launch import (
GATEWAY_APP,
_pooled_child_environments,
main,
managed_pool_database_url,
pool_database_url,
uvicorn_argv,
)
from gateway.supervisor import (
RUST_GATEWAY_BINARY,
ConfigurationFailure,
build_configuration,
gateway_enabled,
)
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings
from litellm.proxy.db.pgbouncer import (
PGBOUNCER_POOLED_ENV_VAR,
ManagedPgBouncerRuntime,
PgBouncerError,
PgBouncerSettings,
)
from litellm.proxy.proxy_server import app as proxy_app
DB_ENV: Final = {
@ -127,6 +146,50 @@ class TestUvicornArgv:
class TestPoolDatabaseUrl:
def test_rust_mode_without_a_writer_stays_database_free(self, password_env: dict[str, str]) -> None:
assert (
managed_pool_database_url(
DatabaseURLSettings.from_env(),
PgBouncerSettings(enabled=True),
{"DATABASE_URL": ""},
)
is None
)
def test_rust_managed_pooler_is_forced_for_a_writer_when_the_flag_is_off(
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
port: Final = _free_port()
binary: Final = _fake_pooler(tmp_path)
monkeypatch.setenv("DATABASE_URL", "postgresql://litellm_pool:p%40ss@db.internal:5432/litellm")
outcome: Final = managed_pool_database_url(
DatabaseURLSettings.from_env(),
PgBouncerSettings(enabled=False, port=port, binary=str(binary)),
os.environ,
)
assert isinstance(outcome, ManagedPgBouncerRuntime), outcome
try:
assert urllib.parse.urlsplit(outcome.pooled_url).port == port
configuration: Final = build_configuration(
("--host", "0.0.0.0", "--port", "4000"),
{RUST_GATEWAY_BINARY: "/bin/true"},
sys.executable,
)
assert not isinstance(configuration, ConfigurationFailure)
python_environment, rust_environment = _pooled_child_environments(
configuration.python_environment,
configuration.rust_environment,
outcome,
)
assert python_environment["DATABASE_URL"] == rust_environment["DATABASE_URL"] == outcome.pooled_url
assert python_environment[PGBOUNCER_POOLED_ENV_VAR] == rust_environment[PGBOUNCER_POOLED_ENV_VAR] == "true"
assert python_environment["LITELLM_PGBOUNCER_ENABLED"] == "false"
assert rust_environment.get("LITELLM_PGBOUNCER_ENABLED") == configuration.rust_environment.get(
"LITELLM_PGBOUNCER_ENABLED"
)
finally:
outcome.stop(time.monotonic() + 2)
def test_a_disabled_pooler_yields_no_url_to_install(self, password_env: dict[str, str]):
settings: Final = DatabaseURLSettings.from_env()
settings.apply_to_env()
@ -158,6 +221,149 @@ class TestPoolDatabaseUrl:
assert "MINTED_TOKEN" not in outcome
class TestRustGatewayConfiguration:
@pytest.fixture
def rust_binary(self, tmp_path: Path) -> Path:
binary: Final = tmp_path / "litellm-gateway"
binary.write_text("#!/bin/sh\nexit 0\n")
binary.chmod(0o700)
return binary
@pytest.fixture
def clear_uvicorn_environment(self, monkeypatch: pytest.MonkeyPatch) -> None:
for name in (
"UVICORN_HOST",
"UVICORN_PORT",
"UVICORN_UDS",
"UVICORN_FD",
"UVICORN_RELOAD",
"UVICORN_PROXY_HEADERS",
"UVICORN_SSL_KEYFILE",
):
monkeypatch.delenv(name, raising=False)
def test_flag_requires_exact_lowercase_values(self) -> None:
assert gateway_enabled({}) is False
assert gateway_enabled({"LITELLM_RUST_GATEWAY": "true"}) is True
assert isinstance(gateway_enabled({"LITELLM_RUST_GATEWAY": "TRUE"}), ConfigurationFailure)
def test_private_worker_keeps_config_rootpath_keepalive_and_unknown_options(
self,
rust_binary: Path,
clear_uvicorn_environment: None,
) -> None:
environ: Final = {
RUST_GATEWAY_BINARY: str(rust_binary),
"CONFIG_FILE_PATH": "/app/config/config.yaml",
"KEEPALIVE_TIMEOUT": "41",
}
configuration: Final = build_configuration(
(
"--workers",
"3",
"--proxy-headers",
"--host",
"0.0.0.0",
"--port",
"4000",
"--root-path",
"/gateway",
"--unknown-option",
"preserved",
"--proxy-headers",
),
environ,
sys.executable,
)
assert not isinstance(configuration, ConfigurationFailure)
assert configuration.python_command == (
sys.executable,
"-m",
"gateway.launch",
"--host",
"127.0.0.1",
"--port",
"4002",
"--proxy-headers",
"--forwarded-allow-ips",
"127.0.0.1",
"--workers",
"3",
"--root-path",
"/gateway",
"--unknown-option",
"preserved",
"--timeout-keep-alive",
"41",
)
assert configuration.python_environment["CONFIG_FILE_PATH"] == "/app/config/config.yaml"
assert configuration.python_environment["LITELLM_RUST_GATEWAY"] == "false"
assert configuration.rust_environment["LITELLM_RUST_GATEWAY"] == "true"
assert configuration.rust_environment["LITELLM_PYTHON_UPSTREAM"] == "http://127.0.0.1:4002"
assert configuration.rust_environment["HOST"] == "0.0.0.0"
assert configuration.rust_environment["PORT"] == "4000"
def test_database_writer_reserves_pgbouncer_port_when_the_python_only_flag_is_false(
self, rust_binary: Path, clear_uvicorn_environment: None
) -> None:
configuration: Final = build_configuration(
("--host", "0.0.0.0", "--port", "4000"),
{
RUST_GATEWAY_BINARY: str(rust_binary),
"DATABASE_URL": "postgresql://app:pw@db/litellm",
"LITELLM_PGBOUNCER_ENABLED": "false",
"LITELLM_PGBOUNCER_PORT": "4000",
},
sys.executable,
)
assert isinstance(configuration, ConfigurationFailure)
assert "PgBouncer" in configuration.reason
assert "same TCP port" in configuration.reason
@pytest.mark.parametrize(
("arguments", "extra_environment", "reason"),
(
(("--no-proxy-headers",), {}, "--no-proxy-headers"),
(("--ssl-certfile", "/tmp/cert.pem"), {}, "TLS"),
(("--env-file", "/tmp/uvicorn.env"), {}, "--env-file"),
(("--uds", "/tmp/gateway.sock"), {}, "Unix socket"),
(("--fd", "5"), {}, "file descriptor"),
(("--http", "h2"), {}, "HTTP/1.1"),
(("--host", "0.0.0.0", "--port", "4000"), {"PORT": "4001"}, "PORT conflicts"),
(("--host", "0.0.0.0", "--port", "4000"), {"LITELLM_GATEWAY_METRICS_PORT": "4002"}, "same TCP port"),
(
("--host", "0.0.0.0", "--port", "4000"),
{"LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS": "20"},
"must allow Rust drain",
),
(
("--host", "0.0.0.0", "--port", "4000"),
{"LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS": "0"},
"must be a positive integer",
),
(
("--host", "0.0.0.0", "--port", "4000"),
{"LITELLM_COLLECTOR_ENABLED": "true", "LITELLM_COLLECTOR_ADDRESS": "tcp://127.0.0.1:4002"},
"same TCP port",
),
),
)
def test_rejects_unsupported_or_conflicting_configuration(
self,
rust_binary: Path,
clear_uvicorn_environment: None,
arguments: tuple[str, ...],
extra_environment: dict[str, str],
reason: str,
) -> None:
environ: Final = {RUST_GATEWAY_BINARY: str(rust_binary), **extra_environment}
outcome: Final = build_configuration(arguments, environ, sys.executable)
assert isinstance(outcome, ConfigurationFailure)
assert reason in outcome.reason
class TestMain:
def test_workers_inherit_the_loopback_url_the_supervisor_installed(
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path