From c1a566099946e87b9740af892d222d2245e864c6 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:12:45 +0000 Subject: [PATCH 1/4] feat(gateway): supervise opt-in Rust gateway deployments --- gateway/Dockerfile | 21 +- gateway/RUST_GATEWAY.md | 60 ++ gateway/launch.py | 14 + gateway/supervisor.py | 621 ++++++++++++++++++ helm/litellm/templates/_helpers.tpl | 8 + .../litellm/templates/gateway/deployment.yaml | 22 +- helm/litellm/tests/rust_gateway_tests.yaml | 140 ++++ litellm-rust/Cargo.lock | 1 + litellm-rust/crates/gateway/Cargo.toml | 1 + .../crates/gateway/src/dispatch/forward.rs | 17 +- litellm-rust/crates/gateway/src/error.rs | 4 + litellm-rust/crates/gateway/src/lib.rs | 1 + .../crates/gateway/src/proxy_headers.rs | 172 +++++ litellm-rust/crates/gateway/src/settings.rs | 45 +- .../test_gateway_supervisor_lifecycle.py | 272 ++++++++ .../test_rust_gateway_trusted_proxy.py | 183 ++++++ tests/unit/gateway/test_launch.py | 132 ++++ 17 files changed, 1705 insertions(+), 9 deletions(-) create mode 100644 gateway/RUST_GATEWAY.md create mode 100644 gateway/supervisor.py create mode 100644 helm/litellm/tests/rust_gateway_tests.yaml create mode 100644 litellm-rust/crates/gateway/src/proxy_headers.rs create mode 100644 tests/integration/configuration/test_gateway_supervisor_lifecycle.py create mode 100644 tests/integration/configuration/test_rust_gateway_trusted_proxy.py diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 6af25df0c5d..224fb1b7737 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -47,6 +47,20 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \ python -c "import maturin; maturin.build_wheel('/wheels')" +RUN --mount=type=cache,target=/root/.cargo/registry \ + --mount=type=cache,target=/root/.cargo/git \ + --mount=type=cache,target=/app/litellm-rust/target \ + CARGO_BIN="$(find /root/.cache /root/.cargo /root/.rustup /usr/local \( -type f -o -type l \) -name cargo -print -quit)" && \ + test -x "$CARGO_BIN" || { echo "Rust cargo executable was not found" >&2; exit 1; } && \ + cd litellm-rust && \ + PATH="$(dirname "$CARGO_BIN"):${PATH}" "$CARGO_BIN" build --locked --release -p litellm-gateway --bin litellm-gateway && \ + mkdir -p /rust-gateway && \ + cp target/release/litellm-gateway /rust-gateway/litellm-gateway && \ + readelf -l /rust-gateway/litellm-gateway > /tmp/litellm-gateway-linkage && \ + grep -q "Requesting program interpreter:" /tmp/litellm-gateway-linkage && \ + readelf -d /rust-gateway/litellm-gateway >> /tmp/litellm-gateway-linkage && \ + grep -q "(NEEDED)" /tmp/litellm-gateway-linkage + # ---------- Builder ---------- FROM $LITELLM_BUILD_IMAGE AS builder @@ -143,12 +157,13 @@ ENV HOME=/home/nonroot \ COPY --from=builder --chown=nonroot:nonroot /app /app COPY --from=builder /opt/prisma /opt/prisma COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer +COPY --from=rust-native /rust-gateway/litellm-gateway /usr/local/bin/litellm-gateway -RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ - find /app/.venv -type d -path "*/tornado/test" -delete && \ +RUN find /app/.venv -type d -path "*/tornado/test" -prune -exec rm -r {} \; && \ chmod -R a+rX /opt/prisma && \ python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \ - python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()" + python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()" && \ + if LITELLM_RUST_GATEWAY=true /usr/local/bin/litellm-gateway 2>&1 | grep -Fq "Error: PythonUpstreamConfig"; then :; else exit 1; fi USER nonroot diff --git a/gateway/RUST_GATEWAY.md b/gateway/RUST_GATEWAY.md new file mode 100644 index 00000000000..03ec7fce358 --- /dev/null +++ b/gateway/RUST_GATEWAY.md @@ -0,0 +1,60 @@ +# Opt-in Rust gateway + +The Rust gateway is an optional public HTTP listener. The existing Python gateway remains the default. Set +`LITELLM_RUST_GATEWAY=true` to start the Rust gateway on the public listener and supervise the existing Python +gateway on loopback port 4002. Set the flag to exactly `false`, or leave it unset, to keep the existing startup +path. No separate enable flag is required for the Python upstream. + +The Rust binary defaults to `/usr/local/bin/litellm-gateway`; local deployments can set +`LITELLM_RUST_GATEWAY_BINARY` to an executable binary. `LITELLM_RUST_GATEWAY_PYTHON_PORT` changes the private +Python port. The private listener must remain on loopback and must not be added to a container or Kubernetes +Service port. + +```sh +docker run --stop-timeout 30 -p 4000:4000 \ + -e LITELLM_RUST_GATEWAY=true \ + -e LITELLM_RUST_GATEWAY_TRUSTED_PROXIES=10.0.0.0/8 \ + -e LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS=30 \ + -e LITELLM_GATEWAY_METRICS_PORT=4001 \ + -e CONFIG_FILE_PATH=/app/config/config.yaml \ + litellm/litellm:main +``` + +The supervisor waits up to 120 seconds for startup, drains Rust for up to 15 seconds, then gives Python 5 seconds +to stop. It also allows a 10-second cleanup margin. `LITELLM_RUST_GATEWAY_STARTUP_TIMEOUT_SECONDS`, +`LITELLM_RUST_GATEWAY_DRAIN_TIMEOUT_SECONDS`, and `LITELLM_RUST_GATEWAY_PYTHON_GRACE_SECONDS` configure those +timeouts. `LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS` must be at least the drain timeout plus Python grace +plus the cleanup margin. Container orchestrators must grant at least this value before sending SIGKILL. For +Docker, set `--stop-timeout` to the same or a larger number; Docker's default stop timeout is shorter. + +The trusted-proxy setting is an optional comma-separated list of IP addresses or CIDRs. An unset or empty list +trusts no external proxy. Wildcard trust is rejected. The Rust gateway canonicalizes forwarded client and scheme +headers using the configured trust list. It preserves the original `Host` and `Origin` headers. Configure only +proxy addresses that are controlled by your deployment. + +The Rust gateway does not support TLS termination or Uvicorn TLS settings. Terminate TLS at the load balancer or +another trusted edge, then forward HTTP to the gateway. Uvicorn reload, Unix-domain and file-descriptor listeners, +environment-file overrides, and explicit `--no-proxy-headers` are rejected in enabled mode. + +The Helm chart keeps the Service on port 4000. Configure `LITELLM_RUST_GATEWAY` through the gateway container's +environment or `envFrom`, and set `gateway.terminationGracePeriodSeconds` to match the supervisor grace setting. +When the metrics sidecar is enabled, the chart exports its reserved port to the supervisor even when the feature +flag comes from `envFrom`, so port collisions are rejected during enabled startup. + +```yaml +gateway: + envConfigMaps: + - gateway-runtime + metricsServer: + enabled: true + port: 4001 + terminationGracePeriodSeconds: 30 +``` + +The `gateway-runtime` ConfigMap can contain `LITELLM_RUST_GATEWAY: "true"` and +`LITELLM_RUST_GATEWAY_TRUSTED_PROXIES: "10.0.0.0/8"`. The chart does not add a private Python port to the pod +ports or Service. + +To roll back, set `LITELLM_RUST_GATEWAY=false` in the same environment source and redeploy, or roll back the +Deployment revision. The feature flag is strict and disabled by default, so removing the opt-in also restores the +existing Python startup path. diff --git a/gateway/launch.py b/gateway/launch.py index d67432caee1..6bf5d1285cc 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -63,6 +63,20 @@ def _serve(argv: Sequence[str]) -> None: def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: + gateway_mode: Final = os.environ.get("LITELLM_RUST_GATEWAY") + if gateway_mode not in (None, "true", "false"): + sys.exit("LiteLLM gateway: LITELLM_RUST_GATEWAY must be exactly 'true' or 'false'") + if gateway_mode == "true": + from gateway.supervisor import ConfigurationFailure, build_configuration, supervise + + configuration: Final = build_configuration(argv, os.environ, sys.executable) + if isinstance(configuration, ConfigurationFailure): + sys.exit(f"LiteLLM gateway supervisor: {configuration.reason}") + failure: Final = supervise(configuration) + if failure is not None: + sys.exit(f"LiteLLM gateway supervisor: {failure.reason}") + return + settings: Final = DatabaseURLSettings.from_env() settings.apply_to_env() pooled_url: Final = pool_database_url(settings, PgBouncerSettings(), os.environ) diff --git a/gateway/supervisor.py b/gateway/supervisor.py new file mode 100644 index 00000000000..a8c8d8d40c9 --- /dev/null +++ b/gateway/supervisor.py @@ -0,0 +1,621 @@ +from __future__ import annotations + +import ctypes +import http.client +import ipaddress +import os +import signal +import socket +import subprocess +import sys +import time +from collections.abc import Callable, Mapping, Sequence +from contextlib import ExitStack +from dataclasses import dataclass +from pathlib import Path +from threading import Event +from types import MappingProxyType +from types import FrameType +from typing import Final, Protocol, cast +from urllib.parse import urlsplit + +import click +from pydantic import TypeAdapter +from uvicorn.main import main as uvicorn_main + +from gateway.launch import uvicorn_argv + +RUST_GATEWAY_FLAG: Final = "LITELLM_RUST_GATEWAY" +RUST_GATEWAY_BINARY: Final = "LITELLM_RUST_GATEWAY_BINARY" +PYTHON_PORT_ENV: Final = "LITELLM_RUST_GATEWAY_PYTHON_PORT" +PYTHON_UPSTREAM_ENV: Final = "LITELLM_PYTHON_UPSTREAM" +TRUSTED_PROXIES_ENV: Final = "LITELLM_RUST_GATEWAY_TRUSTED_PROXIES" +METRICS_PORT_ENV: Final = "LITELLM_GATEWAY_METRICS_PORT" +TERMINATION_GRACE_ENV: Final = "LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS" +DEFAULT_RUST_GATEWAY_BINARY: Final = "/usr/local/bin/litellm-gateway" +DEFAULT_PRIVATE_PORT: Final = 4002 +DEFAULT_PYTHON_GRACE_SECONDS: Final = 5 +DEFAULT_RUST_DRAIN_SECONDS: Final = 15 +DEFAULT_STARTUP_TIMEOUT_SECONDS: Final = 120 +DEFAULT_TERMINATION_GRACE_SECONDS: Final = 30 +TERMINATION_CLEANUP_MARGIN_SECONDS: Final = 10 +LIVENESS_PATH: Final = "/health/liveliness" + + +@dataclass(frozen=True, slots=True) +class ConfigurationFailure: + reason: str + + +@dataclass(frozen=True, slots=True) +class SupervisorFailure: + reason: str + + +@dataclass(frozen=True, slots=True) +class SupervisorConfig: + python_command: tuple[str, ...] + rust_command: tuple[str, ...] + python_environment: Mapping[str, str] + rust_environment: Mapping[str, str] + public_probe_host: str + public_port: int + private_port: int + startup_timeout: int + rust_drain_timeout: int + python_grace_timeout: int + termination_grace_timeout: int = DEFAULT_TERMINATION_GRACE_SECONDS + + +class ChildProcess(Protocol): + @property + def pid(self) -> int: ... + + def poll(self) -> int | None: ... + + def wait(self, timeout: float | None = None) -> int: ... + + +class ProcessRunner(Protocol): + def start(self, command: tuple[str, ...], environment: Mapping[str, str]) -> ChildProcess: ... + + +class SubprocessRunner: + def start(self, command: tuple[str, ...], environment: Mapping[str, str]) -> ChildProcess: + return subprocess.Popen(command, env=environment, start_new_session=True) + + +def gateway_enabled(environ: Mapping[str, str]) -> bool | ConfigurationFailure: + value: Final = environ.get(RUST_GATEWAY_FLAG) + if value is None or value == "false": + return False + if value == "true": + return True + return ConfigurationFailure(f"{RUST_GATEWAY_FLAG} must be exactly 'true' or 'false'") + + +def _port_setting(name: str, default: int, environ: Mapping[str, str]) -> int | ConfigurationFailure: + value: Final = environ.get(name) + if value is None: + return default + try: + port: Final = int(value) + except ValueError: + return ConfigurationFailure(f"{name} must be an integer port") + if not 1 <= port <= 65535: + return ConfigurationFailure(f"{name} must be between 1 and 65535") + return port + + +def _timeout_setting(name: str, default: int, environ: Mapping[str, str]) -> int | ConfigurationFailure: + value: Final = environ.get(name) + if value is None: + return default + try: + timeout: Final = int(value) + except ValueError: + return ConfigurationFailure(f"{name} must be a positive integer") + if timeout < 1: + return ConfigurationFailure(f"{name} must be a positive integer") + return timeout + + +def _boolean_setting(name: str, environ: Mapping[str, str]) -> bool | ConfigurationFailure: + value: Final = environ.get(name) + if value is None: + return False + normalized: Final = value.casefold() + if normalized in ("true", "1", "yes", "y", "on", "t"): + return True + if normalized in ("false", "0", "no", "n", "off", "f"): + return False + return ConfigurationFailure(f"{name} must be a boolean") + + +def _collector_port(environ: Mapping[str, str]) -> int | ConfigurationFailure | None: + enabled: Final = _boolean_setting("LITELLM_COLLECTOR_ENABLED", environ) + if isinstance(enabled, ConfigurationFailure) or not enabled: + return enabled if isinstance(enabled, ConfigurationFailure) else None + address: Final = environ.get("LITELLM_COLLECTOR_ADDRESS") + if address is None or not address.startswith("tcp://"): + return None + try: + parsed: Final = urlsplit(address) + port: Final = parsed.port + except ValueError: + return ConfigurationFailure("LITELLM_COLLECTOR_ADDRESS must contain a valid TCP port") + if port is None: + return ConfigurationFailure("LITELLM_COLLECTOR_ADDRESS must contain a valid TCP port") + if not 1 <= port <= 65535: + return ConfigurationFailure("LITELLM_COLLECTOR_ADDRESS must contain a valid TCP port") + return port + + +def _trusted_proxies(environ: Mapping[str, str]) -> str | ConfigurationFailure: + value: Final = environ.get(TRUSTED_PROXIES_ENV, "") + if not value: + return value + try: + networks: Final = tuple(ipaddress.ip_network(entry.strip(), strict=False) for entry in value.split(",")) + except ValueError: + return ConfigurationFailure(f"{TRUSTED_PROXIES_ENV} must contain IP addresses or CIDRs") + if any(network.prefixlen == 0 for network in networks): + return ConfigurationFailure(f"{TRUSTED_PROXIES_ENV} must not trust every address") + if any(not entry.strip() for entry in value.split(",")): + return ConfigurationFailure(f"{TRUSTED_PROXIES_ENV} contains an empty entry") + return value + + +def _worker_arguments(arguments: tuple[str, ...], private_port: int) -> tuple[str, ...]: + value_options: Final = frozenset(("--host", "--port", "--forwarded-allow-ips")) + boolean_options: Final = frozenset(("--proxy-headers",)) + consumed_values: Final = frozenset( + index + 1 for index, argument in enumerate(arguments[:-1]) if argument in value_options + ) + without_overrides: Final = tuple( + argument + for index, argument in enumerate(arguments) + if index not in consumed_values + and not any( + argument == option or argument.startswith(f"{option}=") for option in value_options | boolean_options + ) + ) + return ( + "--host", + "127.0.0.1", + "--port", + str(private_port), + "--proxy-headers", + "--forwarded-allow-ips", + "127.0.0.1", + *without_overrides[1:], + ) + + +def _context_value(context: click.Context, name: str) -> object: + parameters: Final = TypeAdapter(dict[str, object]).validate_python(context.params) + value: Final = parameters[name] + return value + + +def _first_port_collision(ports: tuple[tuple[str, int], ...]) -> tuple[str, str] | None: + for index, current in enumerate(ports): + for other in ports[index + 1 :]: + if current[1] == other[1]: + return current[0], other[0] + return None + + +def build_configuration( + argv: Sequence[str], + environ: Mapping[str, str], + python_executable: str, +) -> SupervisorConfig | ConfigurationFailure: + arguments: Final = uvicorn_argv(argv, environ) + try: + context: Final = uvicorn_main.make_context( + "uvicorn", + list(arguments), + allow_extra_args=True, + ignore_unknown_options=True, + ) + except click.ClickException as error: + return ConfigurationFailure(f"invalid Uvicorn options: {error.format_message()}") + + if any(argument == "--no-proxy-headers" for argument in arguments): + return ConfigurationFailure("--no-proxy-headers is unsupported with the Rust gateway") + if any(argument.startswith("--ssl-") for argument in arguments): + return ConfigurationFailure("Uvicorn TLS options are unsupported with the Rust gateway") + if any(name.startswith("UVICORN_SSL_") for name in environ): + return ConfigurationFailure("UVICORN_SSL_* settings are unsupported with the Rust gateway") + if _context_value(context, "env_file") is not None: + return ConfigurationFailure("--env-file is unsupported with the Rust gateway") + if _context_value(context, "uds") is not None or _context_value(context, "fd") is not None: + return ConfigurationFailure("Uvicorn Unix socket and file descriptor listeners are unsupported") + if _context_value(context, "reload") is True: + return ConfigurationFailure("Uvicorn reload is unsupported with the Rust gateway") + if _context_value(context, "proxy_headers") is not True: + return ConfigurationFailure("Uvicorn proxy headers must be enabled with the Rust gateway") + + protocol: Final = _context_value(context, "http") + if not isinstance(protocol, str) or protocol not in ("auto", "h11", "httptools"): + return ConfigurationFailure("the Rust gateway supports only Uvicorn HTTP/1.1 protocols") + + host: Final = _context_value(context, "host") + port: Final = _context_value(context, "port") + if ( + not isinstance(host, str) + or not host + or any(character.isspace() or character in "/[]\r\n" for character in host) + ): + return ConfigurationFailure("Uvicorn must have a non-empty public host") + if not isinstance(port, int) or isinstance(port, bool) or not 1 <= port <= 65535: + return ConfigurationFailure("Uvicorn public port must be between 1 and 65535") + if "HOST" in environ and environ["HOST"] != host: + return ConfigurationFailure("HOST conflicts with the effective Uvicorn host") + if "PORT" in environ and environ["PORT"] != str(port): + return ConfigurationFailure("PORT conflicts with the effective Uvicorn port") + + private_port: Final = _port_setting(PYTHON_PORT_ENV, DEFAULT_PRIVATE_PORT, environ) + if isinstance(private_port, ConfigurationFailure): + return private_port + metrics_port: Final = _port_setting(METRICS_PORT_ENV, 0, environ) if METRICS_PORT_ENV in environ else None + if isinstance(metrics_port, ConfigurationFailure): + return metrics_port + pgbouncer_enabled: Final = _boolean_setting("LITELLM_PGBOUNCER_ENABLED", environ) + if isinstance(pgbouncer_enabled, ConfigurationFailure): + return pgbouncer_enabled + pgbouncer_port: Final = _port_setting("LITELLM_PGBOUNCER_PORT", 6432, environ) if pgbouncer_enabled else None + if isinstance(pgbouncer_port, ConfigurationFailure): + return pgbouncer_port + collector_port: Final = _collector_port(environ) + if isinstance(collector_port, ConfigurationFailure): + return collector_port + startup_timeout: Final = _timeout_setting( + "LITELLM_RUST_GATEWAY_STARTUP_TIMEOUT_SECONDS", + DEFAULT_STARTUP_TIMEOUT_SECONDS, + environ, + ) + if isinstance(startup_timeout, ConfigurationFailure): + return startup_timeout + rust_drain_timeout: Final = _timeout_setting( + "LITELLM_RUST_GATEWAY_DRAIN_TIMEOUT_SECONDS", + DEFAULT_RUST_DRAIN_SECONDS, + environ, + ) + if isinstance(rust_drain_timeout, ConfigurationFailure): + return rust_drain_timeout + python_grace_timeout: Final = _timeout_setting( + "LITELLM_RUST_GATEWAY_PYTHON_GRACE_SECONDS", + DEFAULT_PYTHON_GRACE_SECONDS, + environ, + ) + if isinstance(python_grace_timeout, ConfigurationFailure): + return python_grace_timeout + termination_grace_timeout: Final = _timeout_setting( + TERMINATION_GRACE_ENV, + DEFAULT_TERMINATION_GRACE_SECONDS, + environ, + ) + if isinstance(termination_grace_timeout, ConfigurationFailure): + return termination_grace_timeout + if rust_drain_timeout + python_grace_timeout + TERMINATION_CLEANUP_MARGIN_SECONDS > termination_grace_timeout: + return ConfigurationFailure(f"{TERMINATION_GRACE_ENV} must allow Rust drain, Python grace, and cleanup margin") + trusted_proxies: Final = _trusted_proxies(environ) + if isinstance(trusted_proxies, ConfigurationFailure): + return trusted_proxies + + collisions: Final = _first_port_collision( + ( + ("public gateway", port), + ("private Python upstream", private_port), + *(() if metrics_port is None else (("metrics server", metrics_port),)), + *(() if pgbouncer_port is None else (("PgBouncer", pgbouncer_port),)), + *(() if collector_port is None else (("collector", collector_port),)), + ) + ) + if collisions is not None: + return ConfigurationFailure(f"{collisions[0]} and {collisions[1]} use the same TCP port") + + binary: Final = environ.get(RUST_GATEWAY_BINARY, DEFAULT_RUST_GATEWAY_BINARY) + binary_path: Final = Path(binary) + if not binary_path.is_file() or not os.access(binary_path, os.X_OK): + return ConfigurationFailure(f"Rust gateway binary is not executable: {binary}") + + upstream: Final = f"http://127.0.0.1:{private_port}" + configured_upstream: Final = environ.get(PYTHON_UPSTREAM_ENV) + if configured_upstream is not None and configured_upstream != upstream: + return ConfigurationFailure(f"{PYTHON_UPSTREAM_ENV} must equal {upstream}") + + python_environment: Final = MappingProxyType( + { + **environ, + RUST_GATEWAY_FLAG: "false", + } + ) + rust_environment: Final = MappingProxyType( + { + **environ, + RUST_GATEWAY_FLAG: "true", + PYTHON_UPSTREAM_ENV: upstream, + TRUSTED_PROXIES_ENV: trusted_proxies, + "HOST": host, + "PORT": str(port), + } + ) + probe_host: Final = "127.0.0.1" if host in ("0.0.0.0", "*") else "::1" if host == "::" else host + python_command: Final = ( + python_executable, + "-m", + "gateway.launch", + *_worker_arguments(arguments, private_port), + ) + return SupervisorConfig( + python_command=python_command, + rust_command=(binary,), + python_environment=python_environment, + rust_environment=rust_environment, + public_probe_host=probe_host, + public_port=port, + private_port=private_port, + startup_timeout=startup_timeout, + rust_drain_timeout=rust_drain_timeout, + python_grace_timeout=python_grace_timeout, + termination_grace_timeout=termination_grace_timeout, + ) + + +def _enable_subreaper() -> SupervisorFailure | None: + if not sys.platform.startswith("linux"): + return SupervisorFailure("the Rust gateway supervisor requires Linux") + prctl: Final = ctypes.CDLL(None, use_errno=True).prctl + if prctl(36, 1, 0, 0, 0) != 0: + return SupervisorFailure("could not enable Linux child subreaper mode") + return None + + +def _install_signal_handlers( + requested: Event, +) -> tuple[tuple[signal.Signals, object], ...]: + signals: Final = (signal.SIGTERM, signal.SIGINT) + previous: Final = tuple((handled, signal.getsignal(handled)) for handled in signals) + + def request_shutdown(_signum: int, _frame: FrameType | None) -> None: + requested.set() + + for handled in signals: + signal.signal(handled, request_shutdown) + return previous + + +def _restore_signal_handlers( + previous: tuple[tuple[signal.Signals, object], ...], +) -> None: + for handled, handler in previous: + signal.signal(handled, cast(signal.Handlers | Callable[[int, FrameType | None], object] | None, handler)) + + +def _tcp_ready(host: str, port: int) -> bool: + try: + with socket.create_connection((host, port), timeout=0.2): + return True + except OSError: + return False + + +def _http_ready(host: str, port: int) -> bool: + connection: Final = http.client.HTTPConnection(host, port, timeout=0.3) + try: + connection.request("GET", LIVENESS_PATH) + response: Final = connection.getresponse() + response.read() + return response.status == 200 + except (http.client.HTTPException, OSError): + return False + finally: + connection.close() + + +def _group_exists(process_group: int) -> bool: + try: + os.killpg(process_group, 0) + return True + except ProcessLookupError: + return False + except PermissionError: + return True + + +def _signal_group(process_group: int, requested: signal.Signals) -> None: + try: + os.killpg(process_group, requested) + except ProcessLookupError: + pass + + +def _reap_adopted_children(excluded: frozenset[int]) -> None: + children_path: Final = Path(f"/proc/self/task/{os.getpid()}/children") + try: + children: Final = tuple(int(pid) for pid in children_path.read_text().split() if int(pid) not in excluded) + except (OSError, ValueError): + return + for child in children: + try: + os.waitpid(child, os.WNOHANG) + except ChildProcessError: + pass + + +def _adopted_children(excluded: frozenset[int]) -> tuple[int, ...]: + children_path: Final = Path(f"/proc/self/task/{os.getpid()}/children") + try: + return tuple(int(pid) for pid in children_path.read_text().split() if int(pid) not in excluded) + except (OSError, ValueError): + return () + + +def _signal_adopted_child(child: int, requested: signal.Signals) -> None: + process_group: Final = os.getpgid(child) + if process_group == os.getpgrp(): + os.kill(child, requested) + else: + os.killpg(process_group, requested) + + +def _terminate_adopted_children(excluded: frozenset[int]) -> None: + children: Final = _adopted_children(excluded) + for child in children: + try: + _signal_adopted_child(child, signal.SIGTERM) + except ProcessLookupError: + pass + deadline: Final = time.monotonic() + 1 + while time.monotonic() < deadline and _adopted_children(excluded): + _reap_adopted_children(excluded) + time.sleep(0.05) + for child in _adopted_children(excluded): + try: + _signal_adopted_child(child, signal.SIGKILL) + except ProcessLookupError: + pass + final_deadline: Final = time.monotonic() + 1 + while time.monotonic() < final_deadline and _adopted_children(excluded): + _reap_adopted_children(excluded) + time.sleep(0.05) + _reap_adopted_children(excluded) + + +def _wait_for_group( + process: ChildProcess, + deadline: float, + protected_children: frozenset[int], +) -> bool: + while time.monotonic() < deadline and _group_exists(process.pid): + process.poll() + _reap_adopted_children(protected_children) + time.sleep(0.05) + process.poll() + _reap_adopted_children(protected_children) + return not _group_exists(process.pid) + + +def _stop_process_group( + process: ChildProcess, + timeout: int, + protected_children: frozenset[int], +) -> None: + _signal_group(process.pid, signal.SIGTERM) + if not _wait_for_group(process, time.monotonic() + timeout, protected_children): + _signal_group(process.pid, signal.SIGKILL) + _wait_for_group(process, time.monotonic() + 1, protected_children) + try: + process.wait(timeout=1) + except subprocess.TimeoutExpired: + _signal_group(process.pid, signal.SIGKILL) + try: + process.wait(timeout=1) + except subprocess.TimeoutExpired: + pass + _reap_adopted_children(protected_children) + + +def _wait_for_python( + process: ChildProcess, + port: int, + deadline: float, + requested: Event, +) -> bool | SupervisorFailure: + while not requested.is_set() and time.monotonic() < deadline: + if (status := process.poll()) is not None: + return SupervisorFailure(f"Python gateway exited during startup with status {status}") + if _tcp_ready("127.0.0.1", port): + return True + time.sleep(0.05) + return False + + +def _wait_for_rust( + python: ChildProcess, + rust: ChildProcess, + host: str, + port: int, + deadline: float, + requested: Event, +) -> bool | SupervisorFailure: + while not requested.is_set() and time.monotonic() < deadline: + if (python_status := python.poll()) is not None: + return SupervisorFailure(f"Python gateway exited during startup with status {python_status}") + if (rust_status := rust.poll()) is not None: + return SupervisorFailure(f"Rust gateway exited during startup with status {rust_status}") + if _http_ready(host, port): + return True + time.sleep(0.05) + return False + + +def _monitor_children(python: ChildProcess, rust: ChildProcess, requested: Event) -> SupervisorFailure | None: + while not requested.wait(0.1): + if (python_status := python.poll()) is not None: + return SupervisorFailure(f"Python gateway exited unexpectedly with status {python_status}") + if (rust_status := rust.poll()) is not None: + return SupervisorFailure(f"Rust gateway exited unexpectedly with status {rust_status}") + return None + + +def supervise(config: SupervisorConfig, runner: ProcessRunner | None = None) -> SupervisorFailure | None: + subreaper_failure: Final = _enable_subreaper() + if subreaper_failure is not None: + return subreaper_failure + process_runner: Final = runner if runner is not None else SubprocessRunner() + requested: Final = Event() + previous_handlers: Final = _install_signal_handlers(requested) + try: + with ExitStack() as cleanup: + try: + python: Final = process_runner.start(config.python_command, config.python_environment) + except OSError as error: + return SupervisorFailure(f"could not start Python gateway: {error}") + cleanup.callback( + _stop_process_group, + python, + config.python_grace_timeout, + frozenset((python.pid,)), + ) + deadline: Final = time.monotonic() + config.startup_timeout + python_ready: Final = _wait_for_python(python, config.private_port, deadline, requested) + if isinstance(python_ready, SupervisorFailure): + return python_ready + if not python_ready: + return None if requested.is_set() else SupervisorFailure("Python gateway startup timed out") + if requested.is_set(): + return None + try: + rust: Final = process_runner.start(config.rust_command, config.rust_environment) + except OSError as error: + return SupervisorFailure(f"could not start Rust gateway: {error}") + cleanup.callback( + _stop_process_group, + rust, + config.rust_drain_timeout, + frozenset((python.pid, rust.pid)), + ) + rust_ready: Final = _wait_for_rust( + python, + rust, + config.public_probe_host, + config.public_port, + deadline, + requested, + ) + if isinstance(rust_ready, SupervisorFailure): + return rust_ready + if not rust_ready: + return None if requested.is_set() else SupervisorFailure("Rust gateway readiness timed out") + return _monitor_children(python, rust, requested) + finally: + _terminate_adopted_children(frozenset()) + _restore_signal_handlers(previous_handlers) + + +if __name__ == "__main__": + sys.exit("run the Rust gateway through python -m gateway.launch") diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 8cb2fbf441d..ecb7e3375dc 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -599,6 +599,14 @@ ImplementationSpecific {{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}} +{{- define "litellm.gateway.rustGatewayEnabled" -}} +{{- range .Values.gateway.extraEnv -}} +{{- if and (eq .name "LITELLM_RUST_GATEWAY") (eq (toString .value) "true") -}} +true +{{- end -}} +{{- end -}} +{{- end -}} + {{/* Directory of the collector's unix socket, shared by the gateway and collector containers through an emptyDir. Empty when the sidecar is off diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index b0e7f0dcebb..0e2ca35bcd5 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -1,4 +1,16 @@ {{- if .Values.gateway.enabled }} +{{- $rustGatewayEnabled := eq (include "litellm.gateway.rustGatewayEnabled" . | trim) "true" }} +{{- $pythonPort := 4002 }} +{{- $configuredGrace := .Values.gateway.terminationGracePeriodSeconds }} +{{- $gracePeriod := 30 }} +{{- if not (or (kindIs "invalid" $configuredGrace) (eq (printf "%v" $configuredGrace) "")) }} +{{- $gracePeriod = int $configuredGrace }} +{{- end }} +{{- range .Values.gateway.extraEnv }} +{{- if and (eq .name "LITELLM_RUST_GATEWAY_PYTHON_PORT") .value }} +{{- $pythonPort = int .value }} +{{- end }} +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: @@ -79,9 +91,16 @@ spec: {{- if eq (int .Values.gateway.metricsServer.port) 4000 }} {{- fail "gateway.metricsServer.port must differ from the gateway port 4000" }} {{- end }} + {{- if and $rustGatewayEnabled (eq (int .Values.gateway.metricsServer.port) $pythonPort) }} + {{- fail "gateway.metricsServer.port must differ from the private Python gateway port" }} + {{- end }} - name: PROMETHEUS_MULTIPROC_DIR value: {{ include "litellm.gateway.prometheusMultiprocDir" . }} + - name: LITELLM_GATEWAY_METRICS_PORT + value: {{ .Values.gateway.metricsServer.port | quote }} {{- end }} + - name: LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS + value: {{ $gracePeriod | quote }} {{- if .Values.gateway.collector.enabled }} {{- include "litellm.gateway.collectorEnv" . | nindent 12 }} {{- end }} @@ -245,8 +264,5 @@ spec: topologySpreadConstraints: {{- toYaml . | nindent 8 }} {{- end }} - {{- $gracePeriod := .Values.gateway.terminationGracePeriodSeconds }} - {{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }} terminationGracePeriodSeconds: {{ $gracePeriod }} - {{- end }} {{- end }} diff --git a/helm/litellm/tests/rust_gateway_tests.yaml b/helm/litellm/tests/rust_gateway_tests.yaml new file mode 100644 index 00000000000..31c5e936f7d --- /dev/null +++ b/helm/litellm/tests/rust_gateway_tests.yaml @@ -0,0 +1,140 @@ +suite: test opt-in Rust gateway deployment +templates: + - gateway/deployment.yaml + - gateway/service.yaml +values: + - ./values/required.yaml +tests: + - it: leaves HTTP probes and service ports unchanged when disabled + set: + gateway.metricsServer.enabled: true + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /health/liveliness + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_GATEWAY_METRICS_PORT + value: "4001" + any: true + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].ports[0].containerPort + value: 4000 + template: gateway/deployment.yaml + - equal: + path: spec.ports[0].port + value: 4000 + template: gateway/service.yaml + + - it: passes enabled gateway settings and keeps the private listener off Services + set: + gateway.extraEnv: + - name: LITELLM_RUST_GATEWAY + value: "true" + gateway.envConfigMaps: + - gateway-runtime + gateway.metricsServer.enabled: true + gateway.metricsServer.port: 4001 + gateway.collector.enabled: true + gateway.collector.address: tcp://127.0.0.1:4102 + gateway.terminationGracePeriodSeconds: 30 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_RUST_GATEWAY + value: "true" + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_GATEWAY_METRICS_PORT + value: "4001" + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ADDRESS + value: tcp://127.0.0.1:4102 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].envFrom[0].configMapRef.name + value: gateway-runtime + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /health/liveliness + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /health/readiness + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].ports[0].containerPort + value: 4000 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 30 + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS + value: "30" + template: gateway/deployment.yaml + - equal: + path: spec.ports[0].port + value: 4000 + template: gateway/service.yaml + + - it: rejects a metrics port that collides with the default private Python listener + template: gateway/deployment.yaml + set: + gateway.extraEnv: + - name: LITELLM_RUST_GATEWAY + value: "true" + gateway.metricsServer.enabled: true + gateway.metricsServer.port: 4002 + asserts: + - failedTemplate: + errorPattern: gateway.metricsServer.port must differ from the private Python gateway port + + - it: exports metrics collision metadata when the flag comes from envFrom + set: + gateway.envConfigMaps: + - gateway-runtime + gateway.metricsServer.enabled: true + gateway.metricsServer.port: 4001 + asserts: + - equal: + path: spec.template.spec.containers[0].envFrom[0].configMapRef.name + value: gateway-runtime + template: gateway/deployment.yaml + + - it: preserves an explicit zero termination grace for enabled-mode rejection + set: + gateway.extraEnv: + - name: LITELLM_RUST_GATEWAY + value: "true" + gateway.terminationGracePeriodSeconds: 0 + asserts: + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 0 + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS + value: "0" + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_GATEWAY_METRICS_PORT + value: "4001" + template: gateway/deployment.yaml diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 159c43aa717..b14bf6a3f1d 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3767,6 +3767,7 @@ dependencies = [ "http-body-util", "hyper 1.10.1", "hyper-util", + "ipnet", "litellm-auth-types", "litellm-config", "litellm-core-utils", diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index c87824c459a..e93b23d901f 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -12,6 +12,7 @@ envy = "0.4.2" http-body-util = "0.1" hyper = { version = "1", features = ["http1"] } hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] } +ipnet = "2.12" litellm-core-utils.workspace = true litellm-inference.workspace = true litellm-gateway-inference.workspace = true diff --git a/litellm-rust/crates/gateway/src/dispatch/forward.rs b/litellm-rust/crates/gateway/src/dispatch/forward.rs index b31587c96f8..d631014b42e 100644 --- a/litellm-rust/crates/gateway/src/dispatch/forward.rs +++ b/litellm-rust/crates/gateway/src/dispatch/forward.rs @@ -53,6 +53,7 @@ pub struct InvalidUpstream { pub struct Forward { client: Arc>, authority: Authority, + trusted_proxies: Option, } impl Forward { @@ -72,9 +73,15 @@ impl Forward { Ok(Self { client: Arc::new(Client::builder(TokioExecutor::new()).build(HttpConnector::new())), authority, + trusted_proxies: None, }) } + pub fn with_trusted_proxies(mut self, proxies: &str) -> Result { + self.trusted_proxies = Some(crate::proxy_headers::TrustedProxies::parse(proxies)?); + Ok(self) + } + pub(super) async fn send(&self, mut request: Request, decline: Decline) -> Response { let websocket = if request.method() == Method::GET && request.version() == Version::HTTP_11 @@ -126,8 +133,14 @@ impl Forward { if websocket.is_some() { websocket_headers(&mut parts.headers); } - if let Some(ConnectInfo(client)) = parts.extensions.get::>() { - append_forwarded_for(&mut parts.headers, *client); + let peer = parts + .extensions + .get::>() + .map(|info| info.0); + if let Some(policy) = &self.trusted_proxies { + policy.apply(&mut parts.headers, peer); + } else if let Some(client) = peer { + append_forwarded_for(&mut parts.headers, client); } parts .headers diff --git a/litellm-rust/crates/gateway/src/error.rs b/litellm-rust/crates/gateway/src/error.rs index 8149be4a157..25b68843d90 100644 --- a/litellm-rust/crates/gateway/src/error.rs +++ b/litellm-rust/crates/gateway/src/error.rs @@ -1,5 +1,9 @@ #[derive(Debug, thiserror::Error)] pub enum Error { + #[error( + "LITELLM_RUST_GATEWAY_TRUSTED_PROXIES must contain comma-separated IP addresses or CIDRs, not wildcard trust" + )] + TrustedProxies, #[error( "LITELLM_RUST_GATEWAY=true requires LITELLM_PYTHON_UPSTREAM to be a valid http:// origin" )] diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 3435fcbf62e..6e2b43454e9 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -1,6 +1,7 @@ mod auth; mod dispatch; mod error; +mod proxy_headers; mod secrets; mod upgrades; pub use dispatch::{Fallback, Forward, InvalidUpstream}; diff --git a/litellm-rust/crates/gateway/src/proxy_headers.rs b/litellm-rust/crates/gateway/src/proxy_headers.rs new file mode 100644 index 00000000000..4337da91fd3 --- /dev/null +++ b/litellm-rust/crates/gateway/src/proxy_headers.rs @@ -0,0 +1,172 @@ +use std::net::{IpAddr, SocketAddr}; + +use axum::http::{HeaderMap, HeaderValue}; +use ipnet::IpNet; + +#[derive(Clone)] +pub(crate) struct TrustedProxies(Vec); + +impl TrustedProxies { + pub(crate) fn parse(value: &str) -> Result { + if value.trim().is_empty() { + return Ok(Self(Vec::new())); + } + value + .split(',') + .map(|value| { + let value = value.trim(); + let network = value + .parse::() + .or_else(|_| value.parse::().map(IpNet::from)) + .map_err(|_| crate::Error::TrustedProxies)?; + if network.prefix_len() == 0 { + return Err(crate::Error::TrustedProxies); + } + Ok(network) + }) + .collect::, _>>() + .map(Self) + } + + fn contains(&self, address: IpAddr) -> bool { + self.0.iter().any(|network| network.contains(&address)) + } + + pub(crate) fn apply(&self, headers: &mut HeaderMap, peer: Option) { + let trusted = peer.is_some_and(|peer| self.contains(peer.ip())); + let chain = trusted + .then(|| { + headers + .get_all("x-forwarded-for") + .iter() + .map(|value| value.to_str().ok()) + .collect::>>() + .and_then(|values| { + values + .iter() + .flat_map(|value| value.split(',')) + .map(|value| value.trim().parse::().ok()) + .collect::>>() + }) + }) + .flatten(); + let client = chain + .as_ref() + .and_then(|chain| { + chain + .iter() + .rev() + .find(|address| !self.contains(**address)) + .or_else(|| chain.first()) + .copied() + }) + .or_else(|| peer.map(|peer| peer.ip())); + let scheme = if trusted && headers.get_all("x-forwarded-proto").iter().count() == 1 { + match headers + .get("x-forwarded-proto") + .and_then(|value| value.to_str().ok()) + { + Some("https") => "https", + _ => "http", + } + } else { + "http" + }; + for name in [ + "forwarded", + "x-forwarded-for", + "x-forwarded-proto", + "x-forwarded-host", + "x-forwarded-port", + "x-forwarded-prefix", + "x-real-ip", + ] { + headers.remove(name); + } + if let Some(client) = client + && let Ok(value) = HeaderValue::from_str(&client.to_string()) + { + headers.insert("x-forwarded-for", value); + } + headers.insert("x-forwarded-proto", HeaderValue::from_static(scheme)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[rstest] + #[case::direct("", "127.0.0.2", "198.51.100.7", "https", "127.0.0.2", "http")] + #[case::trusted( + "127.0.0.2", + "127.0.0.2", + "198.51.100.7", + "https", + "198.51.100.7", + "https" + )] + #[case::chain( + "10.0.0.0/8", + "10.0.0.1", + "192.0.2.1, 198.51.100.7, 10.0.0.2", + "https", + "198.51.100.7", + "https" + )] + #[case::invalid_chain( + "10.0.0.0/8", + "10.0.0.1", + "198.51.100.7, unknown", + "https", + "10.0.0.1", + "https" + )] + #[case::invalid_scheme( + "127.0.0.2", + "127.0.0.2", + "198.51.100.7", + "https,http", + "198.51.100.7", + "http" + )] + #[case::ipv6("fd00::/8", "fd00::1", "2001:db8::7", "https", "2001:db8::7", "https")] + fn normalizes_proxy_identity( + #[case] trusted: &str, + #[case] peer: &str, + #[case] chain: &str, + #[case] proto: &str, + #[case] client: &str, + #[case] scheme: &str, + ) { + let mut headers = HeaderMap::new(); + headers.insert("host", HeaderValue::from_static("gateway.example")); + headers.insert("origin", HeaderValue::from_static("https://app.example")); + headers.insert("x-forwarded-for", HeaderValue::from_str(chain).unwrap()); + headers.insert("x-forwarded-proto", HeaderValue::from_str(proto).unwrap()); + headers.insert("forwarded", HeaderValue::from_static("proto=https")); + TrustedProxies::parse(trusted).unwrap().apply( + &mut headers, + Some(SocketAddr::new(peer.parse().unwrap(), 1234)), + ); + assert_eq!(headers["x-forwarded-for"], client); + assert_eq!(headers["x-forwarded-proto"], scheme); + assert_eq!(headers["host"], "gateway.example"); + assert_eq!(headers["origin"], "https://app.example"); + assert!(!headers.contains_key("forwarded")); + } + + #[rstest] + #[case("*")] + #[case("0.0.0.0/0")] + #[case("::/0")] + #[case("127.0.0.1,")] + #[case("proxy.example")] + fn invalid_policy_fails_closed(#[case] value: &str) { + assert!(matches!( + TrustedProxies::parse(value), + Err(crate::Error::TrustedProxies) + )); + } +} diff --git a/litellm-rust/crates/gateway/src/settings.rs b/litellm-rust/crates/gateway/src/settings.rs index f549f18131c..e2ff9f92133 100644 --- a/litellm-rust/crates/gateway/src/settings.rs +++ b/litellm-rust/crates/gateway/src/settings.rs @@ -12,6 +12,7 @@ pub(super) struct Settings { pub litellm_config: String, pub litellm_python_upstream: Option, pub litellm_rust_gateway: bool, + pub litellm_rust_gateway_trusted_proxies: Option, rust_log: Option, } @@ -23,6 +24,7 @@ impl Default for Settings { litellm_config: "config.yaml".into(), litellm_python_upstream: None, litellm_rust_gateway: false, + litellm_rust_gateway_trusted_proxies: None, rust_log: None, } } @@ -51,7 +53,11 @@ impl Settings { { return Err(invalid()); } - litellm_gateway::Forward::new(upstream).map_err(|_| invalid()) + let forward = litellm_gateway::Forward::new(upstream).map_err(|_| invalid())?; + match self.litellm_rust_gateway_trusted_proxies.as_deref() { + Some(proxies) => forward.with_trusted_proxies(proxies), + None => Ok(forward), + } } pub fn log_level(&self) -> Level { @@ -192,6 +198,43 @@ mod tests { ); } + #[rstest] + #[case::absent(None)] + #[case::empty(Some(""))] + #[case::explicit_trust(Some("127.0.0.2"))] + fn trusted_proxy_configuration_keeps_absent_and_empty_distinct(#[case] value: Option<&str>) { + let settings: Settings = match value { + Some(trusted) => parse(&[ + ("LITELLM_PYTHON_UPSTREAM", "http://127.0.0.1:4002"), + ("LITELLM_RUST_GATEWAY_TRUSTED_PROXIES", trusted), + ]) + .unwrap(), + None => parse(&[("LITELLM_PYTHON_UPSTREAM", "http://127.0.0.1:4002")]).unwrap(), + }; + + assert_eq!( + settings.litellm_rust_gateway_trusted_proxies.as_deref(), + value + ); + assert!(settings.forwarder().is_ok()); + } + + #[rstest] + #[case::wildcard("*")] + #[case::default_route("0.0.0.0/0")] + fn invalid_trusted_proxy_configuration_fails_forwarder(#[case] value: &str) { + let settings: Settings = parse(&[ + ("LITELLM_PYTHON_UPSTREAM", "http://127.0.0.1:4002"), + ("LITELLM_RUST_GATEWAY_TRUSTED_PROXIES", value), + ]) + .unwrap(); + + assert!(matches!( + settings.forwarder(), + Err(litellm_gateway::Error::TrustedProxies) + )); + } + #[rstest] fn environment_overrides_startup_settings() { let settings: Settings = parse(&[ diff --git a/tests/integration/configuration/test_gateway_supervisor_lifecycle.py b/tests/integration/configuration/test_gateway_supervisor_lifecycle.py new file mode 100644 index 00000000000..9b41c9a83b9 --- /dev/null +++ b/tests/integration/configuration/test_gateway_supervisor_lifecycle.py @@ -0,0 +1,272 @@ +import http.client +import os +import signal +import subprocess +import sys +import time +from pathlib import Path +from typing import Final + +import pytest + +from tests.integration._support.ports import allocate_port + +ROOT: Final = Path(__file__).resolve().parents[3] +SERVER_SOURCE: Final = """\ +import http.server +import signal +import subprocess +import sys +import threading +from pathlib import Path + +role, port, events, mode, child_pid_file = sys.argv[1:] +if role == "python" and mode == "startup-failure": + raise SystemExit(17) + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + if self.path == "/events": + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + self.wfile.write(b"data: connected\\n\\n" if self.path == "/events" else b"ok") + self.wfile.flush() + if self.path == "/events": + threading.Event().wait() + if role == "rust" and mode == "exit": + threading.Timer(0.1, server.shutdown).start() + + def log_message(self, *_args): + pass + +server = http.server.ThreadingHTTPServer(("127.0.0.1", int(port)), Handler) +server.daemon_threads = True +if mode == "orphan": + child = subprocess.Popen( + ( + sys.executable, + "-P", + "-c", + "import signal,time; signal.signal(signal.SIGTERM, signal.SIG_IGN); time.sleep(60)", + ), + start_new_session=True, + ) + Path(child_pid_file).write_text(str(child.pid)) + +def stop(_signum, _frame): + with Path(events).open("a") as output: + output.write(role + "\\n") + if mode not in ("ignore", "orphan"): + threading.Thread(target=server.shutdown).start() + +signal.signal(signal.SIGTERM, stop) +if role == "python" and mode == "exit": + threading.Timer(0.5, server.shutdown).start() +server.serve_forever() +""" +SUPERVISOR_SOURCE: Final = """\ +import os +import subprocess +import sys +from types import MappingProxyType +from gateway.supervisor import SupervisorConfig, supervise + +server, private_port, public_port, events, python_mode, rust_mode, child_pid_file, rust_timeout = sys.argv[1:] +environment = MappingProxyType(dict(os.environ)) +python_command = (sys.executable, server, "python", private_port, events, python_mode, child_pid_file) +rust_command = ( + ("/missing/rust-gateway",) + if rust_mode == "missing" + else (sys.executable, server, "rust", public_port, events, rust_mode, child_pid_file) +) +config = SupervisorConfig( + python_command=python_command, + rust_command=rust_command, + python_environment=environment, + rust_environment=environment, + public_probe_host="127.0.0.1", + public_port=int(public_port), + private_port=int(private_port), + startup_timeout=5, + rust_drain_timeout=int(rust_timeout), + python_grace_timeout=2, +) +failure = supervise(config) +if failure is not None: + print(failure.reason, file=sys.stderr) + sys.exit(1) +""" + + +def _start_supervisor( + tmp_path: Path, + private_port: int, + public_port: int, + events: Path, + python_mode: str = "normal", + rust_mode: str = "normal", + rust_timeout: int = 2, +) -> subprocess.Popen[bytes]: + server: Final = tmp_path / "server.py" + launcher: Final = tmp_path / "supervisor.py" + child_pid_file: Final = tmp_path / "orphan.pid" + server.write_text(SERVER_SOURCE) + launcher.write_text(SUPERVISOR_SOURCE) + return subprocess.Popen( + ( + sys.executable, + "-P", + str(launcher), + str(server), + str(private_port), + str(public_port), + str(events), + python_mode, + rust_mode, + str(child_pid_file), + str(rust_timeout), + ), + cwd=ROOT, + env=os.environ, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + + +def _wait_for_gateway(process: subprocess.Popen[bytes], port: int) -> None: + deadline: Final = time.monotonic() + 10 + while time.monotonic() < deadline: + if process.poll() is not None: + pytest.fail(f"supervisor exited early with status {process.returncode}") + connection: Final = http.client.HTTPConnection("127.0.0.1", port, timeout=0.2) + try: + connection.request("GET", "/health/readiness") + response: Final = connection.getresponse() + response.read() + if response.status == 200: + return + except OSError: + pass + finally: + connection.close() + pytest.fail("supervisor did not start both child listeners") + + +def _terminate_if_running(process: subprocess.Popen[bytes]) -> None: + if process.poll() is not None: + return + os.killpg(process.pid, signal.SIGKILL) # test-quality-ok: Cleanup terminates only this test-owned supervisor group + process.wait(timeout=5) + + +def test_supervisor_drains_rust_before_python(tmp_path: Path) -> None: + private_port: Final = allocate_port() + public_port: Final = allocate_port() + events: Final = tmp_path / "signals" + process: Final = _start_supervisor(tmp_path, private_port, public_port, events) + try: + _wait_for_gateway(process, public_port) + os.kill(process.pid, signal.SIGTERM) # test-quality-ok: The signal is the ordered-shutdown event under test + assert process.wait(timeout=10) == 0 + assert events.read_text().splitlines() == ["rust", "python"] + finally: + _terminate_if_running(process) + + +def test_supervisor_fails_when_python_cannot_start(tmp_path: Path) -> None: + process: Final = _start_supervisor( + tmp_path, + allocate_port(), + allocate_port(), + tmp_path / "signals", + python_mode="startup-failure", + ) + try: + assert process.wait(timeout=10) == 1 + finally: + _terminate_if_running(process) + + +def test_supervisor_cleans_up_python_when_rust_cannot_start(tmp_path: Path) -> None: + events: Final = tmp_path / "signals" + process: Final = _start_supervisor(tmp_path, allocate_port(), allocate_port(), events, rust_mode="missing") + try: + assert process.wait(timeout=10) == 1 + assert events.read_text().splitlines() == ["python"] + finally: + _terminate_if_running(process) + + +@pytest.mark.parametrize("exiting_child", ("python", "rust")) +def test_supervisor_fails_when_either_child_exits(tmp_path: Path, exiting_child: str) -> None: + events: Final = tmp_path / "signals" + process: Final = _start_supervisor( + tmp_path, + allocate_port(), + allocate_port(), + events, + python_mode="exit" if exiting_child == "python" else "normal", + rust_mode="exit" if exiting_child == "rust" else "normal", + ) + try: + assert process.wait(timeout=10) == 1 + finally: + _terminate_if_running(process) + + +def test_supervisor_kills_unresponsive_rust_group_and_reaps_orphan(tmp_path: Path) -> None: + private_port: Final = allocate_port() + public_port: Final = allocate_port() + events: Final = tmp_path / "signals" + child_pid_file: Final = tmp_path / "orphan.pid" + process: Final = _start_supervisor( + tmp_path, + private_port, + public_port, + events, + rust_mode="orphan", + rust_timeout=1, + ) + try: + _wait_for_gateway(process, public_port) + deadline: Final = time.monotonic() + 5 + while not child_pid_file.is_file() and time.monotonic() < deadline: + time.sleep(0.05) + assert child_pid_file.is_file() + child_pid: Final = int(child_pid_file.read_text()) + os.kill(process.pid, signal.SIGTERM) # test-quality-ok: The signal is the orphan-cleanup event under test + assert process.wait(timeout=10) == 0 + assert events.read_text().splitlines() == ["rust", "python"] + assert not Path(f"/proc/{child_pid}").exists() + finally: + _terminate_if_running(process) + + +def test_supervisor_bounds_shutdown_with_an_open_sse_response(tmp_path: Path) -> None: + public_port: Final = allocate_port() + events: Final = tmp_path / "signals" + process: Final = _start_supervisor( + tmp_path, + allocate_port(), + public_port, + events, + rust_mode="ignore", + rust_timeout=1, + ) + connection: Final = http.client.HTTPConnection("127.0.0.1", public_port, timeout=2) + try: + _wait_for_gateway(process, public_port) + connection.request("GET", "/events") + response: Final = connection.getresponse() + assert response.status == 200 + assert response.getheader("Content-Type") == "text/event-stream" + assert response.readline() == b"data: connected\n" + os.kill(process.pid, signal.SIGTERM) # test-quality-ok: The signal starts the bounded SSE drain under test + assert process.wait(timeout=10) == 0 + assert events.read_text().splitlines() == ["rust", "python"] + finally: + connection.close() + _terminate_if_running(process) diff --git a/tests/integration/configuration/test_rust_gateway_trusted_proxy.py b/tests/integration/configuration/test_rust_gateway_trusted_proxy.py new file mode 100644 index 00000000000..83e6726ccb1 --- /dev/null +++ b/tests/integration/configuration/test_rust_gateway_trusted_proxy.py @@ -0,0 +1,183 @@ +import http.client +import os +import signal +import subprocess +import sys +import time +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +import pytest +from pydantic import TypeAdapter + +from tests.integration._support.ports import allocate_port + +ROOT: Final = Path(__file__).resolve().parents[3] +DEFAULT_BINARY: Final = ROOT / "litellm-rust" / "target" / "debug" / "litellm-gateway" +HEADER_OBSERVATION: Final = TypeAdapter(dict[str, str | None]) +APP_SOURCE: Final = """\ +from fastapi import FastAPI, Request + +app = FastAPI() + +@app.get("/health/readiness") +async def readiness(): + return {"status": "healthy"} + +@app.get("/headers") +async def headers(request: Request): + return { + "client": request.client.host if request.client is not None else None, + "host": request.headers.get("host"), + "origin": request.headers.get("origin"), + "xff": request.headers.get("x-forwarded-for"), + "proto": request.headers.get("x-forwarded-proto"), + "scheme": request.url.scheme, + } +""" + + +def _request(address: str, port: int, source: str | None = None) -> dict[str, str | None]: + client: Final = http.client.HTTPConnection( + address, + port, + timeout=1, + source_address=None if source is None else (source, 0), + ) + try: + client.request( + "GET", + "/headers", + headers={ + "Host": "gateway.example", + "Origin": "https://ui.example", + "X-Forwarded-For": "198.51.100.7", + "X-Forwarded-Proto": "https", + "Connection": "close", + }, + ) + response: Final = client.getresponse() + body: Final = response.read() + assert response.status == 200, body + return HEADER_OBSERVATION.validate_json(body) + finally: + client.close() + + +def _wait_until_ready(address: str, port: int, process: subprocess.Popen[bytes]) -> None: + deadline: Final = time.monotonic() + 10 + while time.monotonic() < deadline: + if process.poll() is not None: + _stdout, stderr = process.communicate() + pytest.fail( + f"Rust gateway exited during startup with status {process.returncode}: " + f"{(stderr or b'').decode(errors='replace')}" + ) + connection: Final = http.client.HTTPConnection(address, port, timeout=0.2) + try: + connection.request("GET", "/health/readiness") + response: Final = connection.getresponse() + response.read() + if response.status == 200: + return + except OSError: + pass + finally: + connection.close() + time.sleep(0.05) + pytest.fail("Rust gateway did not forward readiness to the Uvicorn fixture") + + +@pytest.mark.parametrize( + ("trusted_proxies", "expected_client", "expected_proto"), + ( + ("", "127.0.0.2", "http"), + ("127.0.0.2", "198.51.100.7", "https"), + ), +) +def test_rust_gateway_canonicalizes_headers_before_real_uvicorn( + tmp_path: Path, + trusted_proxies: str, + expected_client: str, + expected_proto: str, +) -> None: + binary: Final = Path(os.environ.get("LITELLM_RUST_GATEWAY_BINARY", str(DEFAULT_BINARY))) + if not binary.is_file(): + pytest.skip("build litellm-rust/crates/gateway before running the real Uvicorn trust fixture") + + python_port: Final = allocate_port() + rust_port: Final = allocate_port() + app_path: Final = tmp_path / "fixture.py" + app_path.write_text(APP_SOURCE) + python_environment: Final = { + **os.environ, + "PYTHONPATH": str(tmp_path), + "FORWARDED_ALLOW_IPS": "127.0.0.1", + } + python: Final = subprocess.Popen( + ( + sys.executable, + "-P", + "-m", + "uvicorn", + "fixture:app", + "--host", + "127.0.0.1", + "--port", + str(python_port), + "--proxy-headers", + "--forwarded-allow-ips", + "127.0.0.1", + ), + env=python_environment, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + start_new_session=True, + ) + _wait_until_ready("127.0.0.1", python_port, python) + direct_python: Final = _request("127.0.0.1", python_port, "127.0.0.2") + assert direct_python == { + "client": "127.0.0.2", + "host": "gateway.example", + "origin": "https://ui.example", + "xff": "198.51.100.7", + "proto": "https", + "scheme": "http", + } + rust_environment: Final[Mapping[str, str]] = { + **os.environ, + "HOST": "127.0.0.1", + "PORT": str(rust_port), + "LITELLM_RUST_GATEWAY": "true", + "LITELLM_PYTHON_UPSTREAM": f"http://127.0.0.1:{python_port}", + "LITELLM_RUST_GATEWAY_TRUSTED_PROXIES": trusted_proxies, + } + rust: Final = subprocess.Popen( + (str(binary),), + env=rust_environment, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + start_new_session=True, + ) + try: + _wait_until_ready("127.0.0.1", rust_port, rust) + response: Final = _request("127.0.0.1", rust_port, "127.0.0.2") + + assert response == { + "client": expected_client, + "host": "gateway.example", + "origin": "https://ui.example", + "xff": expected_client, + "proto": expected_proto, + "scheme": expected_proto, + } + finally: + for process in (rust, python): + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) # test-quality-ok: Signal only this fixture's process group + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) # test-quality-ok: Kill only this fixture's process group + process.wait(timeout=5) diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index f76aea030fe..526be6f1c40 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -14,6 +14,12 @@ from uvicorn.importer import import_from_string from uvicorn.main import main as uvicorn_main from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv +from gateway.supervisor import ( + RUST_GATEWAY_BINARY, + ConfigurationFailure, + build_configuration, + gateway_enabled, +) from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings from litellm.proxy.proxy_server import app as proxy_app @@ -158,6 +164,132 @@ class TestPoolDatabaseUrl: assert "MINTED_TOKEN" not in outcome +class TestRustGatewayConfiguration: + @pytest.fixture + def rust_binary(self, tmp_path: Path) -> Path: + binary: Final = tmp_path / "litellm-gateway" + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o700) + return binary + + @pytest.fixture + def clear_uvicorn_environment(self, monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + "UVICORN_HOST", + "UVICORN_PORT", + "UVICORN_UDS", + "UVICORN_FD", + "UVICORN_RELOAD", + "UVICORN_PROXY_HEADERS", + "UVICORN_SSL_KEYFILE", + ): + monkeypatch.delenv(name, raising=False) + + def test_flag_requires_exact_lowercase_values(self) -> None: + assert gateway_enabled({}) is False + assert gateway_enabled({"LITELLM_RUST_GATEWAY": "true"}) is True + assert isinstance(gateway_enabled({"LITELLM_RUST_GATEWAY": "TRUE"}), ConfigurationFailure) + + def test_private_worker_keeps_config_rootpath_keepalive_and_unknown_options( + self, + rust_binary: Path, + clear_uvicorn_environment: None, + ) -> None: + environ: Final = { + RUST_GATEWAY_BINARY: str(rust_binary), + "CONFIG_FILE_PATH": "/app/config/config.yaml", + "KEEPALIVE_TIMEOUT": "41", + } + configuration: Final = build_configuration( + ( + "--workers", + "3", + "--proxy-headers", + "--host", + "0.0.0.0", + "--port", + "4000", + "--root-path", + "/gateway", + "--unknown-option", + "preserved", + "--proxy-headers", + ), + environ, + sys.executable, + ) + + assert not isinstance(configuration, ConfigurationFailure) + assert configuration.python_command == ( + sys.executable, + "-m", + "gateway.launch", + "--host", + "127.0.0.1", + "--port", + "4002", + "--proxy-headers", + "--forwarded-allow-ips", + "127.0.0.1", + "--workers", + "3", + "--root-path", + "/gateway", + "--unknown-option", + "preserved", + "--timeout-keep-alive", + "41", + ) + assert configuration.python_environment["CONFIG_FILE_PATH"] == "/app/config/config.yaml" + assert configuration.python_environment["LITELLM_RUST_GATEWAY"] == "false" + assert configuration.rust_environment["LITELLM_RUST_GATEWAY"] == "true" + assert configuration.rust_environment["LITELLM_PYTHON_UPSTREAM"] == "http://127.0.0.1:4002" + assert configuration.rust_environment["HOST"] == "0.0.0.0" + assert configuration.rust_environment["PORT"] == "4000" + + @pytest.mark.parametrize( + ("arguments", "extra_environment", "reason"), + ( + (("--no-proxy-headers",), {}, "--no-proxy-headers"), + (("--ssl-certfile", "/tmp/cert.pem"), {}, "TLS"), + (("--env-file", "/tmp/uvicorn.env"), {}, "--env-file"), + (("--uds", "/tmp/gateway.sock"), {}, "Unix socket"), + (("--fd", "5"), {}, "file descriptor"), + (("--http", "h2"), {}, "HTTP/1.1"), + (("--host", "0.0.0.0", "--port", "4000"), {"PORT": "4001"}, "PORT conflicts"), + (("--host", "0.0.0.0", "--port", "4000"), {"LITELLM_GATEWAY_METRICS_PORT": "4002"}, "same TCP port"), + ( + ("--host", "0.0.0.0", "--port", "4000"), + {"LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS": "20"}, + "must allow Rust drain", + ), + ( + ("--host", "0.0.0.0", "--port", "4000"), + {"LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS": "0"}, + "must be a positive integer", + ), + ( + ("--host", "0.0.0.0", "--port", "4000"), + {"LITELLM_COLLECTOR_ENABLED": "true", "LITELLM_COLLECTOR_ADDRESS": "tcp://127.0.0.1:4002"}, + "same TCP port", + ), + ), + ) + def test_rejects_unsupported_or_conflicting_configuration( + self, + rust_binary: Path, + clear_uvicorn_environment: None, + arguments: tuple[str, ...], + extra_environment: dict[str, str], + reason: str, + ) -> None: + environ: Final = {RUST_GATEWAY_BINARY: str(rust_binary), **extra_environment} + outcome: Final = build_configuration(arguments, environ, sys.executable) + + assert isinstance(outcome, ConfigurationFailure) + assert reason in outcome.reason + + class TestMain: def test_workers_inherit_the_loopback_url_the_supervisor_installed( self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path From 96a60c253509087476b5bad79fccc221e3184cb5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:29:07 +0000 Subject: [PATCH 2/4] fix(gateway): reap adopted children during supervision --- gateway/RUST_GATEWAY.md | 18 ++++++----- gateway/supervisor.py | 12 +++++-- helm/litellm/tests/rust_gateway_tests.yaml | 13 ++++---- .../test_gateway_supervisor_lifecycle.py | 32 +++++++++++++++++++ 4 files changed, 59 insertions(+), 16 deletions(-) diff --git a/gateway/RUST_GATEWAY.md b/gateway/RUST_GATEWAY.md index 03ec7fce358..f12b6214894 100644 --- a/gateway/RUST_GATEWAY.md +++ b/gateway/RUST_GATEWAY.md @@ -11,13 +11,15 @@ Python port. The private listener must remain on loopback and must not be added Service port. ```sh -docker run --stop-timeout 30 -p 4000:4000 \ +docker build -f gateway/Dockerfile -t litellm-gateway:local . +docker run --stop-timeout 30 --mount type=bind,src="$(pwd)/config.yaml",dst=/app/config/config.yaml,readonly \ + -p 4000:4000 \ -e LITELLM_RUST_GATEWAY=true \ -e LITELLM_RUST_GATEWAY_TRUSTED_PROXIES=10.0.0.0/8 \ -e LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS=30 \ -e LITELLM_GATEWAY_METRICS_PORT=4001 \ -e CONFIG_FILE_PATH=/app/config/config.yaml \ - litellm/litellm:main + litellm-gateway:local ``` The supervisor waits up to 120 seconds for startup, drains Rust for up to 15 seconds, then gives Python 5 seconds @@ -25,21 +27,21 @@ to stop. It also allows a 10-second cleanup margin. `LITELLM_RUST_GATEWAY_STARTU `LITELLM_RUST_GATEWAY_DRAIN_TIMEOUT_SECONDS`, and `LITELLM_RUST_GATEWAY_PYTHON_GRACE_SECONDS` configure those timeouts. `LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS` must be at least the drain timeout plus Python grace plus the cleanup margin. Container orchestrators must grant at least this value before sending SIGKILL. For -Docker, set `--stop-timeout` to the same or a larger number; Docker's default stop timeout is shorter. +Docker, set `--stop-timeout` to the same or a larger number; Docker's default stop timeout is shorter The trusted-proxy setting is an optional comma-separated list of IP addresses or CIDRs. An unset or empty list trusts no external proxy. Wildcard trust is rejected. The Rust gateway canonicalizes forwarded client and scheme headers using the configured trust list. It preserves the original `Host` and `Origin` headers. Configure only -proxy addresses that are controlled by your deployment. +proxy addresses that are controlled by your deployment The Rust gateway does not support TLS termination or Uvicorn TLS settings. Terminate TLS at the load balancer or another trusted edge, then forward HTTP to the gateway. Uvicorn reload, Unix-domain and file-descriptor listeners, -environment-file overrides, and explicit `--no-proxy-headers` are rejected in enabled mode. +environment-file overrides, and explicit `--no-proxy-headers` are rejected in enabled mode The Helm chart keeps the Service on port 4000. Configure `LITELLM_RUST_GATEWAY` through the gateway container's environment or `envFrom`, and set `gateway.terminationGracePeriodSeconds` to match the supervisor grace setting. When the metrics sidecar is enabled, the chart exports its reserved port to the supervisor even when the feature -flag comes from `envFrom`, so port collisions are rejected during enabled startup. +flag comes from `envFrom`, so port collisions are rejected during enabled startup ```yaml gateway: @@ -53,8 +55,8 @@ gateway: The `gateway-runtime` ConfigMap can contain `LITELLM_RUST_GATEWAY: "true"` and `LITELLM_RUST_GATEWAY_TRUSTED_PROXIES: "10.0.0.0/8"`. The chart does not add a private Python port to the pod -ports or Service. +ports or Service To roll back, set `LITELLM_RUST_GATEWAY=false` in the same environment source and redeploy, or roll back the Deployment revision. The feature flag is strict and disabled by default, so removing the opt-in also restores the -existing Python startup path. +existing Python startup path diff --git a/gateway/supervisor.py b/gateway/supervisor.py index a8c8d8d40c9..7b313aebc28 100644 --- a/gateway/supervisor.py +++ b/gateway/supervisor.py @@ -14,8 +14,7 @@ from contextlib import ExitStack from dataclasses import dataclass from pathlib import Path from threading import Event -from types import MappingProxyType -from types import FrameType +from types import FrameType, MappingProxyType from typing import Final, Protocol, cast from urllib.parse import urlsplit @@ -525,12 +524,15 @@ def _wait_for_python( deadline: float, requested: Event, ) -> bool | SupervisorFailure: + protected_children: Final = frozenset((process.pid,)) while not requested.is_set() and time.monotonic() < deadline: + _reap_adopted_children(protected_children) if (status := process.poll()) is not None: return SupervisorFailure(f"Python gateway exited during startup with status {status}") if _tcp_ready("127.0.0.1", port): return True time.sleep(0.05) + _reap_adopted_children(protected_children) return False @@ -542,7 +544,9 @@ def _wait_for_rust( deadline: float, requested: Event, ) -> bool | SupervisorFailure: + protected_children: Final = frozenset((python.pid, rust.pid)) while not requested.is_set() and time.monotonic() < deadline: + _reap_adopted_children(protected_children) if (python_status := python.poll()) is not None: return SupervisorFailure(f"Python gateway exited during startup with status {python_status}") if (rust_status := rust.poll()) is not None: @@ -550,15 +554,19 @@ def _wait_for_rust( if _http_ready(host, port): return True time.sleep(0.05) + _reap_adopted_children(protected_children) return False def _monitor_children(python: ChildProcess, rust: ChildProcess, requested: Event) -> SupervisorFailure | None: + protected_children: Final = frozenset((python.pid, rust.pid)) while not requested.wait(0.1): + _reap_adopted_children(protected_children) if (python_status := python.poll()) is not None: return SupervisorFailure(f"Python gateway exited unexpectedly with status {python_status}") if (rust_status := rust.poll()) is not None: return SupervisorFailure(f"Rust gateway exited unexpectedly with status {rust_status}") + _reap_adopted_children(protected_children) return None diff --git a/helm/litellm/tests/rust_gateway_tests.yaml b/helm/litellm/tests/rust_gateway_tests.yaml index 31c5e936f7d..1b7f34fd572 100644 --- a/helm/litellm/tests/rust_gateway_tests.yaml +++ b/helm/litellm/tests/rust_gateway_tests.yaml @@ -2,6 +2,7 @@ suite: test opt-in Rust gateway deployment templates: - gateway/deployment.yaml - gateway/service.yaml + - gateway/configmap.yaml values: - ./values/required.yaml tests: @@ -114,6 +115,12 @@ tests: path: spec.template.spec.containers[0].envFrom[0].configMapRef.name value: gateway-runtime template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_GATEWAY_METRICS_PORT + value: "4001" + template: gateway/deployment.yaml - it: preserves an explicit zero termination grace for enabled-mode rejection set: @@ -132,9 +139,3 @@ tests: name: LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS value: "0" template: gateway/deployment.yaml - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_GATEWAY_METRICS_PORT - value: "4001" - template: gateway/deployment.yaml diff --git a/tests/integration/configuration/test_gateway_supervisor_lifecycle.py b/tests/integration/configuration/test_gateway_supervisor_lifecycle.py index 9b41c9a83b9..4b2995cb1e9 100644 --- a/tests/integration/configuration/test_gateway_supervisor_lifecycle.py +++ b/tests/integration/configuration/test_gateway_supervisor_lifecycle.py @@ -53,6 +53,9 @@ if mode == "orphan": start_new_session=True, ) Path(child_pid_file).write_text(str(child.pid)) +elif mode == "orphan-exit": + middle_command = "import subprocess,sys,time; child=subprocess.Popen((sys.executable,'-P','-c','import time; time.sleep(0.35)'), start_new_session=True); open(sys.argv[1],'w').write(str(child.pid)); time.sleep(0.2)" + subprocess.Popen((sys.executable, "-P", "-c", middle_command, child_pid_file)) def stop(_signum, _frame): with Path(events).open("a") as output: @@ -245,6 +248,35 @@ def test_supervisor_kills_unresponsive_rust_group_and_reaps_orphan(tmp_path: Pat _terminate_if_running(process) +def test_supervisor_reaps_adopted_child_while_gateway_is_ready(tmp_path: Path) -> None: + public_port: Final = allocate_port() + child_pid_file: Final = tmp_path / "orphan.pid" + process: Final = _start_supervisor( + tmp_path, + allocate_port(), + public_port, + tmp_path / "signals", + rust_mode="orphan-exit", + ) + try: + _wait_for_gateway(process, public_port) + deadline: Final = time.monotonic() + 5 + while not child_pid_file.is_file() and time.monotonic() < deadline: + time.sleep(0.05) + assert child_pid_file.is_file() + child_pid: Final = int(child_pid_file.read_text()) + child_deadline: Final = time.monotonic() + 5 + while Path(f"/proc/{child_pid}").exists() and time.monotonic() < child_deadline: + time.sleep(0.05) + assert not Path(f"/proc/{child_pid}").exists() + assert process.poll() is None + _wait_for_gateway(process, public_port) + os.kill(process.pid, signal.SIGTERM) # test-quality-ok: Signal shutdown after observing the adopted child reap + assert process.wait(timeout=10) == 0 + finally: + _terminate_if_running(process) + + def test_supervisor_bounds_shutdown_with_an_open_sse_response(tmp_path: Path) -> None: public_port: Final = allocate_port() events: Final = tmp_path / "signals" From 5a147fcc277f647e0f18d7763bbbab9db262599c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 23:11:42 +0000 Subject: [PATCH 3/4] feat(gateway): supervise opt-in Rust gateway deployments --- gateway/RUST_GATEWAY.md | 6 + gateway/launch.py | 94 ++++++- gateway/supervisor.py | 248 +++++++++++++----- litellm/proxy/db/pgbouncer.py | 172 ++++++++---- .../test_gateway_supervisor_lifecycle.py | 227 +++++++++++++++- tests/unit/gateway/test_launch.py | 78 +++++- 6 files changed, 692 insertions(+), 133 deletions(-) diff --git a/gateway/RUST_GATEWAY.md b/gateway/RUST_GATEWAY.md index f12b6214894..ab0e212e6cb 100644 --- a/gateway/RUST_GATEWAY.md +++ b/gateway/RUST_GATEWAY.md @@ -10,6 +10,12 @@ The Rust binary defaults to `/usr/local/bin/litellm-gateway`; local deployments Python port. The private listener must remain on loopback and must not be added to a container or Kubernetes Service port. +When the Rust gateway is enabled with a writer `DATABASE_URL`, the supervisor starts one managed PgBouncer for +both gateway children, even when `LITELLM_PGBOUNCER_ENABLED=false`. That flag continues to control optional +pooling in Python-only mode. A database-free Rust gateway stays database-free, and Rust only forwards requests +without initializing a database client. The managed writer pool uses `LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS`; +read-replica connections keep their existing behavior and are outside that writer cap + ```sh docker build -f gateway/Dockerfile -t litellm-gateway:local . docker run --stop-timeout 30 --mount type=bind,src="$(pwd)/config.yaml",dst=/app/config/config.yaml,readonly \ diff --git a/gateway/launch.py b/gateway/launch.py index 6bf5d1285cc..0388e6c9c74 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -14,17 +14,22 @@ Run with: import os import sys +import time from collections.abc import Callable, Mapping, Sequence +from contextlib import ExitStack +from types import MappingProxyType from typing import Final from uvicorn.main import main as uvicorn_main from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import ( + ManagedPgBouncerRuntime, PgBouncerError, PgBouncerSettings, export_pooled_database_url, start_in_container_pgbouncer, + start_managed_pgbouncer, ) GATEWAY_APP: Final = "gateway.main:app" @@ -58,28 +63,95 @@ def pool_database_url( return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth()) +def managed_pool_database_url( + settings: DatabaseURLSettings, + pgbouncer: PgBouncerSettings, + environ: Mapping[str, str], +) -> ManagedPgBouncerRuntime | PgBouncerError | None: + upstream_url: Final = environ.get("DATABASE_URL") + if not upstream_url: + return None + effective_settings: Final = pgbouncer if pgbouncer.enabled else pgbouncer.model_copy(update={"enabled": True}) + return start_managed_pgbouncer(effective_settings, upstream_url, token_auth=settings.token_auth()) + + +def _pooled_child_environments( + python_environment: Mapping[str, str], + rust_environment: Mapping[str, str], + pooler: ManagedPgBouncerRuntime | None, +) -> tuple[Mapping[str, str], Mapping[str, str]]: + if pooler is None: + return ( + MappingProxyType({**python_environment, "LITELLM_PGBOUNCER_ENABLED": "false"}), + rust_environment, + ) + pooled_values: Final = { + "DATABASE_URL": pooler.pooled_url, + "LITELLM_PGBOUNCER_POOLED_DATABASE_URL": "true", + } + return ( + MappingProxyType( + { + **python_environment, + **pooled_values, + "LITELLM_PGBOUNCER_ENABLED": "false", + } + ), + MappingProxyType({**rust_environment, **pooled_values}), + ) + + def _serve(argv: Sequence[str]) -> None: uvicorn_main(tuple(argv), prog_name="uvicorn") +def _stop_starting_pooler(pooler: ManagedPgBouncerRuntime, timeout: int) -> None: + pooler.stop(time.monotonic() + timeout) + + def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: gateway_mode: Final = os.environ.get("LITELLM_RUST_GATEWAY") if gateway_mode not in (None, "true", "false"): sys.exit("LiteLLM gateway: LITELLM_RUST_GATEWAY must be exactly 'true' or 'false'") if gateway_mode == "true": - from gateway.supervisor import ConfigurationFailure, build_configuration, supervise + from dataclasses import replace - configuration: Final = build_configuration(argv, os.environ, sys.executable) - if isinstance(configuration, ConfigurationFailure): - sys.exit(f"LiteLLM gateway supervisor: {configuration.reason}") - failure: Final = supervise(configuration) - if failure is not None: - sys.exit(f"LiteLLM gateway supervisor: {failure.reason}") - return + from gateway.supervisor import ConfigurationFailure, build_configuration, startup_scope, supervise - settings: Final = DatabaseURLSettings.from_env() - settings.apply_to_env() - pooled_url: Final = pool_database_url(settings, PgBouncerSettings(), os.environ) + with ExitStack() as startup_cleanup, startup_scope(): + rust_settings: Final = DatabaseURLSettings.from_env() + rust_settings.apply_to_env() + preflight_configuration: Final = build_configuration(argv, os.environ, sys.executable) + if isinstance(preflight_configuration, ConfigurationFailure): + sys.exit(f"LiteLLM gateway supervisor: {preflight_configuration.reason}") + pooler: Final = managed_pool_database_url(rust_settings, PgBouncerSettings(), os.environ) + if isinstance(pooler, PgBouncerError): + sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooler.reason}") + if pooler is not None: + startup_cleanup.callback( + _stop_starting_pooler, pooler, preflight_configuration.termination_grace_timeout + ) + export_pooled_database_url(pooler.pooled_url) + python_environment, rust_environment = _pooled_child_environments( + preflight_configuration.python_environment, + preflight_configuration.rust_environment, + pooler, + ) + final_configuration: Final = replace( + preflight_configuration, + python_environment=python_environment, + rust_environment=rust_environment, + managed_pooler=pooler, + ) + failure: Final = supervise(final_configuration) + startup_cleanup.pop_all() + if failure is not None: + sys.exit(f"LiteLLM gateway supervisor: {failure.reason}") + return + + python_settings: Final = DatabaseURLSettings.from_env() + python_settings.apply_to_env() + pooled_url: Final = pool_database_url(python_settings, PgBouncerSettings(), os.environ) if isinstance(pooled_url, PgBouncerError): sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}") if pooled_url is not None: diff --git a/gateway/supervisor.py b/gateway/supervisor.py index 7b313aebc28..beca6b0fb58 100644 --- a/gateway/supervisor.py +++ b/gateway/supervisor.py @@ -9,8 +9,8 @@ import socket import subprocess import sys import time -from collections.abc import Callable, Mapping, Sequence -from contextlib import ExitStack +from collections.abc import Callable, Generator, Mapping, Sequence +from contextlib import ExitStack, contextmanager from dataclasses import dataclass from pathlib import Path from threading import Event @@ -23,6 +23,7 @@ from pydantic import TypeAdapter from uvicorn.main import main as uvicorn_main from gateway.launch import uvicorn_argv +from litellm.proxy.db.pgbouncer import ManagedPgBouncerRuntime RUST_GATEWAY_FLAG: Final = "LITELLM_RUST_GATEWAY" RUST_GATEWAY_BINARY: Final = "LITELLM_RUST_GATEWAY_BINARY" @@ -64,6 +65,7 @@ class SupervisorConfig: rust_drain_timeout: int python_grace_timeout: int termination_grace_timeout: int = DEFAULT_TERMINATION_GRACE_SECONDS + managed_pooler: ManagedPgBouncerRuntime | None = None class ChildProcess(Protocol): @@ -261,10 +263,9 @@ def build_configuration( metrics_port: Final = _port_setting(METRICS_PORT_ENV, 0, environ) if METRICS_PORT_ENV in environ else None if isinstance(metrics_port, ConfigurationFailure): return metrics_port - pgbouncer_enabled: Final = _boolean_setting("LITELLM_PGBOUNCER_ENABLED", environ) - if isinstance(pgbouncer_enabled, ConfigurationFailure): - return pgbouncer_enabled - pgbouncer_port: Final = _port_setting("LITELLM_PGBOUNCER_PORT", 6432, environ) if pgbouncer_enabled else None + pgbouncer_port: Final = ( + _port_setting("LITELLM_PGBOUNCER_PORT", 6432, environ) if bool(environ.get("DATABASE_URL")) else None + ) if isinstance(pgbouncer_port, ConfigurationFailure): return pgbouncer_port collector_port: Final = _collector_port(environ) @@ -394,6 +395,25 @@ def _restore_signal_handlers( signal.signal(handled, cast(signal.Handlers | Callable[[int, FrameType | None], object] | None, handler)) +@contextmanager +def startup_scope() -> Generator[None, None, None]: + signals: Final = (signal.SIGTERM, signal.SIGINT) + previous: Final = tuple((handled, signal.getsignal(handled)) for handled in signals) + + def abort_startup(signum: int, _frame: FrameType | None) -> None: + raise SystemExit(128 + signum) + + try: + for handled in signals: + signal.signal(handled, abort_startup) + subreaper_failure: Final = _enable_subreaper() + if subreaper_failure is not None: + raise SystemExit(subreaper_failure.reason) + yield + finally: + _restore_signal_handlers(previous) + + def _tcp_ready(host: str, port: int) -> bool: try: with socket.create_connection((host, port), timeout=0.2): @@ -461,40 +481,69 @@ def _signal_adopted_child(child: int, requested: signal.Signals) -> None: os.killpg(process_group, requested) -def _terminate_adopted_children(excluded: frozenset[int]) -> None: - children: Final = _adopted_children(excluded) - for child in children: +def _reap_supervised_children( + excluded: frozenset[int], + managed_pooler: ManagedPgBouncerRuntime | None, +) -> None: + if managed_pooler is None: + _reap_adopted_children(excluded) + return + with managed_pooler.owned_pid_guard() as pooler_pid: + protected: Final = frozenset((*excluded, *((pooler_pid,) if pooler_pid is not None else ()))) + _reap_adopted_children(protected) + + +def _current_adopted_children( + excluded: frozenset[int], + managed_pooler: ManagedPgBouncerRuntime | None, +) -> tuple[tuple[int, ...], frozenset[int]]: + if managed_pooler is None: + return _adopted_children(excluded), excluded + with managed_pooler.owned_pid_guard() as pooler_pid: + protected: Final = frozenset((*excluded, *((pooler_pid,) if pooler_pid is not None else ()))) + return _adopted_children(protected), protected + + +def _terminate_adopted_children( + excluded: frozenset[int], + deadline: float | None = None, + managed_pooler: ManagedPgBouncerRuntime | None = None, +) -> None: + term_children: Final = _current_adopted_children(excluded, managed_pooler)[0] + for child in term_children: try: _signal_adopted_child(child, signal.SIGTERM) except ProcessLookupError: pass - deadline: Final = time.monotonic() + 1 - while time.monotonic() < deadline and _adopted_children(excluded): - _reap_adopted_children(excluded) + term_deadline: Final = time.monotonic() + 1 if deadline is None else min(deadline, time.monotonic() + 1) + while time.monotonic() < term_deadline and _current_adopted_children(excluded, managed_pooler)[0]: + _reap_supervised_children(excluded, managed_pooler) time.sleep(0.05) - for child in _adopted_children(excluded): + kill_children: Final = _current_adopted_children(excluded, managed_pooler)[0] + for child in kill_children: try: _signal_adopted_child(child, signal.SIGKILL) except ProcessLookupError: pass - final_deadline: Final = time.monotonic() + 1 - while time.monotonic() < final_deadline and _adopted_children(excluded): - _reap_adopted_children(excluded) + final_deadline: Final = time.monotonic() + 1 if deadline is None else min(deadline, time.monotonic() + 1) + while time.monotonic() < final_deadline and _current_adopted_children(excluded, managed_pooler)[0]: + _reap_supervised_children(excluded, managed_pooler) time.sleep(0.05) - _reap_adopted_children(excluded) + _reap_supervised_children(excluded, managed_pooler) def _wait_for_group( process: ChildProcess, deadline: float, protected_children: frozenset[int], + managed_pooler: ManagedPgBouncerRuntime | None, ) -> bool: while time.monotonic() < deadline and _group_exists(process.pid): process.poll() - _reap_adopted_children(protected_children) + _reap_supervised_children(protected_children, managed_pooler) time.sleep(0.05) process.poll() - _reap_adopted_children(protected_children) + _reap_supervised_children(protected_children, managed_pooler) return not _group_exists(process.pid) @@ -502,20 +551,23 @@ def _stop_process_group( process: ChildProcess, timeout: int, protected_children: frozenset[int], + managed_pooler: ManagedPgBouncerRuntime | None, + deadline: float, ) -> None: _signal_group(process.pid, signal.SIGTERM) - if not _wait_for_group(process, time.monotonic() + timeout, protected_children): + stop_deadline: Final = min(deadline, time.monotonic() + timeout) + if not _wait_for_group(process, stop_deadline, protected_children, managed_pooler): _signal_group(process.pid, signal.SIGKILL) - _wait_for_group(process, time.monotonic() + 1, protected_children) + _wait_for_group(process, min(deadline, time.monotonic() + 1), protected_children, managed_pooler) try: - process.wait(timeout=1) + process.wait(timeout=max(0.0, min(1.0, deadline - time.monotonic()))) except subprocess.TimeoutExpired: _signal_group(process.pid, signal.SIGKILL) try: - process.wait(timeout=1) + process.wait(timeout=max(0.0, deadline - time.monotonic())) except subprocess.TimeoutExpired: pass - _reap_adopted_children(protected_children) + _reap_supervised_children(protected_children, managed_pooler) def _wait_for_python( @@ -523,16 +575,17 @@ def _wait_for_python( port: int, deadline: float, requested: Event, + managed_pooler: ManagedPgBouncerRuntime | None, ) -> bool | SupervisorFailure: protected_children: Final = frozenset((process.pid,)) while not requested.is_set() and time.monotonic() < deadline: - _reap_adopted_children(protected_children) + _reap_supervised_children(protected_children, managed_pooler) if (status := process.poll()) is not None: return SupervisorFailure(f"Python gateway exited during startup with status {status}") if _tcp_ready("127.0.0.1", port): return True time.sleep(0.05) - _reap_adopted_children(protected_children) + _reap_supervised_children(protected_children, managed_pooler) return False @@ -543,10 +596,11 @@ def _wait_for_rust( port: int, deadline: float, requested: Event, + managed_pooler: ManagedPgBouncerRuntime | None, ) -> bool | SupervisorFailure: protected_children: Final = frozenset((python.pid, rust.pid)) while not requested.is_set() and time.monotonic() < deadline: - _reap_adopted_children(protected_children) + _reap_supervised_children(protected_children, managed_pooler) if (python_status := python.poll()) is not None: return SupervisorFailure(f"Python gateway exited during startup with status {python_status}") if (rust_status := rust.poll()) is not None: @@ -554,74 +608,126 @@ def _wait_for_rust( if _http_ready(host, port): return True time.sleep(0.05) - _reap_adopted_children(protected_children) + _reap_supervised_children(protected_children, managed_pooler) return False -def _monitor_children(python: ChildProcess, rust: ChildProcess, requested: Event) -> SupervisorFailure | None: +def _monitor_children( + python: ChildProcess, + rust: ChildProcess, + requested: Event, + managed_pooler: ManagedPgBouncerRuntime | None, +) -> SupervisorFailure | None: protected_children: Final = frozenset((python.pid, rust.pid)) while not requested.wait(0.1): - _reap_adopted_children(protected_children) + _reap_supervised_children(protected_children, managed_pooler) if (python_status := python.poll()) is not None: return SupervisorFailure(f"Python gateway exited unexpectedly with status {python_status}") if (rust_status := rust.poll()) is not None: return SupervisorFailure(f"Rust gateway exited unexpectedly with status {rust_status}") - _reap_adopted_children(protected_children) + _reap_supervised_children(protected_children, managed_pooler) return None -def supervise(config: SupervisorConfig, runner: ProcessRunner | None = None) -> SupervisorFailure | None: - subreaper_failure: Final = _enable_subreaper() - if subreaper_failure is not None: - return subreaper_failure - process_runner: Final = runner if runner is not None else SubprocessRunner() - requested: Final = Event() - previous_handlers: Final = _install_signal_handlers(requested) - try: - with ExitStack() as cleanup: - try: - python: Final = process_runner.start(config.python_command, config.python_environment) - except OSError as error: - return SupervisorFailure(f"could not start Python gateway: {error}") - cleanup.callback( - _stop_process_group, - python, - config.python_grace_timeout, - frozenset((python.pid,)), - ) - deadline: Final = time.monotonic() + config.startup_timeout - python_ready: Final = _wait_for_python(python, config.private_port, deadline, requested) - if isinstance(python_ready, SupervisorFailure): - return python_ready - if not python_ready: - return None if requested.is_set() else SupervisorFailure("Python gateway startup timed out") - if requested.is_set(): - return None - try: - rust: Final = process_runner.start(config.rust_command, config.rust_environment) - except OSError as error: - return SupervisorFailure(f"could not start Rust gateway: {error}") - cleanup.callback( - _stop_process_group, - rust, - config.rust_drain_timeout, - frozenset((python.pid, rust.pid)), - ) +def _shutdown( + python: ChildProcess | None, + rust: ChildProcess | None, + config: SupervisorConfig, +) -> None: + deadline: Final = time.monotonic() + config.termination_grace_timeout + protected: Final = frozenset(process.pid for process in (python, rust) if process is not None) + if rust is not None: + _stop_process_group( + rust, + config.rust_drain_timeout, + protected, + config.managed_pooler, + deadline, + ) + if python is not None: + _stop_process_group( + python, + config.python_grace_timeout, + protected, + config.managed_pooler, + deadline, + ) + _terminate_adopted_children(protected, deadline, config.managed_pooler) + if config.managed_pooler is not None: + config.managed_pooler.stop(deadline) + _terminate_adopted_children(frozenset(), deadline) + + +def _supervise_with_python( + config: SupervisorConfig, + process_runner: ProcessRunner, + requested: Event, + python: ChildProcess, +) -> SupervisorFailure | None: + with ExitStack() as python_cleanup: + python_cleanup.callback(_shutdown, python, None, config) + startup_deadline: Final = time.monotonic() + config.startup_timeout + python_ready: Final = _wait_for_python( + python, + config.private_port, + startup_deadline, + requested, + config.managed_pooler, + ) + if isinstance(python_ready, SupervisorFailure): + return python_ready + if not python_ready: + return None if requested.is_set() else SupervisorFailure("Python gateway startup timed out") + if requested.is_set(): + return None + try: + rust: Final = process_runner.start(config.rust_command, config.rust_environment) + except OSError as error: + return SupervisorFailure(f"could not start Rust gateway: {error}") + with ExitStack() as gateway_cleanup: + gateway_cleanup.callback(_shutdown, python, rust, config) + python_cleanup.pop_all() rust_ready: Final = _wait_for_rust( python, rust, config.public_probe_host, config.public_port, - deadline, + startup_deadline, requested, + config.managed_pooler, ) if isinstance(rust_ready, SupervisorFailure): return rust_ready if not rust_ready: return None if requested.is_set() else SupervisorFailure("Rust gateway readiness timed out") - return _monitor_children(python, rust, requested) + return _monitor_children(python, rust, requested, config.managed_pooler) + + +def _supervise( + config: SupervisorConfig, + process_runner: ProcessRunner, + requested: Event, +) -> SupervisorFailure | None: + try: + python: Final = process_runner.start(config.python_command, config.python_environment) + except OSError as error: + _shutdown(None, None, config) + return SupervisorFailure(f"could not start Python gateway: {error}") + return _supervise_with_python(config, process_runner, requested, python) + + +def supervise(config: SupervisorConfig, runner: ProcessRunner | None = None) -> SupervisorFailure | None: + subreaper_failure: Final = _enable_subreaper() + if subreaper_failure is not None: + if config.managed_pooler is not None: + config.managed_pooler.stop(time.monotonic() + config.termination_grace_timeout) + return subreaper_failure + process_runner: Final = runner if runner is not None else SubprocessRunner() + requested: Final = Event() + previous_handlers: Final = _install_signal_handlers(requested) + try: + return _supervise(config, process_runner, requested) finally: - _terminate_adopted_children(frozenset()) _restore_signal_handlers(previous_handlers) diff --git a/litellm/proxy/db/pgbouncer.py b/litellm/proxy/db/pgbouncer.py index c9fbabd3585..781216d3b5c 100644 --- a/litellm/proxy/db/pgbouncer.py +++ b/litellm/proxy/db/pgbouncer.py @@ -46,8 +46,10 @@ import tempfile import threading import time import urllib.parse -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass +from collections.abc import Callable, Generator, Mapping, Sequence +from contextlib import ExitStack +from contextlib import contextmanager +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from types import MappingProxyType @@ -440,10 +442,13 @@ class PgBouncerTokenRefresher: ) return outcome - def stop(self) -> None: + def stop(self, deadline: float | None = None) -> None: self._stopping.set() if self._thread is not None: - self._thread.join() + if deadline is None: + self._thread.join() + else: + self._thread.join(timeout=max(0.0, deadline - time.monotonic())) def _port_open(port: int) -> bool: @@ -487,15 +492,27 @@ def pgbouncer_version(binary: str) -> tuple[int, int] | PgBouncerError: return int(found[1]), int(found[2]) -def _end(process: subprocess.Popen[bytes]) -> None: +def _end(process: subprocess.Popen[bytes], deadline: float | None = None) -> None: if process.poll() is not None: return process.terminate() try: - process.wait(timeout=PGBOUNCER_STOP_GRACE_SECONDS) + remaining: Final = None if deadline is None else max(0.0, deadline - time.monotonic()) + timeout: Final = ( + PGBOUNCER_STOP_GRACE_SECONDS + if remaining is None + else max(0.0, min(PGBOUNCER_STOP_GRACE_SECONDS, remaining - 1.0)) + ) + process.wait(timeout=timeout) except subprocess.TimeoutExpired: process.kill() - process.wait() + if deadline is None: + process.wait() + else: + try: + process.wait(timeout=max(0.0, deadline - time.monotonic())) + except subprocess.TimeoutExpired: + pass class PgBouncerProcess: @@ -537,6 +554,12 @@ class PgBouncerProcess: with self._lock: return None if self._process is None else self._process.pid + @contextmanager + def owned_pid_guard(self) -> Generator[int | None, None, None]: + with self._lock: + process: Final = self._process + yield None if process is None or process.poll() is not None else process.pid + def _spawn(self) -> subprocess.Popen[bytes] | PgBouncerError | None: """Start a child, or None once ``stop`` ran; both take the lock so no child can slip in after a stop. @@ -572,7 +595,7 @@ class PgBouncerProcess: f"within {self.ready_timeout_seconds:.0f}s" ) - def start(self) -> PgBouncerError | None: + def start(self, deadline: float | None = None) -> PgBouncerError | None: """Spawn PgBouncer, wait until it listens on port and unix socket, then supervise it from a daemon thread.""" process: Final = self._spawn() if process is None: @@ -581,7 +604,7 @@ class PgBouncerProcess: return process not_ready: Final = self._wait_ready(process) if not_ready is not None: - self.stop() + self.stop(deadline) return not_ready self._watch(process) return None @@ -631,12 +654,12 @@ class PgBouncerProcess: if self._process is not None: self._process.send_signal(signal.SIGHUP) - def stop(self) -> None: + def stop(self, deadline: float | None = None) -> None: with self._lock: self._stopping.set() process: Final = self._process if process is not None: - _end(process) + _end(process, deadline) def install_pgbouncer_token( @@ -666,7 +689,11 @@ def _install_upstream_password( PgBouncerTokenSource(auth=token_auth, endpoint=parse_iam_endpoint_from_url(upstream_url)), functools.partial(install_pgbouncer_token, plan, runtime_dir, run_as_user, pooler), ) - failed: Final = refresher.start() + try: + failed: Final = refresher.start() + except BaseException: + refresher.stop(time.monotonic() + PGBOUNCER_STOP_GRACE_SECONDS) + raise if failed is not None: return failed return refresher @@ -683,6 +710,85 @@ def _only_in_this_process(action: Callable[[], None]) -> Callable[[], None]: return run +def _stop_managed_pooler(pooler: PgBouncerProcess) -> None: + pooler.stop(time.monotonic() + PGBOUNCER_STOP_GRACE_SECONDS) + + +def _stop_managed_refresher(refresher: PgBouncerTokenRefresher) -> None: + refresher.stop(time.monotonic() + PGBOUNCER_STOP_GRACE_SECONDS) + + +@dataclass(frozen=True, slots=True) +class ManagedPgBouncerRuntime: + pooled_url: str = field(repr=False) + pooler: PgBouncerProcess + refresher: PgBouncerTokenRefresher | None + runtime_dir: Path + + @contextmanager + def owned_pid_guard(self) -> Generator[int | None, None, None]: + with self.pooler.owned_pid_guard() as pid: + yield pid + + def stop(self, deadline: float | None = None) -> None: + try: + self.pooler.stop(deadline) + finally: + if self.refresher is not None: + self.refresher.stop(deadline) + shutil.rmtree(self.runtime_dir, ignore_errors=True) + + +def start_managed_pgbouncer( + settings: PgBouncerSettings, + upstream_url: str, + token_auth: DatabaseTokenAuth | None = None, +) -> ManagedPgBouncerRuntime | PgBouncerError: + version: Final = pgbouncer_version(settings.binary) + if isinstance(version, PgBouncerError): + return version + if version < PGBOUNCER_MIN_VERSION: + return PgBouncerError( + f"PgBouncer {version[0]}.{version[1]} keeps running after failing to bind its TCP port, so the proxy " + f"cannot tell it apart from another listener; {PGBOUNCER_MIN_VERSION[0]}.{PGBOUNCER_MIN_VERSION[1]} " + "or newer is required" + ) + runtime_dir: Final = Path(tempfile.mkdtemp(prefix="litellm-pgbouncer-")) + run_as_user: Final = PGBOUNCER_UNPRIVILEGED_USER if os.geteuid() == 0 else None + with ExitStack() as startup_cleanup: + startup_cleanup.callback(shutil.rmtree, runtime_dir, ignore_errors=True) + plan: Final = plan_pgbouncer(upstream_url, settings, runtime_dir, run_as_user) + if isinstance(plan, PgBouncerError): + return plan + ini_path: Final = write_pgbouncer_ini(plan, runtime_dir, run_as_user) + if isinstance(ini_path, PgBouncerError): + return ini_path + pooler: Final = PgBouncerProcess( + argv=(settings.binary, str(ini_path)), + port=settings.port, + socket_path=unix_socket_path(runtime_dir, settings.port), + ) + startup_cleanup.callback(_stop_managed_pooler, pooler) + refresher_result: Final = _install_upstream_password( + plan, runtime_dir, run_as_user, pooler, token_auth, upstream_url + ) + if isinstance(refresher_result, PgBouncerError): + return refresher_result + if refresher_result is not None: + startup_cleanup.callback(_stop_managed_refresher, refresher_result) + failed: Final = pooler.start(time.monotonic() + PGBOUNCER_STOP_GRACE_SECONDS) + if failed is not None: + return failed + runtime: Final = ManagedPgBouncerRuntime( + pooled_url=plan.pooled_url, + pooler=pooler, + refresher=refresher_result, + runtime_dir=runtime_dir, + ) + startup_cleanup.pop_all() + return runtime + + def start_in_container_pgbouncer( settings: PgBouncerSettings, upstream_url: str, @@ -698,46 +804,16 @@ def start_in_container_pgbouncer( ``nobody``. With ``token_auth`` the password on ``upstream_url`` is ignored: the pooler mints its own tokens and renews them for as long as it runs. """ - version: Final = pgbouncer_version(settings.binary) - if isinstance(version, PgBouncerError): - return version - if version < PGBOUNCER_MIN_VERSION: - return PgBouncerError( - f"PgBouncer {version[0]}.{version[1]} keeps running after failing to bind its TCP port, so the proxy " - f"cannot tell it apart from another listener; {PGBOUNCER_MIN_VERSION[0]}.{PGBOUNCER_MIN_VERSION[1]} " - "or newer is required" - ) - runtime_dir: Final = Path(tempfile.mkdtemp(prefix="litellm-pgbouncer-")) - register_exit_hook(_only_in_this_process(lambda: shutil.rmtree(runtime_dir, ignore_errors=True))) - run_as_user: Final = PGBOUNCER_UNPRIVILEGED_USER if os.geteuid() == 0 else None - plan: Final = plan_pgbouncer(upstream_url, settings, runtime_dir, run_as_user) - if isinstance(plan, PgBouncerError): - return plan - ini_path: Final = write_pgbouncer_ini(plan, runtime_dir, run_as_user) - if isinstance(ini_path, PgBouncerError): - return ini_path - pooler: Final = PgBouncerProcess( - argv=(settings.binary, str(ini_path)), - port=settings.port, - socket_path=unix_socket_path(runtime_dir, settings.port), - ) - refresher: Final = _install_upstream_password(plan, runtime_dir, run_as_user, pooler, token_auth, upstream_url) - if isinstance(refresher, PgBouncerError): - return refresher - failed: Final = pooler.start() - if failed is not None: - if refresher is not None: - refresher.stop() - return failed - register_exit_hook(_only_in_this_process(pooler.stop)) - if refresher is not None: - register_exit_hook(_only_in_this_process(refresher.stop)) + runtime: Final = start_managed_pgbouncer(settings, upstream_url, token_auth) + if isinstance(runtime, PgBouncerError): + return runtime + register_exit_hook(_only_in_this_process(runtime.stop)) verbose_proxy_logger.info( "In-container pgbouncer (pid %s) listening on %s:%s; capping this pod at %s upstream database connections%s.", - pooler.pid, + runtime.pooler.pid, PGBOUNCER_LISTEN_ADDR, settings.port, settings.max_db_connections, "" if token_auth is None else f" and renewing its {token_auth.label} before each one expires", ) - return plan.pooled_url + return runtime.pooled_url diff --git a/tests/integration/configuration/test_gateway_supervisor_lifecycle.py b/tests/integration/configuration/test_gateway_supervisor_lifecycle.py index 4b2995cb1e9..e4596481b0c 100644 --- a/tests/integration/configuration/test_gateway_supervisor_lifecycle.py +++ b/tests/integration/configuration/test_gateway_supervisor_lifecycle.py @@ -3,12 +3,19 @@ import os import signal import subprocess import sys +import textwrap import time +from collections.abc import Iterator from pathlib import Path from typing import Final import pytest +from litellm.proxy.db.pgbouncer import ( + ManagedPgBouncerRuntime, + PgBouncerSettings, + start_managed_pgbouncer, +) from tests.integration._support.ports import allocate_port ROOT: Final = Path(__file__).resolve().parents[3] @@ -72,7 +79,11 @@ SUPERVISOR_SOURCE: Final = """\ import os import subprocess import sys +import threading +import time +from dataclasses import replace from types import MappingProxyType +from pathlib import Path from gateway.supervisor import SupervisorConfig, supervise server, private_port, public_port, events, python_mode, rust_mode, child_pid_file, rust_timeout = sys.argv[1:] @@ -95,6 +106,35 @@ config = SupervisorConfig( rust_drain_timeout=int(rust_timeout), python_grace_timeout=2, ) +if "TEST_MANAGED_PGBOUNCER_BINARY" in os.environ: + from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, start_managed_pgbouncer + + managed_pooler = start_managed_pgbouncer( + PgBouncerSettings( + enabled=True, + port=int(os.environ["TEST_MANAGED_PGBOUNCER_PORT"]), + binary=os.environ["TEST_MANAGED_PGBOUNCER_BINARY"], + ), + "postgresql://litellm:password@127.0.0.1/litellm", + ) + if isinstance(managed_pooler, PgBouncerError): + print(managed_pooler.reason, file=sys.stderr) + sys.exit(1) + pooler_pid_file = Path(os.environ["TEST_MANAGED_PGBOUNCER_PID_FILE"]) + + def write_pooler_pid(): + temporary = pooler_pid_file.with_suffix(".tmp") + temporary.write_text(str(managed_pooler.pooler.pid)) + temporary.replace(pooler_pid_file) + + def record_pooler_pid(): + while True: + write_pooler_pid() + time.sleep(0.05) + + write_pooler_pid() + threading.Thread(target=record_pooler_pid, daemon=True).start() + config = replace(config, managed_pooler=managed_pooler) failure = supervise(config) if failure is not None: print(failure.reason, file=sys.stderr) @@ -110,12 +150,25 @@ def _start_supervisor( python_mode: str = "normal", rust_mode: str = "normal", rust_timeout: int = 2, + managed_pooler: tuple[Path, int, Path] | None = None, ) -> subprocess.Popen[bytes]: server: Final = tmp_path / "server.py" launcher: Final = tmp_path / "supervisor.py" child_pid_file: Final = tmp_path / "orphan.pid" server.write_text(SERVER_SOURCE) launcher.write_text(SUPERVISOR_SOURCE) + environment: Final = { + **os.environ, + **( + {} + if managed_pooler is None + else { + "TEST_MANAGED_PGBOUNCER_BINARY": str(managed_pooler[0]), + "TEST_MANAGED_PGBOUNCER_PORT": str(managed_pooler[1]), + "TEST_MANAGED_PGBOUNCER_PID_FILE": str(managed_pooler[2]), + } + ), + } return subprocess.Popen( ( sys.executable, @@ -131,7 +184,7 @@ def _start_supervisor( str(rust_timeout), ), cwd=ROOT, - env=os.environ, + env=environment, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, @@ -139,6 +192,151 @@ def _start_supervisor( ) +def _fake_pgbouncer(tmp_path: Path) -> Path: + binary: Final = tmp_path / "fake-pgbouncer" + binary.write_text( + textwrap.dedent( + f"""\ + #!{sys.executable} + import configparser, select, socket, sys + if sys.argv[1:] == ["--version"]: + print("PgBouncer 1.25.2") + sys.exit(0) + configuration = configparser.ConfigParser() + configuration.read(sys.argv[1]) + port = configuration.getint("pgbouncer", "listen_port") + tcp = socket.socket() + tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + tcp.bind(("127.0.0.1", port)) + tcp.listen() + unix = socket.socket(socket.AF_UNIX) + unix.bind(configuration.get("pgbouncer", "unix_socket_dir") + f"/.s.PGSQL.{{port}}") + unix.listen() + while True: + for listener in select.select((tcp, unix), [], [])[0]: + listener.accept()[0].close() + """ + ) + ) + binary.chmod(0o700) + return binary + + +def _slow_starting_pgbouncer(tmp_path: Path, pid_file: Path, config_file: Path) -> Path: + binary: Final = tmp_path / "slow-pgbouncer" + binary.write_text( + textwrap.dedent( + f"""\ + #!{sys.executable} + import os, signal, sys, time + from pathlib import Path + if sys.argv[1:] == ["--version"]: + print("PgBouncer 1.26.0") + sys.exit(0) + Path({str(config_file)!r}).write_text(sys.argv[1]) + Path({str(pid_file)!r}).write_text(str(os.getpid())) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + time.sleep(60) + """ + ) + ) + binary.chmod(0o700) + return binary + + +def test_managed_pgbouncer_stop_is_bounded_and_guards_its_owned_pid(tmp_path: Path) -> None: + port: Final = allocate_port() + runtime: Final = start_managed_pgbouncer( + PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pgbouncer(tmp_path))), + "postgresql://litellm:password@127.0.0.1/litellm", + ) + assert isinstance(runtime, ManagedPgBouncerRuntime), runtime + pid: Final = runtime.pooler.pid + deadline: Final = time.monotonic() + 2 + try: + with runtime.owned_pid_guard() as guarded_pid: + assert guarded_pid == pid + finally: + runtime.stop(deadline) + assert pid is not None + assert not Path(f"/proc/{pid}").exists() + assert not runtime.runtime_dir.exists() + + +def test_launch_cleans_up_managed_pooler_when_startup_is_interrupted(tmp_path: Path) -> None: + pid_file: Final = tmp_path / "pooler.pid" + config_file: Final = tmp_path / "pooler-config" + process: Final = subprocess.Popen( + ( + sys.executable, + "-m", + "gateway.launch", + "--host", + "127.0.0.1", + "--port", + str(allocate_port()), + ), + cwd=ROOT, + env={ + **os.environ, + "DATABASE_URL": "postgresql://litellm:password@127.0.0.1/litellm", + "LITELLM_PGBOUNCER_BINARY": str(_slow_starting_pgbouncer(tmp_path, pid_file, config_file)), + "LITELLM_PGBOUNCER_ENABLED": "false", + "LITELLM_PGBOUNCER_PORT": str(allocate_port()), + "LITELLM_RUST_GATEWAY": "true", + "LITELLM_RUST_GATEWAY_BINARY": "/bin/true", + }, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + try: + pooler_pid: Final = _wait_for_pid(pid_file) + os.kill(process.pid, signal.SIGTERM) + assert process.wait(timeout=20) == 143 + assert not Path(f"/proc/{pooler_pid}").exists() + runtime_dir: Final = Path(config_file.read_text()).parent + assert not runtime_dir.exists() + finally: + _terminate_if_running(process) + + +def _wait_for_pid(pid_file: Path) -> int: + deadline: Final = time.monotonic() + 5 + while time.monotonic() < deadline: + try: + pid: Final = int(pid_file.read_text()) + except (OSError, ValueError): + time.sleep(0.05) + continue + if Path(f"/proc/{pid}").exists(): + return pid + time.sleep(0.05) + pytest.fail("managed PgBouncer did not record a live process") + + +def _recorded_pooler_pids(pid_file: Path, deadline: float) -> Iterator[int]: + while time.monotonic() < deadline: + try: + yield int(pid_file.read_text()) + except (OSError, ValueError): + pass + time.sleep(0.05) + + +def _wait_for_replacement_pooler(pid_file: Path, original_pid: int) -> int | None: + deadline: Final = time.monotonic() + 5 + return next( + ( + pid + for pid in _recorded_pooler_pids(pid_file, deadline) + if pid != original_pid and Path(f"/proc/{pid}").exists() + ), + None, + ) + + def _wait_for_gateway(process: subprocess.Popen[bytes], port: int) -> None: deadline: Final = time.monotonic() + 10 while time.monotonic() < deadline: @@ -251,15 +449,18 @@ def test_supervisor_kills_unresponsive_rust_group_and_reaps_orphan(tmp_path: Pat def test_supervisor_reaps_adopted_child_while_gateway_is_ready(tmp_path: Path) -> None: public_port: Final = allocate_port() child_pid_file: Final = tmp_path / "orphan.pid" + pooler_pid_file: Final = tmp_path / "pooler.pid" process: Final = _start_supervisor( tmp_path, allocate_port(), public_port, tmp_path / "signals", rust_mode="orphan-exit", + managed_pooler=(_fake_pgbouncer(tmp_path), allocate_port(), pooler_pid_file), ) try: _wait_for_gateway(process, public_port) + pooler_pid: Final = _wait_for_pid(pooler_pid_file) deadline: Final = time.monotonic() + 5 while not child_pid_file.is_file() and time.monotonic() < deadline: time.sleep(0.05) @@ -270,9 +471,33 @@ def test_supervisor_reaps_adopted_child_while_gateway_is_ready(tmp_path: Path) - time.sleep(0.05) assert not Path(f"/proc/{child_pid}").exists() assert process.poll() is None + assert Path(f"/proc/{pooler_pid}").exists() + os.kill(pooler_pid, signal.SIGKILL) # test-quality-ok: The signal exercises supervised PgBouncer restart + replacement_pid: Final = _wait_for_replacement_pooler(pooler_pid_file, pooler_pid) + assert replacement_pid is not None + assert replacement_pid != pooler_pid _wait_for_gateway(process, public_port) os.kill(process.pid, signal.SIGTERM) # test-quality-ok: Signal shutdown after observing the adopted child reap assert process.wait(timeout=10) == 0 + assert not Path(f"/proc/{replacement_pid}").exists() + finally: + _terminate_if_running(process) + + +def test_supervisor_stops_managed_pooler_when_python_startup_fails(tmp_path: Path) -> None: + pooler_pid_file: Final = tmp_path / "pooler.pid" + process: Final = _start_supervisor( + tmp_path, + allocate_port(), + allocate_port(), + tmp_path / "signals", + python_mode="startup-failure", + managed_pooler=(_fake_pgbouncer(tmp_path), allocate_port(), pooler_pid_file), + ) + try: + assert process.wait(timeout=10) == 1 + pooler_pid: Final = int(pooler_pid_file.read_text()) + assert not Path(f"/proc/{pooler_pid}").exists() finally: _terminate_if_running(process) diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index 526be6f1c40..900b3dbeddd 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -3,6 +3,7 @@ import os import socket import sys import textwrap +import time import urllib.parse from collections.abc import Iterator from pathlib import Path @@ -13,7 +14,14 @@ import pytest from uvicorn.importer import import_from_string from uvicorn.main import main as uvicorn_main -from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv +from gateway.launch import ( + GATEWAY_APP, + _pooled_child_environments, + main, + managed_pool_database_url, + pool_database_url, + uvicorn_argv, +) from gateway.supervisor import ( RUST_GATEWAY_BINARY, ConfigurationFailure, @@ -21,7 +29,12 @@ from gateway.supervisor import ( gateway_enabled, ) from litellm.proxy.db.db_url_settings import DatabaseURLSettings -from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings +from litellm.proxy.db.pgbouncer import ( + PGBOUNCER_POOLED_ENV_VAR, + ManagedPgBouncerRuntime, + PgBouncerError, + PgBouncerSettings, +) from litellm.proxy.proxy_server import app as proxy_app DB_ENV: Final = { @@ -133,6 +146,50 @@ class TestUvicornArgv: class TestPoolDatabaseUrl: + def test_rust_mode_without_a_writer_stays_database_free(self, password_env: dict[str, str]) -> None: + assert ( + managed_pool_database_url( + DatabaseURLSettings.from_env(), + PgBouncerSettings(enabled=True), + {"DATABASE_URL": ""}, + ) + is None + ) + + def test_rust_managed_pooler_is_forced_for_a_writer_when_the_flag_is_off( + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ): + port: Final = _free_port() + binary: Final = _fake_pooler(tmp_path) + monkeypatch.setenv("DATABASE_URL", "postgresql://litellm_pool:p%40ss@db.internal:5432/litellm") + outcome: Final = managed_pool_database_url( + DatabaseURLSettings.from_env(), + PgBouncerSettings(enabled=False, port=port, binary=str(binary)), + os.environ, + ) + assert isinstance(outcome, ManagedPgBouncerRuntime), outcome + try: + assert urllib.parse.urlsplit(outcome.pooled_url).port == port + configuration: Final = build_configuration( + ("--host", "0.0.0.0", "--port", "4000"), + {RUST_GATEWAY_BINARY: "/bin/true"}, + sys.executable, + ) + assert not isinstance(configuration, ConfigurationFailure) + python_environment, rust_environment = _pooled_child_environments( + configuration.python_environment, + configuration.rust_environment, + outcome, + ) + assert python_environment["DATABASE_URL"] == rust_environment["DATABASE_URL"] == outcome.pooled_url + assert python_environment[PGBOUNCER_POOLED_ENV_VAR] == rust_environment[PGBOUNCER_POOLED_ENV_VAR] == "true" + assert python_environment["LITELLM_PGBOUNCER_ENABLED"] == "false" + assert rust_environment.get("LITELLM_PGBOUNCER_ENABLED") == configuration.rust_environment.get( + "LITELLM_PGBOUNCER_ENABLED" + ) + finally: + outcome.stop(time.monotonic() + 2) + def test_a_disabled_pooler_yields_no_url_to_install(self, password_env: dict[str, str]): settings: Final = DatabaseURLSettings.from_env() settings.apply_to_env() @@ -247,6 +304,23 @@ class TestRustGatewayConfiguration: assert configuration.rust_environment["HOST"] == "0.0.0.0" assert configuration.rust_environment["PORT"] == "4000" + def test_database_writer_reserves_pgbouncer_port_when_the_python_only_flag_is_false( + self, rust_binary: Path, clear_uvicorn_environment: None + ) -> None: + configuration: Final = build_configuration( + ("--host", "0.0.0.0", "--port", "4000"), + { + RUST_GATEWAY_BINARY: str(rust_binary), + "DATABASE_URL": "postgresql://app:pw@db/litellm", + "LITELLM_PGBOUNCER_ENABLED": "false", + "LITELLM_PGBOUNCER_PORT": "4000", + }, + sys.executable, + ) + assert isinstance(configuration, ConfigurationFailure) + assert "PgBouncer" in configuration.reason + assert "same TCP port" in configuration.reason + @pytest.mark.parametrize( ("arguments", "extra_environment", "reason"), ( From 4b57d19672aab355c0f07882fc2d2ea38dfb05b9 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:09:44 +0000 Subject: [PATCH 4/4] docs(gateway): consolidate Rust guidance in AGENTS.md --- gateway/AGENTS.md | 17 +++++++++++ gateway/RUST_GATEWAY.md | 68 ----------------------------------------- 2 files changed, 17 insertions(+), 68 deletions(-) create mode 100644 gateway/AGENTS.md delete mode 100644 gateway/RUST_GATEWAY.md diff --git a/gateway/AGENTS.md b/gateway/AGENTS.md new file mode 100644 index 00000000000..951abe49e59 --- /dev/null +++ b/gateway/AGENTS.md @@ -0,0 +1,17 @@ +# Gateway + +## Rust gateway + +- `LITELLM_RUST_GATEWAY=true` enables the public Rust listener. Unset or exact `false` preserves Python-only startup +- Keep `LITELLM_RUST` independent. It controls the in-process Rust bridge +- Rust forwards every request to Python, which owns authentication, routing, policy and provider calls. Do not add native request selection or database initialization here +- `launch.py` owns startup and pooling. `supervisor.py` coordinates container processes, not application requests. A Python or Rust child exit fails the container rather than restarting that child +- Start Python on loopback `127.0.0.1:4002` before Rust on public port 4000. Reserve 4001 for metrics and never expose the private Python port in container or Service ports +- With a writer `DATABASE_URL`, start one managed PgBouncer even when `LITELLM_PGBOUNCER_ENABLED=false`. Both children receive the same pooled URL and marker, and Python must not start another pooler +- Keep database-free mode database-free. Read replicas remain outside the shared writer connection cap +- Drain Rust, stop Python, then stop PgBouncer and reap remaining children within the shutdown deadline. Defaults allow 15 seconds for Rust, 5 for Python and a 10-second cleanup margin +- Docker stop timeout and Helm termination grace must cover `LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS`, which defaults to 30 +- Configure `LITELLM_RUST_GATEWAY_TRUSTED_PROXIES` with explicit ingress IPs or CIDRs. Empty trusts no external proxy, wildcard trust is rejected, and `Host` and `Origin` are preserved +- Terminate TLS at the edge. Enabled mode rejects Uvicorn TLS, reload, Unix-socket and file-descriptor listeners, env-file overrides and disabled proxy headers +- The image packages `/usr/local/bin/litellm-gateway`. Local runs may override `LITELLM_RUST_GATEWAY_BINARY` and the private port through `LITELLM_RUST_GATEWAY_PYTHON_PORT` +- Roll back by removing the Rust gateway opt-in or setting it to `false` and redeploying diff --git a/gateway/RUST_GATEWAY.md b/gateway/RUST_GATEWAY.md deleted file mode 100644 index ab0e212e6cb..00000000000 --- a/gateway/RUST_GATEWAY.md +++ /dev/null @@ -1,68 +0,0 @@ -# Opt-in Rust gateway - -The Rust gateway is an optional public HTTP listener. The existing Python gateway remains the default. Set -`LITELLM_RUST_GATEWAY=true` to start the Rust gateway on the public listener and supervise the existing Python -gateway on loopback port 4002. Set the flag to exactly `false`, or leave it unset, to keep the existing startup -path. No separate enable flag is required for the Python upstream. - -The Rust binary defaults to `/usr/local/bin/litellm-gateway`; local deployments can set -`LITELLM_RUST_GATEWAY_BINARY` to an executable binary. `LITELLM_RUST_GATEWAY_PYTHON_PORT` changes the private -Python port. The private listener must remain on loopback and must not be added to a container or Kubernetes -Service port. - -When the Rust gateway is enabled with a writer `DATABASE_URL`, the supervisor starts one managed PgBouncer for -both gateway children, even when `LITELLM_PGBOUNCER_ENABLED=false`. That flag continues to control optional -pooling in Python-only mode. A database-free Rust gateway stays database-free, and Rust only forwards requests -without initializing a database client. The managed writer pool uses `LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS`; -read-replica connections keep their existing behavior and are outside that writer cap - -```sh -docker build -f gateway/Dockerfile -t litellm-gateway:local . -docker run --stop-timeout 30 --mount type=bind,src="$(pwd)/config.yaml",dst=/app/config/config.yaml,readonly \ - -p 4000:4000 \ - -e LITELLM_RUST_GATEWAY=true \ - -e LITELLM_RUST_GATEWAY_TRUSTED_PROXIES=10.0.0.0/8 \ - -e LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS=30 \ - -e LITELLM_GATEWAY_METRICS_PORT=4001 \ - -e CONFIG_FILE_PATH=/app/config/config.yaml \ - litellm-gateway:local -``` - -The supervisor waits up to 120 seconds for startup, drains Rust for up to 15 seconds, then gives Python 5 seconds -to stop. It also allows a 10-second cleanup margin. `LITELLM_RUST_GATEWAY_STARTUP_TIMEOUT_SECONDS`, -`LITELLM_RUST_GATEWAY_DRAIN_TIMEOUT_SECONDS`, and `LITELLM_RUST_GATEWAY_PYTHON_GRACE_SECONDS` configure those -timeouts. `LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS` must be at least the drain timeout plus Python grace -plus the cleanup margin. Container orchestrators must grant at least this value before sending SIGKILL. For -Docker, set `--stop-timeout` to the same or a larger number; Docker's default stop timeout is shorter - -The trusted-proxy setting is an optional comma-separated list of IP addresses or CIDRs. An unset or empty list -trusts no external proxy. Wildcard trust is rejected. The Rust gateway canonicalizes forwarded client and scheme -headers using the configured trust list. It preserves the original `Host` and `Origin` headers. Configure only -proxy addresses that are controlled by your deployment - -The Rust gateway does not support TLS termination or Uvicorn TLS settings. Terminate TLS at the load balancer or -another trusted edge, then forward HTTP to the gateway. Uvicorn reload, Unix-domain and file-descriptor listeners, -environment-file overrides, and explicit `--no-proxy-headers` are rejected in enabled mode - -The Helm chart keeps the Service on port 4000. Configure `LITELLM_RUST_GATEWAY` through the gateway container's -environment or `envFrom`, and set `gateway.terminationGracePeriodSeconds` to match the supervisor grace setting. -When the metrics sidecar is enabled, the chart exports its reserved port to the supervisor even when the feature -flag comes from `envFrom`, so port collisions are rejected during enabled startup - -```yaml -gateway: - envConfigMaps: - - gateway-runtime - metricsServer: - enabled: true - port: 4001 - terminationGracePeriodSeconds: 30 -``` - -The `gateway-runtime` ConfigMap can contain `LITELLM_RUST_GATEWAY: "true"` and -`LITELLM_RUST_GATEWAY_TRUSTED_PROXIES: "10.0.0.0/8"`. The chart does not add a private Python port to the pod -ports or Service - -To roll back, set `LITELLM_RUST_GATEWAY=false` in the same environment source and redeploy, or roll back the -Deployment revision. The feature flag is strict and disabled by default, so removing the opt-in also restores the -existing Python startup path