diff --git a/gateway/AGENTS.md b/gateway/AGENTS.md new file mode 100644 index 00000000000..951abe49e59 --- /dev/null +++ b/gateway/AGENTS.md @@ -0,0 +1,17 @@ +# Gateway + +## Rust gateway + +- `LITELLM_RUST_GATEWAY=true` enables the public Rust listener. Unset or exact `false` preserves Python-only startup +- Keep `LITELLM_RUST` independent. It controls the in-process Rust bridge +- Rust forwards every request to Python, which owns authentication, routing, policy and provider calls. Do not add native request selection or database initialization here +- `launch.py` owns startup and pooling. `supervisor.py` coordinates container processes, not application requests. A Python or Rust child exit fails the container rather than restarting that child +- Start Python on loopback `127.0.0.1:4002` before Rust on public port 4000. Reserve 4001 for metrics and never expose the private Python port in container or Service ports +- With a writer `DATABASE_URL`, start one managed PgBouncer even when `LITELLM_PGBOUNCER_ENABLED=false`. Both children receive the same pooled URL and marker, and Python must not start another pooler +- Keep database-free mode database-free. Read replicas remain outside the shared writer connection cap +- Drain Rust, stop Python, then stop PgBouncer and reap remaining children within the shutdown deadline. Defaults allow 15 seconds for Rust, 5 for Python and a 10-second cleanup margin +- Docker stop timeout and Helm termination grace must cover `LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS`, which defaults to 30 +- Configure `LITELLM_RUST_GATEWAY_TRUSTED_PROXIES` with explicit ingress IPs or CIDRs. Empty trusts no external proxy, wildcard trust is rejected, and `Host` and `Origin` are preserved +- Terminate TLS at the edge. Enabled mode rejects Uvicorn TLS, reload, Unix-socket and file-descriptor listeners, env-file overrides and disabled proxy headers +- The image packages `/usr/local/bin/litellm-gateway`. Local runs may override `LITELLM_RUST_GATEWAY_BINARY` and the private port through `LITELLM_RUST_GATEWAY_PYTHON_PORT` +- Roll back by removing the Rust gateway opt-in or setting it to `false` and redeploying diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 69068cfc8a5..b39eb05d687 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -47,6 +47,20 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \ python -c "import maturin; maturin.build_wheel('/wheels')" +RUN --mount=type=cache,target=/root/.cargo/registry \ + --mount=type=cache,target=/root/.cargo/git \ + --mount=type=cache,target=/app/litellm-rust/target \ + CARGO_BIN="$(find /root/.cache /root/.cargo /root/.rustup /usr/local \( -type f -o -type l \) -name cargo -print -quit)" && \ + test -x "$CARGO_BIN" || { echo "Rust cargo executable was not found" >&2; exit 1; } && \ + cd litellm-rust && \ + PATH="$(dirname "$CARGO_BIN"):${PATH}" "$CARGO_BIN" build --locked --release -p litellm-gateway --bin litellm-gateway && \ + mkdir -p /rust-gateway && \ + cp target/release/litellm-gateway /rust-gateway/litellm-gateway && \ + readelf -l /rust-gateway/litellm-gateway > /tmp/litellm-gateway-linkage && \ + grep -q "Requesting program interpreter:" /tmp/litellm-gateway-linkage && \ + readelf -d /rust-gateway/litellm-gateway >> /tmp/litellm-gateway-linkage && \ + grep -q "(NEEDED)" /tmp/litellm-gateway-linkage + # ---------- Builder ---------- FROM $LITELLM_BUILD_IMAGE AS builder @@ -148,12 +162,13 @@ COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma COPY --from=builder /opt/prisma /opt/prisma COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer +COPY --from=rust-native /rust-gateway/litellm-gateway /usr/local/bin/litellm-gateway -RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ - find /app/.venv -type d -path "*/tornado/test" -delete && \ +RUN find /app/.venv -type d -path "*/tornado/test" -prune -exec rm -r {} \; && \ chmod -R a+rX /opt/prisma && \ python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \ - python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()" + python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()" && \ + if LITELLM_RUST_GATEWAY=true /usr/local/bin/litellm-gateway 2>&1 | grep -Fq "Error: PythonUpstreamConfig"; then :; else exit 1; fi USER nonroot diff --git a/gateway/launch.py b/gateway/launch.py index 69e2b7c0506..0cc5887385e 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -14,7 +14,10 @@ Run with: import os import sys +import time from collections.abc import Callable, Mapping, Sequence +from contextlib import ExitStack +from types import MappingProxyType from typing import Final from uvicorn.main import main as uvicorn_main @@ -22,11 +25,13 @@ from uvicorn.main import main as uvicorn_main from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import ( PGBOUNCER_MANAGED_SHUTDOWN_SECONDS, + ManagedPgBouncerRuntime, PgBouncerError, PgBouncerSettings, export_pooled_database_url, pgbouncer_startup_signals, start_in_container_pgbouncer, + start_managed_pgbouncer, ) GATEWAY_APP: Final = "gateway.main:app" @@ -65,11 +70,92 @@ def pool_database_url( ) +def managed_pool_database_url( + settings: DatabaseURLSettings, + pgbouncer: PgBouncerSettings, + environ: Mapping[str, str], +) -> ManagedPgBouncerRuntime | PgBouncerError | None: + upstream_url: Final = environ.get("DATABASE_URL") + if not upstream_url: + return None + effective_settings: Final = pgbouncer if pgbouncer.enabled else pgbouncer.model_copy(update={"enabled": True}) + return start_managed_pgbouncer(effective_settings, upstream_url, token_auth=settings.token_auth()) + + +def _pooled_child_environments( + python_environment: Mapping[str, str], + rust_environment: Mapping[str, str], + pooler: ManagedPgBouncerRuntime | None, +) -> tuple[Mapping[str, str], Mapping[str, str]]: + if pooler is None: + return ( + MappingProxyType({**python_environment, "LITELLM_PGBOUNCER_ENABLED": "false"}), + rust_environment, + ) + pooled_values: Final = { + "DATABASE_URL": pooler.pooled_url, + "LITELLM_PGBOUNCER_POOLED_DATABASE_URL": "true", + } + return ( + MappingProxyType( + { + **python_environment, + **pooled_values, + "LITELLM_PGBOUNCER_ENABLED": "false", + } + ), + MappingProxyType({**rust_environment, **pooled_values}), + ) + + def _serve(argv: Sequence[str]) -> None: uvicorn_main(tuple(argv), prog_name="uvicorn") +def _stop_starting_pooler(pooler: ManagedPgBouncerRuntime, timeout: int) -> None: + pooler.stop(time.monotonic() + timeout) + + def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: + gateway_mode: Final = os.environ.get("LITELLM_RUST_GATEWAY") + if gateway_mode not in (None, "true", "false"): + sys.exit("LiteLLM gateway: LITELLM_RUST_GATEWAY must be exactly 'true' or 'false'") + if gateway_mode == "true": + from dataclasses import replace + + from gateway.supervisor import ConfigurationFailure, build_configuration, startup_scope, supervise + + with ExitStack() as startup_cleanup, startup_scope(): + rust_settings: Final = DatabaseURLSettings.from_env() + rust_settings.apply_to_env() + preflight_configuration: Final = build_configuration(argv, os.environ, sys.executable) + if isinstance(preflight_configuration, ConfigurationFailure): + sys.exit(f"LiteLLM gateway supervisor: {preflight_configuration.reason}") + pooler: Final = managed_pool_database_url(rust_settings, PgBouncerSettings(), os.environ) + if isinstance(pooler, PgBouncerError): + sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooler.reason}") + if pooler is not None: + startup_cleanup.callback( + _stop_starting_pooler, pooler, preflight_configuration.termination_grace_timeout + ) + export_pooled_database_url(pooler.pooled_url) + python_environment, rust_environment = _pooled_child_environments( + preflight_configuration.python_environment, + preflight_configuration.rust_environment, + pooler, + ) + final_configuration: Final = replace( + preflight_configuration, + python_environment=python_environment, + rust_environment=rust_environment, + managed_pooler=pooler, + ) + failure: Final = supervise(final_configuration) + startup_cleanup.pop_all() + if failure is not None: + sys.exit(f"LiteLLM gateway supervisor: {failure.reason}") + return + pgbouncer: Final = PgBouncerSettings() with pgbouncer_startup_signals(pgbouncer.enabled): settings: Final = DatabaseURLSettings.from_env() diff --git a/gateway/supervisor.py b/gateway/supervisor.py new file mode 100644 index 00000000000..e88ab702d45 --- /dev/null +++ b/gateway/supervisor.py @@ -0,0 +1,725 @@ +from __future__ import annotations + +import ctypes +import http.client +import ipaddress +import os +import signal +import socket +import subprocess +import sys +import time +from collections.abc import Callable, Generator, Mapping, Sequence +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from pathlib import Path +from threading import Event +from types import FrameType, MappingProxyType +from typing import Final, Protocol, cast +from urllib.parse import urlsplit + +import click +from pydantic import TypeAdapter +from uvicorn.main import main as uvicorn_main + +from gateway.launch import uvicorn_argv +from litellm.proxy.db.pgbouncer import ManagedPgBouncerRuntime, pgbouncer_startup_signals + +RUST_GATEWAY_FLAG: Final = "LITELLM_RUST_GATEWAY" +RUST_GATEWAY_BINARY: Final = "LITELLM_RUST_GATEWAY_BINARY" +PYTHON_PORT_ENV: Final = "LITELLM_RUST_GATEWAY_PYTHON_PORT" +PYTHON_UPSTREAM_ENV: Final = "LITELLM_PYTHON_UPSTREAM" +TRUSTED_PROXIES_ENV: Final = "LITELLM_RUST_GATEWAY_TRUSTED_PROXIES" +METRICS_PORT_ENV: Final = "LITELLM_GATEWAY_METRICS_PORT" +TERMINATION_GRACE_ENV: Final = "LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS" +DEFAULT_RUST_GATEWAY_BINARY: Final = "/usr/local/bin/litellm-gateway" +DEFAULT_PRIVATE_PORT: Final = 4002 +DEFAULT_PYTHON_GRACE_SECONDS: Final = 5 +DEFAULT_RUST_DRAIN_SECONDS: Final = 15 +DEFAULT_STARTUP_TIMEOUT_SECONDS: Final = 120 +DEFAULT_TERMINATION_GRACE_SECONDS: Final = 30 +TERMINATION_CLEANUP_MARGIN_SECONDS: Final = 10 +LIVENESS_PATH: Final = "/health/liveliness" + + +@dataclass(frozen=True, slots=True) +class ConfigurationFailure: + reason: str + + +@dataclass(frozen=True, slots=True) +class SupervisorFailure: + reason: str + + +@dataclass(frozen=True, slots=True) +class SupervisorConfig: + python_command: tuple[str, ...] + rust_command: tuple[str, ...] + python_environment: Mapping[str, str] + rust_environment: Mapping[str, str] + public_probe_host: str + public_port: int + private_port: int + startup_timeout: int + rust_drain_timeout: int + python_grace_timeout: int + termination_grace_timeout: int = DEFAULT_TERMINATION_GRACE_SECONDS + managed_pooler: ManagedPgBouncerRuntime | None = None + + +class ChildProcess(Protocol): + @property + def pid(self) -> int: ... + + def poll(self) -> int | None: ... + + def wait(self, timeout: float | None = None) -> int: ... + + +class ProcessRunner(Protocol): + def start(self, command: tuple[str, ...], environment: Mapping[str, str]) -> ChildProcess: ... + + +class SubprocessRunner: + def start(self, command: tuple[str, ...], environment: Mapping[str, str]) -> ChildProcess: + return subprocess.Popen(command, env=environment, start_new_session=True) + + +def gateway_enabled(environ: Mapping[str, str]) -> bool | ConfigurationFailure: + value: Final = environ.get(RUST_GATEWAY_FLAG) + if value is None or value == "false": + return False + if value == "true": + return True + return ConfigurationFailure(f"{RUST_GATEWAY_FLAG} must be exactly 'true' or 'false'") + + +def _port_setting(name: str, default: int, environ: Mapping[str, str]) -> int | ConfigurationFailure: + value: Final = environ.get(name) + if value is None: + return default + try: + port: Final = int(value) + except ValueError: + return ConfigurationFailure(f"{name} must be an integer port") + if not 1 <= port <= 65535: + return ConfigurationFailure(f"{name} must be between 1 and 65535") + return port + + +def _timeout_setting(name: str, default: int, environ: Mapping[str, str]) -> int | ConfigurationFailure: + value: Final = environ.get(name) + if value is None: + return default + try: + timeout: Final = int(value) + except ValueError: + return ConfigurationFailure(f"{name} must be a positive integer") + if timeout < 1: + return ConfigurationFailure(f"{name} must be a positive integer") + return timeout + + +def _boolean_setting(name: str, environ: Mapping[str, str]) -> bool | ConfigurationFailure: + value: Final = environ.get(name) + if value is None: + return False + normalized: Final = value.casefold() + if normalized in ("true", "1", "yes", "y", "on", "t"): + return True + if normalized in ("false", "0", "no", "n", "off", "f"): + return False + return ConfigurationFailure(f"{name} must be a boolean") + + +def _collector_port(environ: Mapping[str, str]) -> int | ConfigurationFailure | None: + enabled: Final = _boolean_setting("LITELLM_COLLECTOR_ENABLED", environ) + if isinstance(enabled, ConfigurationFailure) or not enabled: + return enabled if isinstance(enabled, ConfigurationFailure) else None + address: Final = environ.get("LITELLM_COLLECTOR_ADDRESS") + if address is None or not address.startswith("tcp://"): + return None + try: + parsed: Final = urlsplit(address) + port: Final = parsed.port + except ValueError: + return ConfigurationFailure("LITELLM_COLLECTOR_ADDRESS must contain a valid TCP port") + if port is None: + return ConfigurationFailure("LITELLM_COLLECTOR_ADDRESS must contain a valid TCP port") + if not 1 <= port <= 65535: + return ConfigurationFailure("LITELLM_COLLECTOR_ADDRESS must contain a valid TCP port") + return port + + +def _trusted_proxies(environ: Mapping[str, str]) -> str | ConfigurationFailure: + value: Final = environ.get(TRUSTED_PROXIES_ENV, "") + if not value: + return value + try: + networks: Final = tuple(ipaddress.ip_network(entry.strip(), strict=False) for entry in value.split(",")) + except ValueError: + return ConfigurationFailure(f"{TRUSTED_PROXIES_ENV} must contain IP addresses or CIDRs") + if any(network.prefixlen == 0 for network in networks): + return ConfigurationFailure(f"{TRUSTED_PROXIES_ENV} must not trust every address") + if any(not entry.strip() for entry in value.split(",")): + return ConfigurationFailure(f"{TRUSTED_PROXIES_ENV} contains an empty entry") + return value + + +def _worker_arguments(arguments: tuple[str, ...], private_port: int) -> tuple[str, ...]: + value_options: Final = frozenset(("--host", "--port", "--forwarded-allow-ips")) + boolean_options: Final = frozenset(("--proxy-headers",)) + consumed_values: Final = frozenset( + index + 1 for index, argument in enumerate(arguments[:-1]) if argument in value_options + ) + without_overrides: Final = tuple( + argument + for index, argument in enumerate(arguments) + if index not in consumed_values + and not any( + argument == option or argument.startswith(f"{option}=") for option in value_options | boolean_options + ) + ) + return ( + "--host", + "127.0.0.1", + "--port", + str(private_port), + "--proxy-headers", + "--forwarded-allow-ips", + "127.0.0.1", + *without_overrides[1:], + ) + + +def _context_value(context: click.Context, name: str) -> object: + parameters: Final = TypeAdapter(dict[str, object]).validate_python(context.params) + value: Final = parameters[name] + return value + + +def _first_port_collision(ports: tuple[tuple[str, int], ...]) -> tuple[str, str] | None: + for index, current in enumerate(ports): + for other in ports[index + 1 :]: + if current[1] == other[1]: + return current[0], other[0] + return None + + +def build_configuration( + argv: Sequence[str], + environ: Mapping[str, str], + python_executable: str, +) -> SupervisorConfig | ConfigurationFailure: + arguments: Final = uvicorn_argv(argv, environ) + try: + context: Final = uvicorn_main.make_context( + "uvicorn", + list(arguments), + allow_extra_args=True, + ignore_unknown_options=True, + ) + except click.ClickException as error: + return ConfigurationFailure(f"invalid Uvicorn options: {error.format_message()}") + + if any(argument == "--no-proxy-headers" for argument in arguments): + return ConfigurationFailure("--no-proxy-headers is unsupported with the Rust gateway") + if any(argument.startswith("--ssl-") for argument in arguments): + return ConfigurationFailure("Uvicorn TLS options are unsupported with the Rust gateway") + if any(name.startswith("UVICORN_SSL_") for name in environ): + return ConfigurationFailure("UVICORN_SSL_* settings are unsupported with the Rust gateway") + if _context_value(context, "env_file") is not None: + return ConfigurationFailure("--env-file is unsupported with the Rust gateway") + if _context_value(context, "uds") is not None or _context_value(context, "fd") is not None: + return ConfigurationFailure("Uvicorn Unix socket and file descriptor listeners are unsupported") + if _context_value(context, "reload") is True: + return ConfigurationFailure("Uvicorn reload is unsupported with the Rust gateway") + if _context_value(context, "proxy_headers") is not True: + return ConfigurationFailure("Uvicorn proxy headers must be enabled with the Rust gateway") + + protocol: Final = _context_value(context, "http") + if not isinstance(protocol, str) or protocol not in ("auto", "h11", "httptools"): + return ConfigurationFailure("the Rust gateway supports only Uvicorn HTTP/1.1 protocols") + + host: Final = _context_value(context, "host") + port: Final = _context_value(context, "port") + if ( + not isinstance(host, str) + or not host + or any(character.isspace() or character in "/[]\r\n" for character in host) + ): + return ConfigurationFailure("Uvicorn must have a non-empty public host") + if not isinstance(port, int) or isinstance(port, bool) or not 1 <= port <= 65535: + return ConfigurationFailure("Uvicorn public port must be between 1 and 65535") + if "HOST" in environ and environ["HOST"] != host: + return ConfigurationFailure("HOST conflicts with the effective Uvicorn host") + if "PORT" in environ and environ["PORT"] != str(port): + return ConfigurationFailure("PORT conflicts with the effective Uvicorn port") + + private_port: Final = _port_setting(PYTHON_PORT_ENV, DEFAULT_PRIVATE_PORT, environ) + if isinstance(private_port, ConfigurationFailure): + return private_port + metrics_port: Final = _port_setting(METRICS_PORT_ENV, 0, environ) if METRICS_PORT_ENV in environ else None + if isinstance(metrics_port, ConfigurationFailure): + return metrics_port + pgbouncer_port: Final = ( + _port_setting("LITELLM_PGBOUNCER_PORT", 6432, environ) if bool(environ.get("DATABASE_URL")) else None + ) + if isinstance(pgbouncer_port, ConfigurationFailure): + return pgbouncer_port + collector_port: Final = _collector_port(environ) + if isinstance(collector_port, ConfigurationFailure): + return collector_port + startup_timeout: Final = _timeout_setting( + "LITELLM_RUST_GATEWAY_STARTUP_TIMEOUT_SECONDS", + DEFAULT_STARTUP_TIMEOUT_SECONDS, + environ, + ) + if isinstance(startup_timeout, ConfigurationFailure): + return startup_timeout + rust_drain_timeout: Final = _timeout_setting( + "LITELLM_RUST_GATEWAY_DRAIN_TIMEOUT_SECONDS", + DEFAULT_RUST_DRAIN_SECONDS, + environ, + ) + if isinstance(rust_drain_timeout, ConfigurationFailure): + return rust_drain_timeout + python_grace_timeout: Final = _timeout_setting( + "LITELLM_RUST_GATEWAY_PYTHON_GRACE_SECONDS", + DEFAULT_PYTHON_GRACE_SECONDS, + environ, + ) + if isinstance(python_grace_timeout, ConfigurationFailure): + return python_grace_timeout + termination_grace_timeout: Final = _timeout_setting( + TERMINATION_GRACE_ENV, + DEFAULT_TERMINATION_GRACE_SECONDS, + environ, + ) + if isinstance(termination_grace_timeout, ConfigurationFailure): + return termination_grace_timeout + if rust_drain_timeout + python_grace_timeout + TERMINATION_CLEANUP_MARGIN_SECONDS > termination_grace_timeout: + return ConfigurationFailure(f"{TERMINATION_GRACE_ENV} must allow Rust drain, Python grace, and cleanup margin") + trusted_proxies: Final = _trusted_proxies(environ) + if isinstance(trusted_proxies, ConfigurationFailure): + return trusted_proxies + + collisions: Final = _first_port_collision( + ( + ("public gateway", port), + ("private Python upstream", private_port), + *(() if metrics_port is None else (("metrics server", metrics_port),)), + *(() if pgbouncer_port is None else (("PgBouncer", pgbouncer_port),)), + *(() if collector_port is None else (("collector", collector_port),)), + ) + ) + if collisions is not None: + return ConfigurationFailure(f"{collisions[0]} and {collisions[1]} use the same TCP port") + + binary: Final = environ.get(RUST_GATEWAY_BINARY, DEFAULT_RUST_GATEWAY_BINARY) + binary_path: Final = Path(binary) + if not binary_path.is_file() or not os.access(binary_path, os.X_OK): + return ConfigurationFailure(f"Rust gateway binary is not executable: {binary}") + + upstream: Final = f"http://127.0.0.1:{private_port}" + configured_upstream: Final = environ.get(PYTHON_UPSTREAM_ENV) + if configured_upstream is not None and configured_upstream != upstream: + return ConfigurationFailure(f"{PYTHON_UPSTREAM_ENV} must equal {upstream}") + + python_environment: Final = MappingProxyType( + { + **environ, + RUST_GATEWAY_FLAG: "false", + } + ) + rust_environment: Final = MappingProxyType( + { + **environ, + RUST_GATEWAY_FLAG: "true", + PYTHON_UPSTREAM_ENV: upstream, + TRUSTED_PROXIES_ENV: trusted_proxies, + "HOST": host, + "PORT": str(port), + } + ) + probe_host: Final = "127.0.0.1" if host in ("0.0.0.0", "*") else "::1" if host == "::" else host + python_command: Final = ( + python_executable, + "-m", + "gateway.launch", + *_worker_arguments(arguments, private_port), + ) + return SupervisorConfig( + python_command=python_command, + rust_command=(binary,), + python_environment=python_environment, + rust_environment=rust_environment, + public_probe_host=probe_host, + public_port=port, + private_port=private_port, + startup_timeout=startup_timeout, + rust_drain_timeout=rust_drain_timeout, + python_grace_timeout=python_grace_timeout, + termination_grace_timeout=termination_grace_timeout, + ) + + +def _enable_subreaper() -> SupervisorFailure | None: + if not sys.platform.startswith("linux"): + return SupervisorFailure("the Rust gateway supervisor requires Linux") + prctl: Final = ctypes.CDLL(None, use_errno=True).prctl + if prctl(36, 1, 0, 0, 0) != 0: + return SupervisorFailure("could not enable Linux child subreaper mode") + return None + + +def _install_signal_handlers( + requested: Event, +) -> tuple[tuple[signal.Signals, object], ...]: + signals: Final = (signal.SIGTERM, signal.SIGINT) + previous: Final = tuple((handled, signal.getsignal(handled)) for handled in signals) + + def request_shutdown(_signum: int, _frame: FrameType | None) -> None: + requested.set() + + for handled in signals: + signal.signal(handled, request_shutdown) + return previous + + +def _restore_signal_handlers( + previous: tuple[tuple[signal.Signals, object], ...], +) -> None: + for handled, handler in previous: + signal.signal(handled, cast(signal.Handlers | Callable[[int, FrameType | None], object] | None, handler)) + + +@contextmanager +def startup_scope() -> Generator[None, None, None]: + with pgbouncer_startup_signals(True): + subreaper_failure: Final = _enable_subreaper() + if subreaper_failure is not None: + raise SystemExit(subreaper_failure.reason) + yield + + +def _tcp_ready(host: str, port: int) -> bool: + try: + with socket.create_connection((host, port), timeout=0.2): + return True + except OSError: + return False + + +def _http_ready(host: str, port: int) -> bool: + connection: Final = http.client.HTTPConnection(host, port, timeout=0.3) + try: + connection.request("GET", LIVENESS_PATH) + response: Final = connection.getresponse() + response.read() + return response.status == 200 + except (http.client.HTTPException, OSError): + return False + finally: + connection.close() + + +def _group_exists(process_group: int) -> bool: + try: + os.killpg(process_group, 0) + return True + except ProcessLookupError: + return False + except PermissionError: + return True + + +def _signal_group(process_group: int, requested: signal.Signals) -> None: + try: + os.killpg(process_group, requested) + except ProcessLookupError: + pass + + +def _reap_adopted_children(excluded: frozenset[int]) -> None: + children_path: Final = Path(f"/proc/self/task/{os.getpid()}/children") + try: + children: Final = tuple(int(pid) for pid in children_path.read_text().split() if int(pid) not in excluded) + except (OSError, ValueError): + return + for child in children: + try: + os.waitpid(child, os.WNOHANG) + except ChildProcessError: + pass + + +def _adopted_children(excluded: frozenset[int]) -> tuple[int, ...]: + children_path: Final = Path(f"/proc/self/task/{os.getpid()}/children") + try: + return tuple(int(pid) for pid in children_path.read_text().split() if int(pid) not in excluded) + except (OSError, ValueError): + return () + + +def _signal_adopted_child(child: int, requested: signal.Signals) -> None: + process_group: Final = os.getpgid(child) + if process_group == os.getpgrp(): + os.kill(child, requested) + else: + os.killpg(process_group, requested) + + +def _reap_supervised_children( + excluded: frozenset[int], + managed_pooler: ManagedPgBouncerRuntime | None, +) -> None: + if managed_pooler is None: + _reap_adopted_children(excluded) + return + with managed_pooler.owned_pid_guard() as pooler_pid: + protected: Final = frozenset((*excluded, *((pooler_pid,) if pooler_pid is not None else ()))) + _reap_adopted_children(protected) + + +def _current_adopted_children( + excluded: frozenset[int], + managed_pooler: ManagedPgBouncerRuntime | None, +) -> tuple[tuple[int, ...], frozenset[int]]: + if managed_pooler is None: + return _adopted_children(excluded), excluded + with managed_pooler.owned_pid_guard() as pooler_pid: + protected: Final = frozenset((*excluded, *((pooler_pid,) if pooler_pid is not None else ()))) + return _adopted_children(protected), protected + + +def _terminate_adopted_children( + excluded: frozenset[int], + deadline: float | None = None, + managed_pooler: ManagedPgBouncerRuntime | None = None, +) -> None: + term_children: Final = _current_adopted_children(excluded, managed_pooler)[0] + for child in term_children: + try: + _signal_adopted_child(child, signal.SIGTERM) + except ProcessLookupError: + pass + term_deadline: Final = time.monotonic() + 1 if deadline is None else min(deadline, time.monotonic() + 1) + while time.monotonic() < term_deadline and _current_adopted_children(excluded, managed_pooler)[0]: + _reap_supervised_children(excluded, managed_pooler) + time.sleep(0.05) + kill_children: Final = _current_adopted_children(excluded, managed_pooler)[0] + for child in kill_children: + try: + _signal_adopted_child(child, signal.SIGKILL) + except ProcessLookupError: + pass + final_deadline: Final = time.monotonic() + 1 if deadline is None else min(deadline, time.monotonic() + 1) + while time.monotonic() < final_deadline and _current_adopted_children(excluded, managed_pooler)[0]: + _reap_supervised_children(excluded, managed_pooler) + time.sleep(0.05) + _reap_supervised_children(excluded, managed_pooler) + + +def _wait_for_group( + process: ChildProcess, + deadline: float, + protected_children: frozenset[int], + managed_pooler: ManagedPgBouncerRuntime | None, +) -> bool: + while time.monotonic() < deadline and _group_exists(process.pid): + process.poll() + _reap_supervised_children(protected_children, managed_pooler) + time.sleep(0.05) + process.poll() + _reap_supervised_children(protected_children, managed_pooler) + return not _group_exists(process.pid) + + +def _stop_process_group( + process: ChildProcess, + timeout: int, + protected_children: frozenset[int], + managed_pooler: ManagedPgBouncerRuntime | None, + deadline: float, +) -> None: + _signal_group(process.pid, signal.SIGTERM) + stop_deadline: Final = min(deadline, time.monotonic() + timeout) + if not _wait_for_group(process, stop_deadline, protected_children, managed_pooler): + _signal_group(process.pid, signal.SIGKILL) + _wait_for_group(process, min(deadline, time.monotonic() + 1), protected_children, managed_pooler) + try: + process.wait(timeout=max(0.0, min(1.0, deadline - time.monotonic()))) + except subprocess.TimeoutExpired: + _signal_group(process.pid, signal.SIGKILL) + try: + process.wait(timeout=max(0.0, deadline - time.monotonic())) + except subprocess.TimeoutExpired: + pass + _reap_supervised_children(protected_children, managed_pooler) + + +def _wait_for_python( + process: ChildProcess, + port: int, + deadline: float, + requested: Event, + managed_pooler: ManagedPgBouncerRuntime | None, +) -> bool | SupervisorFailure: + protected_children: Final = frozenset((process.pid,)) + while not requested.is_set() and time.monotonic() < deadline: + _reap_supervised_children(protected_children, managed_pooler) + if (status := process.poll()) is not None: + return SupervisorFailure(f"Python gateway exited during startup with status {status}") + if _tcp_ready("127.0.0.1", port): + return True + time.sleep(0.05) + _reap_supervised_children(protected_children, managed_pooler) + return False + + +def _wait_for_rust( + python: ChildProcess, + rust: ChildProcess, + host: str, + port: int, + deadline: float, + requested: Event, + managed_pooler: ManagedPgBouncerRuntime | None, +) -> bool | SupervisorFailure: + protected_children: Final = frozenset((python.pid, rust.pid)) + while not requested.is_set() and time.monotonic() < deadline: + _reap_supervised_children(protected_children, managed_pooler) + if (python_status := python.poll()) is not None: + return SupervisorFailure(f"Python gateway exited during startup with status {python_status}") + if (rust_status := rust.poll()) is not None: + return SupervisorFailure(f"Rust gateway exited during startup with status {rust_status}") + if _http_ready(host, port): + return True + time.sleep(0.05) + _reap_supervised_children(protected_children, managed_pooler) + return False + + +def _monitor_children( + python: ChildProcess, + rust: ChildProcess, + requested: Event, + managed_pooler: ManagedPgBouncerRuntime | None, +) -> SupervisorFailure | None: + protected_children: Final = frozenset((python.pid, rust.pid)) + while not requested.wait(0.1): + _reap_supervised_children(protected_children, managed_pooler) + if (python_status := python.poll()) is not None: + return SupervisorFailure(f"Python gateway exited unexpectedly with status {python_status}") + if (rust_status := rust.poll()) is not None: + return SupervisorFailure(f"Rust gateway exited unexpectedly with status {rust_status}") + _reap_supervised_children(protected_children, managed_pooler) + return None + + +def _shutdown( + python: ChildProcess | None, + rust: ChildProcess | None, + config: SupervisorConfig, +) -> None: + deadline: Final = time.monotonic() + config.termination_grace_timeout + protected: Final = frozenset(process.pid for process in (python, rust) if process is not None) + if rust is not None: + _stop_process_group( + rust, + config.rust_drain_timeout, + protected, + config.managed_pooler, + deadline, + ) + if python is not None: + _stop_process_group( + python, + config.python_grace_timeout, + protected, + config.managed_pooler, + deadline, + ) + _terminate_adopted_children(protected, deadline, config.managed_pooler) + if config.managed_pooler is not None: + config.managed_pooler.stop(deadline) + _terminate_adopted_children(frozenset(), deadline) + + +def _supervise_with_python( + config: SupervisorConfig, + process_runner: ProcessRunner, + requested: Event, + python: ChildProcess, +) -> SupervisorFailure | None: + with ExitStack() as python_cleanup: + python_cleanup.callback(_shutdown, python, None, config) + startup_deadline: Final = time.monotonic() + config.startup_timeout + python_ready: Final = _wait_for_python( + python, + config.private_port, + startup_deadline, + requested, + config.managed_pooler, + ) + if isinstance(python_ready, SupervisorFailure): + return python_ready + if not python_ready: + return None if requested.is_set() else SupervisorFailure("Python gateway startup timed out") + if requested.is_set(): + return None + try: + rust: Final = process_runner.start(config.rust_command, config.rust_environment) + except OSError as error: + return SupervisorFailure(f"could not start Rust gateway: {error}") + with ExitStack() as gateway_cleanup: + gateway_cleanup.callback(_shutdown, python, rust, config) + python_cleanup.pop_all() + rust_ready: Final = _wait_for_rust( + python, + rust, + config.public_probe_host, + config.public_port, + startup_deadline, + requested, + config.managed_pooler, + ) + if isinstance(rust_ready, SupervisorFailure): + return rust_ready + if not rust_ready: + return None if requested.is_set() else SupervisorFailure("Rust gateway readiness timed out") + return _monitor_children(python, rust, requested, config.managed_pooler) + + +def _supervise( + config: SupervisorConfig, + process_runner: ProcessRunner, + requested: Event, +) -> SupervisorFailure | None: + try: + python: Final = process_runner.start(config.python_command, config.python_environment) + except OSError as error: + _shutdown(None, None, config) + return SupervisorFailure(f"could not start Python gateway: {error}") + return _supervise_with_python(config, process_runner, requested, python) + + +def supervise(config: SupervisorConfig, runner: ProcessRunner | None = None) -> SupervisorFailure | None: + subreaper_failure: Final = _enable_subreaper() + if subreaper_failure is not None: + if config.managed_pooler is not None: + config.managed_pooler.stop(time.monotonic() + config.termination_grace_timeout) + return subreaper_failure + process_runner: Final = runner if runner is not None else SubprocessRunner() + requested: Final = Event() + previous_handlers: Final = _install_signal_handlers(requested) + try: + return _supervise(config, process_runner, requested) + finally: + _restore_signal_handlers(previous_handlers) + + +if __name__ == "__main__": + sys.exit("run the Rust gateway through python -m gateway.launch") diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 8cb2fbf441d..ecb7e3375dc 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -599,6 +599,14 @@ ImplementationSpecific {{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}} +{{- define "litellm.gateway.rustGatewayEnabled" -}} +{{- range .Values.gateway.extraEnv -}} +{{- if and (eq .name "LITELLM_RUST_GATEWAY") (eq (toString .value) "true") -}} +true +{{- end -}} +{{- end -}} +{{- end -}} + {{/* Directory of the collector's unix socket, shared by the gateway and collector containers through an emptyDir. Empty when the sidecar is off diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index b0e7f0dcebb..0e2ca35bcd5 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -1,4 +1,16 @@ {{- if .Values.gateway.enabled }} +{{- $rustGatewayEnabled := eq (include "litellm.gateway.rustGatewayEnabled" . | trim) "true" }} +{{- $pythonPort := 4002 }} +{{- $configuredGrace := .Values.gateway.terminationGracePeriodSeconds }} +{{- $gracePeriod := 30 }} +{{- if not (or (kindIs "invalid" $configuredGrace) (eq (printf "%v" $configuredGrace) "")) }} +{{- $gracePeriod = int $configuredGrace }} +{{- end }} +{{- range .Values.gateway.extraEnv }} +{{- if and (eq .name "LITELLM_RUST_GATEWAY_PYTHON_PORT") .value }} +{{- $pythonPort = int .value }} +{{- end }} +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: @@ -79,9 +91,16 @@ spec: {{- if eq (int .Values.gateway.metricsServer.port) 4000 }} {{- fail "gateway.metricsServer.port must differ from the gateway port 4000" }} {{- end }} + {{- if and $rustGatewayEnabled (eq (int .Values.gateway.metricsServer.port) $pythonPort) }} + {{- fail "gateway.metricsServer.port must differ from the private Python gateway port" }} + {{- end }} - name: PROMETHEUS_MULTIPROC_DIR value: {{ include "litellm.gateway.prometheusMultiprocDir" . }} + - name: LITELLM_GATEWAY_METRICS_PORT + value: {{ .Values.gateway.metricsServer.port | quote }} {{- end }} + - name: LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS + value: {{ $gracePeriod | quote }} {{- if .Values.gateway.collector.enabled }} {{- include "litellm.gateway.collectorEnv" . | nindent 12 }} {{- end }} @@ -245,8 +264,5 @@ spec: topologySpreadConstraints: {{- toYaml . | nindent 8 }} {{- end }} - {{- $gracePeriod := .Values.gateway.terminationGracePeriodSeconds }} - {{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }} terminationGracePeriodSeconds: {{ $gracePeriod }} - {{- end }} {{- end }} diff --git a/helm/litellm/tests/rust_gateway_tests.yaml b/helm/litellm/tests/rust_gateway_tests.yaml new file mode 100644 index 00000000000..1b7f34fd572 --- /dev/null +++ b/helm/litellm/tests/rust_gateway_tests.yaml @@ -0,0 +1,141 @@ +suite: test opt-in Rust gateway deployment +templates: + - gateway/deployment.yaml + - gateway/service.yaml + - gateway/configmap.yaml +values: + - ./values/required.yaml +tests: + - it: leaves HTTP probes and service ports unchanged when disabled + set: + gateway.metricsServer.enabled: true + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /health/liveliness + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_GATEWAY_METRICS_PORT + value: "4001" + any: true + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].ports[0].containerPort + value: 4000 + template: gateway/deployment.yaml + - equal: + path: spec.ports[0].port + value: 4000 + template: gateway/service.yaml + + - it: passes enabled gateway settings and keeps the private listener off Services + set: + gateway.extraEnv: + - name: LITELLM_RUST_GATEWAY + value: "true" + gateway.envConfigMaps: + - gateway-runtime + gateway.metricsServer.enabled: true + gateway.metricsServer.port: 4001 + gateway.collector.enabled: true + gateway.collector.address: tcp://127.0.0.1:4102 + gateway.terminationGracePeriodSeconds: 30 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_RUST_GATEWAY + value: "true" + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_GATEWAY_METRICS_PORT + value: "4001" + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ADDRESS + value: tcp://127.0.0.1:4102 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].envFrom[0].configMapRef.name + value: gateway-runtime + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /health/liveliness + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /health/readiness + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].ports[0].containerPort + value: 4000 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 30 + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS + value: "30" + template: gateway/deployment.yaml + - equal: + path: spec.ports[0].port + value: 4000 + template: gateway/service.yaml + + - it: rejects a metrics port that collides with the default private Python listener + template: gateway/deployment.yaml + set: + gateway.extraEnv: + - name: LITELLM_RUST_GATEWAY + value: "true" + gateway.metricsServer.enabled: true + gateway.metricsServer.port: 4002 + asserts: + - failedTemplate: + errorPattern: gateway.metricsServer.port must differ from the private Python gateway port + + - it: exports metrics collision metadata when the flag comes from envFrom + set: + gateway.envConfigMaps: + - gateway-runtime + gateway.metricsServer.enabled: true + gateway.metricsServer.port: 4001 + asserts: + - equal: + path: spec.template.spec.containers[0].envFrom[0].configMapRef.name + value: gateway-runtime + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_GATEWAY_METRICS_PORT + value: "4001" + template: gateway/deployment.yaml + + - it: preserves an explicit zero termination grace for enabled-mode rejection + set: + gateway.extraEnv: + - name: LITELLM_RUST_GATEWAY + value: "true" + gateway.terminationGracePeriodSeconds: 0 + asserts: + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 0 + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS + value: "0" + template: gateway/deployment.yaml diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index d3ed40048c7..2a3fd01f73a 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3770,6 +3770,7 @@ dependencies = [ "http-body-util", "hyper 1.10.1", "hyper-util", + "ipnet", "litellm-auth-types", "litellm-config", "litellm-core-utils", diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index c87824c459a..e93b23d901f 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -12,6 +12,7 @@ envy = "0.4.2" http-body-util = "0.1" hyper = { version = "1", features = ["http1"] } hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] } +ipnet = "2.12" litellm-core-utils.workspace = true litellm-inference.workspace = true litellm-gateway-inference.workspace = true diff --git a/litellm-rust/crates/gateway/src/dispatch/forward.rs b/litellm-rust/crates/gateway/src/dispatch/forward.rs index b31587c96f8..d631014b42e 100644 --- a/litellm-rust/crates/gateway/src/dispatch/forward.rs +++ b/litellm-rust/crates/gateway/src/dispatch/forward.rs @@ -53,6 +53,7 @@ pub struct InvalidUpstream { pub struct Forward { client: Arc>, authority: Authority, + trusted_proxies: Option, } impl Forward { @@ -72,9 +73,15 @@ impl Forward { Ok(Self { client: Arc::new(Client::builder(TokioExecutor::new()).build(HttpConnector::new())), authority, + trusted_proxies: None, }) } + pub fn with_trusted_proxies(mut self, proxies: &str) -> Result { + self.trusted_proxies = Some(crate::proxy_headers::TrustedProxies::parse(proxies)?); + Ok(self) + } + pub(super) async fn send(&self, mut request: Request, decline: Decline) -> Response { let websocket = if request.method() == Method::GET && request.version() == Version::HTTP_11 @@ -126,8 +133,14 @@ impl Forward { if websocket.is_some() { websocket_headers(&mut parts.headers); } - if let Some(ConnectInfo(client)) = parts.extensions.get::>() { - append_forwarded_for(&mut parts.headers, *client); + let peer = parts + .extensions + .get::>() + .map(|info| info.0); + if let Some(policy) = &self.trusted_proxies { + policy.apply(&mut parts.headers, peer); + } else if let Some(client) = peer { + append_forwarded_for(&mut parts.headers, client); } parts .headers diff --git a/litellm-rust/crates/gateway/src/error.rs b/litellm-rust/crates/gateway/src/error.rs index 8149be4a157..25b68843d90 100644 --- a/litellm-rust/crates/gateway/src/error.rs +++ b/litellm-rust/crates/gateway/src/error.rs @@ -1,5 +1,9 @@ #[derive(Debug, thiserror::Error)] pub enum Error { + #[error( + "LITELLM_RUST_GATEWAY_TRUSTED_PROXIES must contain comma-separated IP addresses or CIDRs, not wildcard trust" + )] + TrustedProxies, #[error( "LITELLM_RUST_GATEWAY=true requires LITELLM_PYTHON_UPSTREAM to be a valid http:// origin" )] diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 3435fcbf62e..6e2b43454e9 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -1,6 +1,7 @@ mod auth; mod dispatch; mod error; +mod proxy_headers; mod secrets; mod upgrades; pub use dispatch::{Fallback, Forward, InvalidUpstream}; diff --git a/litellm-rust/crates/gateway/src/proxy_headers.rs b/litellm-rust/crates/gateway/src/proxy_headers.rs new file mode 100644 index 00000000000..4337da91fd3 --- /dev/null +++ b/litellm-rust/crates/gateway/src/proxy_headers.rs @@ -0,0 +1,172 @@ +use std::net::{IpAddr, SocketAddr}; + +use axum::http::{HeaderMap, HeaderValue}; +use ipnet::IpNet; + +#[derive(Clone)] +pub(crate) struct TrustedProxies(Vec); + +impl TrustedProxies { + pub(crate) fn parse(value: &str) -> Result { + if value.trim().is_empty() { + return Ok(Self(Vec::new())); + } + value + .split(',') + .map(|value| { + let value = value.trim(); + let network = value + .parse::() + .or_else(|_| value.parse::().map(IpNet::from)) + .map_err(|_| crate::Error::TrustedProxies)?; + if network.prefix_len() == 0 { + return Err(crate::Error::TrustedProxies); + } + Ok(network) + }) + .collect::, _>>() + .map(Self) + } + + fn contains(&self, address: IpAddr) -> bool { + self.0.iter().any(|network| network.contains(&address)) + } + + pub(crate) fn apply(&self, headers: &mut HeaderMap, peer: Option) { + let trusted = peer.is_some_and(|peer| self.contains(peer.ip())); + let chain = trusted + .then(|| { + headers + .get_all("x-forwarded-for") + .iter() + .map(|value| value.to_str().ok()) + .collect::>>() + .and_then(|values| { + values + .iter() + .flat_map(|value| value.split(',')) + .map(|value| value.trim().parse::().ok()) + .collect::>>() + }) + }) + .flatten(); + let client = chain + .as_ref() + .and_then(|chain| { + chain + .iter() + .rev() + .find(|address| !self.contains(**address)) + .or_else(|| chain.first()) + .copied() + }) + .or_else(|| peer.map(|peer| peer.ip())); + let scheme = if trusted && headers.get_all("x-forwarded-proto").iter().count() == 1 { + match headers + .get("x-forwarded-proto") + .and_then(|value| value.to_str().ok()) + { + Some("https") => "https", + _ => "http", + } + } else { + "http" + }; + for name in [ + "forwarded", + "x-forwarded-for", + "x-forwarded-proto", + "x-forwarded-host", + "x-forwarded-port", + "x-forwarded-prefix", + "x-real-ip", + ] { + headers.remove(name); + } + if let Some(client) = client + && let Ok(value) = HeaderValue::from_str(&client.to_string()) + { + headers.insert("x-forwarded-for", value); + } + headers.insert("x-forwarded-proto", HeaderValue::from_static(scheme)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[rstest] + #[case::direct("", "127.0.0.2", "198.51.100.7", "https", "127.0.0.2", "http")] + #[case::trusted( + "127.0.0.2", + "127.0.0.2", + "198.51.100.7", + "https", + "198.51.100.7", + "https" + )] + #[case::chain( + "10.0.0.0/8", + "10.0.0.1", + "192.0.2.1, 198.51.100.7, 10.0.0.2", + "https", + "198.51.100.7", + "https" + )] + #[case::invalid_chain( + "10.0.0.0/8", + "10.0.0.1", + "198.51.100.7, unknown", + "https", + "10.0.0.1", + "https" + )] + #[case::invalid_scheme( + "127.0.0.2", + "127.0.0.2", + "198.51.100.7", + "https,http", + "198.51.100.7", + "http" + )] + #[case::ipv6("fd00::/8", "fd00::1", "2001:db8::7", "https", "2001:db8::7", "https")] + fn normalizes_proxy_identity( + #[case] trusted: &str, + #[case] peer: &str, + #[case] chain: &str, + #[case] proto: &str, + #[case] client: &str, + #[case] scheme: &str, + ) { + let mut headers = HeaderMap::new(); + headers.insert("host", HeaderValue::from_static("gateway.example")); + headers.insert("origin", HeaderValue::from_static("https://app.example")); + headers.insert("x-forwarded-for", HeaderValue::from_str(chain).unwrap()); + headers.insert("x-forwarded-proto", HeaderValue::from_str(proto).unwrap()); + headers.insert("forwarded", HeaderValue::from_static("proto=https")); + TrustedProxies::parse(trusted).unwrap().apply( + &mut headers, + Some(SocketAddr::new(peer.parse().unwrap(), 1234)), + ); + assert_eq!(headers["x-forwarded-for"], client); + assert_eq!(headers["x-forwarded-proto"], scheme); + assert_eq!(headers["host"], "gateway.example"); + assert_eq!(headers["origin"], "https://app.example"); + assert!(!headers.contains_key("forwarded")); + } + + #[rstest] + #[case("*")] + #[case("0.0.0.0/0")] + #[case("::/0")] + #[case("127.0.0.1,")] + #[case("proxy.example")] + fn invalid_policy_fails_closed(#[case] value: &str) { + assert!(matches!( + TrustedProxies::parse(value), + Err(crate::Error::TrustedProxies) + )); + } +} diff --git a/litellm-rust/crates/gateway/src/settings.rs b/litellm-rust/crates/gateway/src/settings.rs index f549f18131c..e2ff9f92133 100644 --- a/litellm-rust/crates/gateway/src/settings.rs +++ b/litellm-rust/crates/gateway/src/settings.rs @@ -12,6 +12,7 @@ pub(super) struct Settings { pub litellm_config: String, pub litellm_python_upstream: Option, pub litellm_rust_gateway: bool, + pub litellm_rust_gateway_trusted_proxies: Option, rust_log: Option, } @@ -23,6 +24,7 @@ impl Default for Settings { litellm_config: "config.yaml".into(), litellm_python_upstream: None, litellm_rust_gateway: false, + litellm_rust_gateway_trusted_proxies: None, rust_log: None, } } @@ -51,7 +53,11 @@ impl Settings { { return Err(invalid()); } - litellm_gateway::Forward::new(upstream).map_err(|_| invalid()) + let forward = litellm_gateway::Forward::new(upstream).map_err(|_| invalid())?; + match self.litellm_rust_gateway_trusted_proxies.as_deref() { + Some(proxies) => forward.with_trusted_proxies(proxies), + None => Ok(forward), + } } pub fn log_level(&self) -> Level { @@ -192,6 +198,43 @@ mod tests { ); } + #[rstest] + #[case::absent(None)] + #[case::empty(Some(""))] + #[case::explicit_trust(Some("127.0.0.2"))] + fn trusted_proxy_configuration_keeps_absent_and_empty_distinct(#[case] value: Option<&str>) { + let settings: Settings = match value { + Some(trusted) => parse(&[ + ("LITELLM_PYTHON_UPSTREAM", "http://127.0.0.1:4002"), + ("LITELLM_RUST_GATEWAY_TRUSTED_PROXIES", trusted), + ]) + .unwrap(), + None => parse(&[("LITELLM_PYTHON_UPSTREAM", "http://127.0.0.1:4002")]).unwrap(), + }; + + assert_eq!( + settings.litellm_rust_gateway_trusted_proxies.as_deref(), + value + ); + assert!(settings.forwarder().is_ok()); + } + + #[rstest] + #[case::wildcard("*")] + #[case::default_route("0.0.0.0/0")] + fn invalid_trusted_proxy_configuration_fails_forwarder(#[case] value: &str) { + let settings: Settings = parse(&[ + ("LITELLM_PYTHON_UPSTREAM", "http://127.0.0.1:4002"), + ("LITELLM_RUST_GATEWAY_TRUSTED_PROXIES", value), + ]) + .unwrap(); + + assert!(matches!( + settings.forwarder(), + Err(litellm_gateway::Error::TrustedProxies) + )); + } + #[rstest] fn environment_overrides_startup_settings() { let settings: Settings = parse(&[ diff --git a/tests/integration/configuration/test_gateway_supervisor_lifecycle.py b/tests/integration/configuration/test_gateway_supervisor_lifecycle.py new file mode 100644 index 00000000000..ea738a76db0 --- /dev/null +++ b/tests/integration/configuration/test_gateway_supervisor_lifecycle.py @@ -0,0 +1,647 @@ +import http.client +import os +import signal +import subprocess +import sys +import textwrap +import time +from collections.abc import Iterator, Mapping +from contextlib import ExitStack +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import pytest + +from gateway.launch import ( + _pooled_child_environments, # pyright: ignore[reportPrivateUsage] # exercise the child env policy +) +from gateway.supervisor import ChildProcess, SupervisorConfig, SupervisorFailure, supervise +from litellm.proxy.db.pgbouncer import ( + ManagedPgBouncerRuntime, + PgBouncerSettings, + start_managed_pgbouncer, +) +from tests.integration._support.ports import allocate_port + +ROOT: Final = Path(__file__).resolve().parents[3] +SERVER_SOURCE: Final = """\ +import http.server +import signal +import subprocess +import sys +import threading +from pathlib import Path + +role, port, events, mode, child_pid_file = sys.argv[1:] +if role == "python" and mode == "startup-failure": + raise SystemExit(17) + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + if self.path == "/events": + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + self.wfile.write(b"data: connected\\n\\n" if self.path == "/events" else b"ok") + self.wfile.flush() + if self.path == "/events": + threading.Event().wait() + if role == "rust" and mode == "exit": + threading.Timer(0.1, server.shutdown).start() + + def log_message(self, *_args): + pass + +server = http.server.ThreadingHTTPServer(("127.0.0.1", int(port)), Handler) +server.daemon_threads = True +if mode == "orphan": + child = subprocess.Popen( + ( + sys.executable, + "-P", + "-c", + "import signal,time; signal.signal(signal.SIGTERM, signal.SIG_IGN); time.sleep(60)", + ), + start_new_session=True, + ) + Path(child_pid_file).write_text(str(child.pid)) +elif mode == "orphan-exit": + middle_command = "import subprocess,sys,time; child=subprocess.Popen((sys.executable,'-P','-c','import time; time.sleep(0.35)'), start_new_session=True); open(sys.argv[1],'w').write(str(child.pid)); time.sleep(0.2)" + subprocess.Popen((sys.executable, "-P", "-c", middle_command, child_pid_file)) + +def stop(_signum, _frame): + with Path(events).open("a") as output: + output.write(role + "\\n") + if mode not in ("ignore", "orphan"): + threading.Thread(target=server.shutdown).start() + +signal.signal(signal.SIGTERM, stop) +if role == "python" and mode == "exit": + threading.Timer(0.5, server.shutdown).start() +server.serve_forever() +""" +LAUNCH_MAIN_SOURCE: Final = """\ +import os +from pathlib import Path +from gateway.launch import main + +def serve(_argv): + Path(os.environ["TEST_SERVE_MARKER"]).write_text( + os.environ["LITELLM_PGBOUNCER_ENABLED"] + "|" + os.environ.get("DATABASE_URL", "") + ) + +main((), serve) +""" +SUPERVISOR_SOURCE: Final = """\ +import os +import subprocess +import sys +import threading +import time +from dataclasses import replace +from types import MappingProxyType +from pathlib import Path +from gateway.supervisor import SupervisorConfig, supervise + +server, private_port, public_port, events, python_mode, rust_mode, child_pid_file, rust_timeout = sys.argv[1:] +environment = MappingProxyType(dict(os.environ)) +python_command = (sys.executable, server, "python", private_port, events, python_mode, child_pid_file) +rust_command = ( + ("/missing/rust-gateway",) + if rust_mode == "missing" + else (sys.executable, server, "rust", public_port, events, rust_mode, child_pid_file) +) +config = SupervisorConfig( + python_command=python_command, + rust_command=rust_command, + python_environment=environment, + rust_environment=environment, + public_probe_host="127.0.0.1", + public_port=int(public_port), + private_port=int(private_port), + startup_timeout=5, + rust_drain_timeout=int(rust_timeout), + python_grace_timeout=2, +) +if "TEST_MANAGED_PGBOUNCER_BINARY" in os.environ: + from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, start_managed_pgbouncer + + managed_pooler = start_managed_pgbouncer( + PgBouncerSettings( + enabled=True, + port=int(os.environ["TEST_MANAGED_PGBOUNCER_PORT"]), + binary=os.environ["TEST_MANAGED_PGBOUNCER_BINARY"], + ), + "postgresql://litellm:password@127.0.0.1/litellm", + ) + if isinstance(managed_pooler, PgBouncerError): + print(managed_pooler.reason, file=sys.stderr) + sys.exit(1) + pooler_pid_file = Path(os.environ["TEST_MANAGED_PGBOUNCER_PID_FILE"]) + + def write_pooler_pid(): + temporary = pooler_pid_file.with_suffix(".tmp") + temporary.write_text(str(managed_pooler.pooler.pid)) + temporary.replace(pooler_pid_file) + + def record_pooler_pid(): + while True: + write_pooler_pid() + time.sleep(0.05) + + write_pooler_pid() + threading.Thread(target=record_pooler_pid, daemon=True).start() + config = replace(config, managed_pooler=managed_pooler) +failure = supervise(config) +if failure is not None: + print(failure.reason, file=sys.stderr) + sys.exit(1) +""" + + +@dataclass(frozen=True, slots=True) +class _FailAfterPythonStartRunner: + python_command: tuple[str, ...] + python_pid_file: Path + cleanup: ExitStack + + def start(self, command: tuple[str, ...], environment: Mapping[str, str]) -> ChildProcess: + if command != self.python_command: + raise OSError("injected Rust gateway start failure") + process: Final = subprocess.Popen( + command, + env=environment, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + self.cleanup.callback(_terminate_if_running, process) + self.python_pid_file.write_text(str(process.pid)) + return process + + +def _start_supervisor( + tmp_path: Path, + private_port: int, + public_port: int, + events: Path, + python_mode: str = "normal", + rust_mode: str = "normal", + rust_timeout: int = 2, + managed_pooler: tuple[Path, int, Path] | None = None, +) -> subprocess.Popen[bytes]: + server: Final = tmp_path / "server.py" + launcher: Final = tmp_path / "supervisor.py" + child_pid_file: Final = tmp_path / "orphan.pid" + server.write_text(SERVER_SOURCE) + launcher.write_text(SUPERVISOR_SOURCE) + environment: Final = { + **os.environ, + **( + {} + if managed_pooler is None + else { + "TEST_MANAGED_PGBOUNCER_BINARY": str(managed_pooler[0]), + "TEST_MANAGED_PGBOUNCER_PORT": str(managed_pooler[1]), + "TEST_MANAGED_PGBOUNCER_PID_FILE": str(managed_pooler[2]), + } + ), + } + return subprocess.Popen( + ( + sys.executable, + "-P", + str(launcher), + str(server), + str(private_port), + str(public_port), + str(events), + python_mode, + rust_mode, + str(child_pid_file), + str(rust_timeout), + ), + cwd=ROOT, + env=environment, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + + +def _fake_pgbouncer(tmp_path: Path) -> Path: + binary: Final = tmp_path / "fake-pgbouncer" + binary.write_text( + textwrap.dedent( + f"""\ + #!{sys.executable} + import configparser, select, socket, sys + if sys.argv[1:] == ["--version"]: + print("PgBouncer 1.25.2") + sys.exit(0) + configuration = configparser.ConfigParser() + configuration.read(sys.argv[1]) + port = configuration.getint("pgbouncer", "listen_port") + tcp = socket.socket() + tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + tcp.bind(("127.0.0.1", port)) + tcp.listen() + unix = socket.socket(socket.AF_UNIX) + unix.bind(configuration.get("pgbouncer", "unix_socket_dir") + f"/.s.PGSQL.{{port}}") + unix.listen() + while True: + for listener in select.select((tcp, unix), [], [])[0]: + listener.accept()[0].close() + """ + ) + ) + binary.chmod(0o700) + return binary + + +def _slow_starting_pgbouncer(tmp_path: Path, pid_file: Path, config_file: Path) -> Path: + binary: Final = tmp_path / "slow-pgbouncer" + binary.write_text( + textwrap.dedent( + f"""\ + #!{sys.executable} + import os, signal, sys, time + from pathlib import Path + if sys.argv[1:] == ["--version"]: + print("PgBouncer 1.26.0") + sys.exit(0) + Path({str(config_file)!r}).write_text(sys.argv[1]) + Path({str(pid_file)!r}).write_text(str(os.getpid())) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + time.sleep(60) + """ + ) + ) + binary.chmod(0o700) + return binary + + +def test_managed_pgbouncer_stop_is_bounded_and_guards_its_owned_pid(tmp_path: Path) -> None: + port: Final = allocate_port() + runtime: Final = start_managed_pgbouncer( + PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pgbouncer(tmp_path))), + "postgresql://litellm:password@127.0.0.1/litellm", + ) + assert isinstance(runtime, ManagedPgBouncerRuntime), runtime + pid: Final = runtime.pooler.pid + deadline: Final = time.monotonic() + 2 + try: + with runtime.owned_pid_guard() as guarded_pid: + assert guarded_pid == pid + finally: + runtime.stop(deadline) + assert pid is not None + assert not Path(f"/proc/{pid}").exists() + assert not runtime.runtime_dir.exists() + + +def test_launch_cleans_up_managed_pooler_when_startup_is_interrupted(tmp_path: Path) -> None: + pid_file: Final = tmp_path / "pooler.pid" + config_file: Final = tmp_path / "pooler-config" + process: Final = subprocess.Popen( # test-quality-ok: child must import the working-tree gateway.launch module + ( + sys.executable, + "-m", + "gateway.launch", + "--host", + "127.0.0.1", + "--port", + str(allocate_port()), + ), + cwd=ROOT, + env={ + **os.environ, + "DATABASE_URL": "postgresql://litellm:password@127.0.0.1/litellm", + "LITELLM_PGBOUNCER_BINARY": str(_slow_starting_pgbouncer(tmp_path, pid_file, config_file)), + "LITELLM_PGBOUNCER_ENABLED": "false", + "LITELLM_PGBOUNCER_PORT": str(allocate_port()), + "LITELLM_RUST_GATEWAY": "true", + "LITELLM_RUST_GATEWAY_BINARY": "/bin/true", + }, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + try: + pooler_pid: Final = _wait_for_pid(pid_file) + os.kill(process.pid, signal.SIGTERM) # test-quality-ok: Interrupt startup to exercise managed cleanup + assert process.wait(timeout=20) == 143 + assert not Path(f"/proc/{pooler_pid}").exists() + runtime_dir: Final = Path(config_file.read_text()).parent + assert not runtime_dir.exists() + finally: + _terminate_if_running(process) + + +def _wait_for_pid(pid_file: Path) -> int: + deadline: Final = time.monotonic() + 5 + pooler_pid: Final = next( + (pid for pid in _recorded_pooler_pids(pid_file, deadline) if Path(f"/proc/{pid}").exists()), + None, + ) + if pooler_pid is None: + pytest.fail("managed PgBouncer did not record a live process") + return pooler_pid + + +def _recorded_pooler_pids(pid_file: Path, deadline: float) -> Iterator[int]: + while time.monotonic() < deadline: + try: + yield int(pid_file.read_text()) + except (OSError, ValueError): + pass + time.sleep(0.05) + + +def _wait_for_replacement_pooler(pid_file: Path, original_pid: int) -> int | None: + deadline: Final = time.monotonic() + 5 + return next( + ( + pid + for pid in _recorded_pooler_pids(pid_file, deadline) + if pid != original_pid and Path(f"/proc/{pid}").exists() + ), + None, + ) + + +def test_database_free_python_child_runs_launch_with_pooling_disabled(tmp_path: Path) -> None: + serve_marker: Final = tmp_path / "serve" + inherited_environment: Final = { + name: value + for name, value in os.environ.items() + if not name.startswith("DATABASE_") and name not in ("DIRECT_URL", "LITELLM_PGBOUNCER_POOLED_DATABASE_URL") + } + rust_environment: Final = { + **inherited_environment, + "LITELLM_RUST_GATEWAY": "false", + "LITELLM_PGBOUNCER_ENABLED": "true", + "TEST_SERVE_MARKER": str(serve_marker), + } + python_environment, _ = _pooled_child_environments(rust_environment, {}, None) + child: Final = subprocess.run( + (sys.executable, "-P", "-c", LAUNCH_MAIN_SOURCE), + cwd=ROOT, + env=python_environment, + capture_output=True, + timeout=10, + ) + assert child.returncode == 0, child.stderr.decode(errors="replace") + assert serve_marker.read_text() == "false|" + + +def _wait_for_gateway(process: subprocess.Popen[bytes], port: int) -> None: + deadline: Final = time.monotonic() + 10 + while time.monotonic() < deadline: + if process.poll() is not None: + pytest.fail(f"supervisor exited early with status {process.returncode}") + if _gateway_readiness_status(port) == 200: + return + time.sleep(0.05) + pytest.fail("supervisor did not start both child listeners") + + +def _gateway_readiness_status(port: int) -> int | None: + connection: Final = http.client.HTTPConnection("127.0.0.1", port, timeout=0.2) + try: + connection.request("GET", "/health/readiness") + response: Final = connection.getresponse() + response.read() + return response.status + except OSError: + return None + finally: + connection.close() + + +def _terminate_if_running(process: subprocess.Popen[bytes]) -> None: + if process.poll() is not None: + return + os.killpg(process.pid, signal.SIGKILL) # test-quality-ok: Cleanup terminates only this test-owned supervisor group + process.wait(timeout=5) + + +def test_supervisor_drains_rust_before_python(tmp_path: Path) -> None: + private_port: Final = allocate_port() + public_port: Final = allocate_port() + events: Final = tmp_path / "signals" + process: Final = _start_supervisor(tmp_path, private_port, public_port, events) + try: + _wait_for_gateway(process, public_port) + os.kill(process.pid, signal.SIGTERM) # test-quality-ok: The signal is the ordered-shutdown event under test + assert process.wait(timeout=10) == 0 + assert events.read_text().splitlines() == ["rust", "python"] + finally: + _terminate_if_running(process) + + +def test_supervisor_fails_when_python_cannot_start(tmp_path: Path) -> None: + process: Final = _start_supervisor( + tmp_path, + allocate_port(), + allocate_port(), + tmp_path / "signals", + python_mode="startup-failure", + ) + try: + assert process.wait(timeout=10) == 1 + finally: + _terminate_if_running(process) + + +def test_supervisor_cleans_up_python_when_rust_cannot_start(tmp_path: Path) -> None: + events: Final = tmp_path / "signals" + process: Final = _start_supervisor(tmp_path, allocate_port(), allocate_port(), events, rust_mode="missing") + try: + assert process.wait(timeout=10) == 1 + assert events.read_text().splitlines() == ["python"] + finally: + _terminate_if_running(process) + + +def test_supervisor_cleans_children_and_managed_pooler_when_rust_start_raises(tmp_path: Path) -> None: + server: Final = tmp_path / "server.py" + server.write_text(SERVER_SOURCE) + python_pid_file: Final = tmp_path / "python.pid" + python_command: Final = ( + sys.executable, + "-P", + str(server), + "python", + str(allocate_port()), + str(tmp_path / "events"), + "normal", + str(tmp_path / "orphan.pid"), + ) + pooler: Final = start_managed_pgbouncer( + PgBouncerSettings(enabled=True, port=allocate_port(), binary=str(_fake_pgbouncer(tmp_path))), + "postgresql://litellm:password@127.0.0.1/litellm", + ) + assert isinstance(pooler, ManagedPgBouncerRuntime), pooler + pooler_pid: Final = pooler.pooler.pid + config: Final = SupervisorConfig( + python_command=python_command, + rust_command=("/injected/rust-gateway",), + python_environment={}, + rust_environment={}, + public_probe_host="127.0.0.1", + public_port=allocate_port(), + private_port=int(python_command[4]), + startup_timeout=5, + rust_drain_timeout=1, + python_grace_timeout=1, + termination_grace_timeout=5, + managed_pooler=pooler, + ) + try: + with ExitStack() as child_cleanup: + failure: Final = supervise( + config, + runner=_FailAfterPythonStartRunner(python_command, python_pid_file, child_cleanup), + ) + assert isinstance(failure, SupervisorFailure) + assert "injected Rust gateway start failure" in failure.reason + python_pid: Final = int(python_pid_file.read_text()) + assert not Path(f"/proc/{python_pid}").exists() + assert pooler_pid is not None + assert not Path(f"/proc/{pooler_pid}").exists() + assert not pooler.runtime_dir.exists() + finally: + pooler.stop(time.monotonic() + 2) + + +@pytest.mark.parametrize("exiting_child", ("python", "rust")) +def test_supervisor_fails_when_either_child_exits(tmp_path: Path, exiting_child: str) -> None: + events: Final = tmp_path / "signals" + process: Final = _start_supervisor( + tmp_path, + allocate_port(), + allocate_port(), + events, + python_mode="exit" if exiting_child == "python" else "normal", + rust_mode="exit" if exiting_child == "rust" else "normal", + ) + try: + assert process.wait(timeout=10) == 1 + finally: + _terminate_if_running(process) + + +def test_supervisor_kills_unresponsive_rust_group_and_reaps_orphan(tmp_path: Path) -> None: + private_port: Final = allocate_port() + public_port: Final = allocate_port() + events: Final = tmp_path / "signals" + child_pid_file: Final = tmp_path / "orphan.pid" + process: Final = _start_supervisor( + tmp_path, + private_port, + public_port, + events, + rust_mode="orphan", + rust_timeout=1, + ) + try: + _wait_for_gateway(process, public_port) + deadline: Final = time.monotonic() + 5 + while not child_pid_file.is_file() and time.monotonic() < deadline: + time.sleep(0.05) + assert child_pid_file.is_file() + child_pid: Final = int(child_pid_file.read_text()) + os.kill(process.pid, signal.SIGTERM) # test-quality-ok: The signal is the orphan-cleanup event under test + assert process.wait(timeout=10) == 0 + assert events.read_text().splitlines() == ["rust", "python"] + assert not Path(f"/proc/{child_pid}").exists() + finally: + _terminate_if_running(process) + + +def test_supervisor_reaps_adopted_child_while_gateway_is_ready(tmp_path: Path) -> None: + public_port: Final = allocate_port() + child_pid_file: Final = tmp_path / "orphan.pid" + pooler_pid_file: Final = tmp_path / "pooler.pid" + process: Final = _start_supervisor( + tmp_path, + allocate_port(), + public_port, + tmp_path / "signals", + rust_mode="orphan-exit", + managed_pooler=(_fake_pgbouncer(tmp_path), allocate_port(), pooler_pid_file), + ) + try: + _wait_for_gateway(process, public_port) + pooler_pid: Final = _wait_for_pid(pooler_pid_file) + deadline: Final = time.monotonic() + 5 + while not child_pid_file.is_file() and time.monotonic() < deadline: + time.sleep(0.05) + assert child_pid_file.is_file() + child_pid: Final = int(child_pid_file.read_text()) + child_deadline: Final = time.monotonic() + 5 + while Path(f"/proc/{child_pid}").exists() and time.monotonic() < child_deadline: + time.sleep(0.05) + assert not Path(f"/proc/{child_pid}").exists() + assert process.poll() is None + assert Path(f"/proc/{pooler_pid}").exists() + os.kill(pooler_pid, signal.SIGKILL) # test-quality-ok: The signal exercises supervised PgBouncer restart + replacement_pid: Final = _wait_for_replacement_pooler(pooler_pid_file, pooler_pid) + assert replacement_pid is not None + assert replacement_pid != pooler_pid + _wait_for_gateway(process, public_port) + os.kill(process.pid, signal.SIGTERM) # test-quality-ok: Signal shutdown after observing the adopted child reap + assert process.wait(timeout=10) == 0 + assert not Path(f"/proc/{replacement_pid}").exists() + finally: + _terminate_if_running(process) + + +def test_supervisor_stops_managed_pooler_when_python_startup_fails(tmp_path: Path) -> None: + pooler_pid_file: Final = tmp_path / "pooler.pid" + process: Final = _start_supervisor( + tmp_path, + allocate_port(), + allocate_port(), + tmp_path / "signals", + python_mode="startup-failure", + managed_pooler=(_fake_pgbouncer(tmp_path), allocate_port(), pooler_pid_file), + ) + try: + assert process.wait(timeout=10) == 1 + pooler_pid: Final = int(pooler_pid_file.read_text()) + assert not Path(f"/proc/{pooler_pid}").exists() + finally: + _terminate_if_running(process) + + +def test_supervisor_bounds_shutdown_with_an_open_sse_response(tmp_path: Path) -> None: + public_port: Final = allocate_port() + events: Final = tmp_path / "signals" + process: Final = _start_supervisor( + tmp_path, + allocate_port(), + public_port, + events, + rust_mode="ignore", + rust_timeout=1, + ) + connection: Final = http.client.HTTPConnection("127.0.0.1", public_port, timeout=2) + try: + _wait_for_gateway(process, public_port) + connection.request("GET", "/events") + response: Final = connection.getresponse() + assert response.status == 200 + assert response.getheader("Content-Type") == "text/event-stream" + assert response.readline() == b"data: connected\n" + os.kill(process.pid, signal.SIGTERM) # test-quality-ok: The signal starts the bounded SSE drain under test + assert process.wait(timeout=10) == 0 + assert events.read_text().splitlines() == ["rust", "python"] + finally: + connection.close() + _terminate_if_running(process) diff --git a/tests/integration/configuration/test_rust_gateway_trusted_proxy.py b/tests/integration/configuration/test_rust_gateway_trusted_proxy.py new file mode 100644 index 00000000000..83e6726ccb1 --- /dev/null +++ b/tests/integration/configuration/test_rust_gateway_trusted_proxy.py @@ -0,0 +1,183 @@ +import http.client +import os +import signal +import subprocess +import sys +import time +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +import pytest +from pydantic import TypeAdapter + +from tests.integration._support.ports import allocate_port + +ROOT: Final = Path(__file__).resolve().parents[3] +DEFAULT_BINARY: Final = ROOT / "litellm-rust" / "target" / "debug" / "litellm-gateway" +HEADER_OBSERVATION: Final = TypeAdapter(dict[str, str | None]) +APP_SOURCE: Final = """\ +from fastapi import FastAPI, Request + +app = FastAPI() + +@app.get("/health/readiness") +async def readiness(): + return {"status": "healthy"} + +@app.get("/headers") +async def headers(request: Request): + return { + "client": request.client.host if request.client is not None else None, + "host": request.headers.get("host"), + "origin": request.headers.get("origin"), + "xff": request.headers.get("x-forwarded-for"), + "proto": request.headers.get("x-forwarded-proto"), + "scheme": request.url.scheme, + } +""" + + +def _request(address: str, port: int, source: str | None = None) -> dict[str, str | None]: + client: Final = http.client.HTTPConnection( + address, + port, + timeout=1, + source_address=None if source is None else (source, 0), + ) + try: + client.request( + "GET", + "/headers", + headers={ + "Host": "gateway.example", + "Origin": "https://ui.example", + "X-Forwarded-For": "198.51.100.7", + "X-Forwarded-Proto": "https", + "Connection": "close", + }, + ) + response: Final = client.getresponse() + body: Final = response.read() + assert response.status == 200, body + return HEADER_OBSERVATION.validate_json(body) + finally: + client.close() + + +def _wait_until_ready(address: str, port: int, process: subprocess.Popen[bytes]) -> None: + deadline: Final = time.monotonic() + 10 + while time.monotonic() < deadline: + if process.poll() is not None: + _stdout, stderr = process.communicate() + pytest.fail( + f"Rust gateway exited during startup with status {process.returncode}: " + f"{(stderr or b'').decode(errors='replace')}" + ) + connection: Final = http.client.HTTPConnection(address, port, timeout=0.2) + try: + connection.request("GET", "/health/readiness") + response: Final = connection.getresponse() + response.read() + if response.status == 200: + return + except OSError: + pass + finally: + connection.close() + time.sleep(0.05) + pytest.fail("Rust gateway did not forward readiness to the Uvicorn fixture") + + +@pytest.mark.parametrize( + ("trusted_proxies", "expected_client", "expected_proto"), + ( + ("", "127.0.0.2", "http"), + ("127.0.0.2", "198.51.100.7", "https"), + ), +) +def test_rust_gateway_canonicalizes_headers_before_real_uvicorn( + tmp_path: Path, + trusted_proxies: str, + expected_client: str, + expected_proto: str, +) -> None: + binary: Final = Path(os.environ.get("LITELLM_RUST_GATEWAY_BINARY", str(DEFAULT_BINARY))) + if not binary.is_file(): + pytest.skip("build litellm-rust/crates/gateway before running the real Uvicorn trust fixture") + + python_port: Final = allocate_port() + rust_port: Final = allocate_port() + app_path: Final = tmp_path / "fixture.py" + app_path.write_text(APP_SOURCE) + python_environment: Final = { + **os.environ, + "PYTHONPATH": str(tmp_path), + "FORWARDED_ALLOW_IPS": "127.0.0.1", + } + python: Final = subprocess.Popen( + ( + sys.executable, + "-P", + "-m", + "uvicorn", + "fixture:app", + "--host", + "127.0.0.1", + "--port", + str(python_port), + "--proxy-headers", + "--forwarded-allow-ips", + "127.0.0.1", + ), + env=python_environment, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + start_new_session=True, + ) + _wait_until_ready("127.0.0.1", python_port, python) + direct_python: Final = _request("127.0.0.1", python_port, "127.0.0.2") + assert direct_python == { + "client": "127.0.0.2", + "host": "gateway.example", + "origin": "https://ui.example", + "xff": "198.51.100.7", + "proto": "https", + "scheme": "http", + } + rust_environment: Final[Mapping[str, str]] = { + **os.environ, + "HOST": "127.0.0.1", + "PORT": str(rust_port), + "LITELLM_RUST_GATEWAY": "true", + "LITELLM_PYTHON_UPSTREAM": f"http://127.0.0.1:{python_port}", + "LITELLM_RUST_GATEWAY_TRUSTED_PROXIES": trusted_proxies, + } + rust: Final = subprocess.Popen( + (str(binary),), + env=rust_environment, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + start_new_session=True, + ) + try: + _wait_until_ready("127.0.0.1", rust_port, rust) + response: Final = _request("127.0.0.1", rust_port, "127.0.0.2") + + assert response == { + "client": expected_client, + "host": "gateway.example", + "origin": "https://ui.example", + "xff": expected_client, + "proto": expected_proto, + "scheme": expected_proto, + } + finally: + for process in (rust, python): + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) # test-quality-ok: Signal only this fixture's process group + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) # test-quality-ok: Kill only this fixture's process group + process.wait(timeout=5) diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index f76aea030fe..900b3dbeddd 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -3,6 +3,7 @@ import os import socket import sys import textwrap +import time import urllib.parse from collections.abc import Iterator from pathlib import Path @@ -13,9 +14,27 @@ import pytest from uvicorn.importer import import_from_string from uvicorn.main import main as uvicorn_main -from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv +from gateway.launch import ( + GATEWAY_APP, + _pooled_child_environments, + main, + managed_pool_database_url, + pool_database_url, + uvicorn_argv, +) +from gateway.supervisor import ( + RUST_GATEWAY_BINARY, + ConfigurationFailure, + build_configuration, + gateway_enabled, +) from litellm.proxy.db.db_url_settings import DatabaseURLSettings -from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings +from litellm.proxy.db.pgbouncer import ( + PGBOUNCER_POOLED_ENV_VAR, + ManagedPgBouncerRuntime, + PgBouncerError, + PgBouncerSettings, +) from litellm.proxy.proxy_server import app as proxy_app DB_ENV: Final = { @@ -127,6 +146,50 @@ class TestUvicornArgv: class TestPoolDatabaseUrl: + def test_rust_mode_without_a_writer_stays_database_free(self, password_env: dict[str, str]) -> None: + assert ( + managed_pool_database_url( + DatabaseURLSettings.from_env(), + PgBouncerSettings(enabled=True), + {"DATABASE_URL": ""}, + ) + is None + ) + + def test_rust_managed_pooler_is_forced_for_a_writer_when_the_flag_is_off( + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ): + port: Final = _free_port() + binary: Final = _fake_pooler(tmp_path) + monkeypatch.setenv("DATABASE_URL", "postgresql://litellm_pool:p%40ss@db.internal:5432/litellm") + outcome: Final = managed_pool_database_url( + DatabaseURLSettings.from_env(), + PgBouncerSettings(enabled=False, port=port, binary=str(binary)), + os.environ, + ) + assert isinstance(outcome, ManagedPgBouncerRuntime), outcome + try: + assert urllib.parse.urlsplit(outcome.pooled_url).port == port + configuration: Final = build_configuration( + ("--host", "0.0.0.0", "--port", "4000"), + {RUST_GATEWAY_BINARY: "/bin/true"}, + sys.executable, + ) + assert not isinstance(configuration, ConfigurationFailure) + python_environment, rust_environment = _pooled_child_environments( + configuration.python_environment, + configuration.rust_environment, + outcome, + ) + assert python_environment["DATABASE_URL"] == rust_environment["DATABASE_URL"] == outcome.pooled_url + assert python_environment[PGBOUNCER_POOLED_ENV_VAR] == rust_environment[PGBOUNCER_POOLED_ENV_VAR] == "true" + assert python_environment["LITELLM_PGBOUNCER_ENABLED"] == "false" + assert rust_environment.get("LITELLM_PGBOUNCER_ENABLED") == configuration.rust_environment.get( + "LITELLM_PGBOUNCER_ENABLED" + ) + finally: + outcome.stop(time.monotonic() + 2) + def test_a_disabled_pooler_yields_no_url_to_install(self, password_env: dict[str, str]): settings: Final = DatabaseURLSettings.from_env() settings.apply_to_env() @@ -158,6 +221,149 @@ class TestPoolDatabaseUrl: assert "MINTED_TOKEN" not in outcome +class TestRustGatewayConfiguration: + @pytest.fixture + def rust_binary(self, tmp_path: Path) -> Path: + binary: Final = tmp_path / "litellm-gateway" + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o700) + return binary + + @pytest.fixture + def clear_uvicorn_environment(self, monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + "UVICORN_HOST", + "UVICORN_PORT", + "UVICORN_UDS", + "UVICORN_FD", + "UVICORN_RELOAD", + "UVICORN_PROXY_HEADERS", + "UVICORN_SSL_KEYFILE", + ): + monkeypatch.delenv(name, raising=False) + + def test_flag_requires_exact_lowercase_values(self) -> None: + assert gateway_enabled({}) is False + assert gateway_enabled({"LITELLM_RUST_GATEWAY": "true"}) is True + assert isinstance(gateway_enabled({"LITELLM_RUST_GATEWAY": "TRUE"}), ConfigurationFailure) + + def test_private_worker_keeps_config_rootpath_keepalive_and_unknown_options( + self, + rust_binary: Path, + clear_uvicorn_environment: None, + ) -> None: + environ: Final = { + RUST_GATEWAY_BINARY: str(rust_binary), + "CONFIG_FILE_PATH": "/app/config/config.yaml", + "KEEPALIVE_TIMEOUT": "41", + } + configuration: Final = build_configuration( + ( + "--workers", + "3", + "--proxy-headers", + "--host", + "0.0.0.0", + "--port", + "4000", + "--root-path", + "/gateway", + "--unknown-option", + "preserved", + "--proxy-headers", + ), + environ, + sys.executable, + ) + + assert not isinstance(configuration, ConfigurationFailure) + assert configuration.python_command == ( + sys.executable, + "-m", + "gateway.launch", + "--host", + "127.0.0.1", + "--port", + "4002", + "--proxy-headers", + "--forwarded-allow-ips", + "127.0.0.1", + "--workers", + "3", + "--root-path", + "/gateway", + "--unknown-option", + "preserved", + "--timeout-keep-alive", + "41", + ) + assert configuration.python_environment["CONFIG_FILE_PATH"] == "/app/config/config.yaml" + assert configuration.python_environment["LITELLM_RUST_GATEWAY"] == "false" + assert configuration.rust_environment["LITELLM_RUST_GATEWAY"] == "true" + assert configuration.rust_environment["LITELLM_PYTHON_UPSTREAM"] == "http://127.0.0.1:4002" + assert configuration.rust_environment["HOST"] == "0.0.0.0" + assert configuration.rust_environment["PORT"] == "4000" + + def test_database_writer_reserves_pgbouncer_port_when_the_python_only_flag_is_false( + self, rust_binary: Path, clear_uvicorn_environment: None + ) -> None: + configuration: Final = build_configuration( + ("--host", "0.0.0.0", "--port", "4000"), + { + RUST_GATEWAY_BINARY: str(rust_binary), + "DATABASE_URL": "postgresql://app:pw@db/litellm", + "LITELLM_PGBOUNCER_ENABLED": "false", + "LITELLM_PGBOUNCER_PORT": "4000", + }, + sys.executable, + ) + assert isinstance(configuration, ConfigurationFailure) + assert "PgBouncer" in configuration.reason + assert "same TCP port" in configuration.reason + + @pytest.mark.parametrize( + ("arguments", "extra_environment", "reason"), + ( + (("--no-proxy-headers",), {}, "--no-proxy-headers"), + (("--ssl-certfile", "/tmp/cert.pem"), {}, "TLS"), + (("--env-file", "/tmp/uvicorn.env"), {}, "--env-file"), + (("--uds", "/tmp/gateway.sock"), {}, "Unix socket"), + (("--fd", "5"), {}, "file descriptor"), + (("--http", "h2"), {}, "HTTP/1.1"), + (("--host", "0.0.0.0", "--port", "4000"), {"PORT": "4001"}, "PORT conflicts"), + (("--host", "0.0.0.0", "--port", "4000"), {"LITELLM_GATEWAY_METRICS_PORT": "4002"}, "same TCP port"), + ( + ("--host", "0.0.0.0", "--port", "4000"), + {"LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS": "20"}, + "must allow Rust drain", + ), + ( + ("--host", "0.0.0.0", "--port", "4000"), + {"LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS": "0"}, + "must be a positive integer", + ), + ( + ("--host", "0.0.0.0", "--port", "4000"), + {"LITELLM_COLLECTOR_ENABLED": "true", "LITELLM_COLLECTOR_ADDRESS": "tcp://127.0.0.1:4002"}, + "same TCP port", + ), + ), + ) + def test_rejects_unsupported_or_conflicting_configuration( + self, + rust_binary: Path, + clear_uvicorn_environment: None, + arguments: tuple[str, ...], + extra_environment: dict[str, str], + reason: str, + ) -> None: + environ: Final = {RUST_GATEWAY_BINARY: str(rust_binary), **extra_environment} + outcome: Final = build_configuration(arguments, environ, sys.executable) + + assert isinstance(outcome, ConfigurationFailure) + assert reason in outcome.reason + + class TestMain: def test_workers_inherit_the_loopback_url_the_supervisor_installed( self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path