mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 4b57d19672 into 0c85038dc3
This commit is contained in:
commit
14e341b710
17 changed files with 2290 additions and 11 deletions
17
gateway/AGENTS.md
Normal file
17
gateway/AGENTS.md
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# Gateway
|
||||
|
||||
## Rust gateway
|
||||
|
||||
- `LITELLM_RUST_GATEWAY=true` enables the public Rust listener. Unset or exact `false` preserves Python-only startup
|
||||
- Keep `LITELLM_RUST` independent. It controls the in-process Rust bridge
|
||||
- Rust forwards every request to Python, which owns authentication, routing, policy and provider calls. Do not add native request selection or database initialization here
|
||||
- `launch.py` owns startup and pooling. `supervisor.py` coordinates container processes, not application requests. A Python or Rust child exit fails the container rather than restarting that child
|
||||
- Start Python on loopback `127.0.0.1:4002` before Rust on public port 4000. Reserve 4001 for metrics and never expose the private Python port in container or Service ports
|
||||
- With a writer `DATABASE_URL`, start one managed PgBouncer even when `LITELLM_PGBOUNCER_ENABLED=false`. Both children receive the same pooled URL and marker, and Python must not start another pooler
|
||||
- Keep database-free mode database-free. Read replicas remain outside the shared writer connection cap
|
||||
- Drain Rust, stop Python, then stop PgBouncer and reap remaining children within the shutdown deadline. Defaults allow 15 seconds for Rust, 5 for Python and a 10-second cleanup margin
|
||||
- Docker stop timeout and Helm termination grace must cover `LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS`, which defaults to 30
|
||||
- Configure `LITELLM_RUST_GATEWAY_TRUSTED_PROXIES` with explicit ingress IPs or CIDRs. Empty trusts no external proxy, wildcard trust is rejected, and `Host` and `Origin` are preserved
|
||||
- Terminate TLS at the edge. Enabled mode rejects Uvicorn TLS, reload, Unix-socket and file-descriptor listeners, env-file overrides and disabled proxy headers
|
||||
- The image packages `/usr/local/bin/litellm-gateway`. Local runs may override `LITELLM_RUST_GATEWAY_BINARY` and the private port through `LITELLM_RUST_GATEWAY_PYTHON_PORT`
|
||||
- Roll back by removing the Rust gateway opt-in or setting it to `false` and redeploying
|
||||
|
|
@ -47,6 +47,20 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
|||
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
|
||||
python -c "import maturin; maturin.build_wheel('/wheels')"
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cargo/registry \
|
||||
--mount=type=cache,target=/root/.cargo/git \
|
||||
--mount=type=cache,target=/app/litellm-rust/target \
|
||||
CARGO_BIN="$(find /root/.cache /root/.cargo /root/.rustup /usr/local \( -type f -o -type l \) -name cargo -print -quit)" && \
|
||||
test -x "$CARGO_BIN" || { echo "Rust cargo executable was not found" >&2; exit 1; } && \
|
||||
cd litellm-rust && \
|
||||
PATH="$(dirname "$CARGO_BIN"):${PATH}" "$CARGO_BIN" build --locked --release -p litellm-gateway --bin litellm-gateway && \
|
||||
mkdir -p /rust-gateway && \
|
||||
cp target/release/litellm-gateway /rust-gateway/litellm-gateway && \
|
||||
readelf -l /rust-gateway/litellm-gateway > /tmp/litellm-gateway-linkage && \
|
||||
grep -q "Requesting program interpreter:" /tmp/litellm-gateway-linkage && \
|
||||
readelf -d /rust-gateway/litellm-gateway >> /tmp/litellm-gateway-linkage && \
|
||||
grep -q "(NEEDED)" /tmp/litellm-gateway-linkage
|
||||
|
||||
# ---------- Builder ----------
|
||||
FROM $LITELLM_BUILD_IMAGE AS builder
|
||||
|
||||
|
|
@ -148,12 +162,13 @@ COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh
|
|||
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
|
||||
COPY --from=builder /opt/prisma /opt/prisma
|
||||
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
|
||||
COPY --from=rust-native /rust-gateway/litellm-gateway /usr/local/bin/litellm-gateway
|
||||
|
||||
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
|
||||
find /app/.venv -type d -path "*/tornado/test" -delete && \
|
||||
RUN find /app/.venv -type d -path "*/tornado/test" -prune -exec rm -r {} \; && \
|
||||
chmod -R a+rX /opt/prisma && \
|
||||
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \
|
||||
python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()"
|
||||
python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()" && \
|
||||
if LITELLM_RUST_GATEWAY=true /usr/local/bin/litellm-gateway 2>&1 | grep -Fq "Error: PythonUpstreamConfig"; then :; else exit 1; fi
|
||||
|
||||
USER nonroot
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,10 @@ Run with:
|
|||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from contextlib import ExitStack
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from uvicorn.main import main as uvicorn_main
|
||||
|
|
@ -22,11 +25,13 @@ from uvicorn.main import main as uvicorn_main
|
|||
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
||||
from litellm.proxy.db.pgbouncer import (
|
||||
PGBOUNCER_MANAGED_SHUTDOWN_SECONDS,
|
||||
ManagedPgBouncerRuntime,
|
||||
PgBouncerError,
|
||||
PgBouncerSettings,
|
||||
export_pooled_database_url,
|
||||
pgbouncer_startup_signals,
|
||||
start_in_container_pgbouncer,
|
||||
start_managed_pgbouncer,
|
||||
)
|
||||
|
||||
GATEWAY_APP: Final = "gateway.main:app"
|
||||
|
|
@ -65,11 +70,92 @@ def pool_database_url(
|
|||
)
|
||||
|
||||
|
||||
def managed_pool_database_url(
|
||||
settings: DatabaseURLSettings,
|
||||
pgbouncer: PgBouncerSettings,
|
||||
environ: Mapping[str, str],
|
||||
) -> ManagedPgBouncerRuntime | PgBouncerError | None:
|
||||
upstream_url: Final = environ.get("DATABASE_URL")
|
||||
if not upstream_url:
|
||||
return None
|
||||
effective_settings: Final = pgbouncer if pgbouncer.enabled else pgbouncer.model_copy(update={"enabled": True})
|
||||
return start_managed_pgbouncer(effective_settings, upstream_url, token_auth=settings.token_auth())
|
||||
|
||||
|
||||
def _pooled_child_environments(
|
||||
python_environment: Mapping[str, str],
|
||||
rust_environment: Mapping[str, str],
|
||||
pooler: ManagedPgBouncerRuntime | None,
|
||||
) -> tuple[Mapping[str, str], Mapping[str, str]]:
|
||||
if pooler is None:
|
||||
return (
|
||||
MappingProxyType({**python_environment, "LITELLM_PGBOUNCER_ENABLED": "false"}),
|
||||
rust_environment,
|
||||
)
|
||||
pooled_values: Final = {
|
||||
"DATABASE_URL": pooler.pooled_url,
|
||||
"LITELLM_PGBOUNCER_POOLED_DATABASE_URL": "true",
|
||||
}
|
||||
return (
|
||||
MappingProxyType(
|
||||
{
|
||||
**python_environment,
|
||||
**pooled_values,
|
||||
"LITELLM_PGBOUNCER_ENABLED": "false",
|
||||
}
|
||||
),
|
||||
MappingProxyType({**rust_environment, **pooled_values}),
|
||||
)
|
||||
|
||||
|
||||
def _serve(argv: Sequence[str]) -> None:
|
||||
uvicorn_main(tuple(argv), prog_name="uvicorn")
|
||||
|
||||
|
||||
def _stop_starting_pooler(pooler: ManagedPgBouncerRuntime, timeout: int) -> None:
|
||||
pooler.stop(time.monotonic() + timeout)
|
||||
|
||||
|
||||
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
|
||||
gateway_mode: Final = os.environ.get("LITELLM_RUST_GATEWAY")
|
||||
if gateway_mode not in (None, "true", "false"):
|
||||
sys.exit("LiteLLM gateway: LITELLM_RUST_GATEWAY must be exactly 'true' or 'false'")
|
||||
if gateway_mode == "true":
|
||||
from dataclasses import replace
|
||||
|
||||
from gateway.supervisor import ConfigurationFailure, build_configuration, startup_scope, supervise
|
||||
|
||||
with ExitStack() as startup_cleanup, startup_scope():
|
||||
rust_settings: Final = DatabaseURLSettings.from_env()
|
||||
rust_settings.apply_to_env()
|
||||
preflight_configuration: Final = build_configuration(argv, os.environ, sys.executable)
|
||||
if isinstance(preflight_configuration, ConfigurationFailure):
|
||||
sys.exit(f"LiteLLM gateway supervisor: {preflight_configuration.reason}")
|
||||
pooler: Final = managed_pool_database_url(rust_settings, PgBouncerSettings(), os.environ)
|
||||
if isinstance(pooler, PgBouncerError):
|
||||
sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooler.reason}")
|
||||
if pooler is not None:
|
||||
startup_cleanup.callback(
|
||||
_stop_starting_pooler, pooler, preflight_configuration.termination_grace_timeout
|
||||
)
|
||||
export_pooled_database_url(pooler.pooled_url)
|
||||
python_environment, rust_environment = _pooled_child_environments(
|
||||
preflight_configuration.python_environment,
|
||||
preflight_configuration.rust_environment,
|
||||
pooler,
|
||||
)
|
||||
final_configuration: Final = replace(
|
||||
preflight_configuration,
|
||||
python_environment=python_environment,
|
||||
rust_environment=rust_environment,
|
||||
managed_pooler=pooler,
|
||||
)
|
||||
failure: Final = supervise(final_configuration)
|
||||
startup_cleanup.pop_all()
|
||||
if failure is not None:
|
||||
sys.exit(f"LiteLLM gateway supervisor: {failure.reason}")
|
||||
return
|
||||
|
||||
pgbouncer: Final = PgBouncerSettings()
|
||||
with pgbouncer_startup_signals(pgbouncer.enabled):
|
||||
settings: Final = DatabaseURLSettings.from_env()
|
||||
|
|
|
|||
725
gateway/supervisor.py
Normal file
725
gateway/supervisor.py
Normal file
|
|
@ -0,0 +1,725 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import http.client
|
||||
import ipaddress
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Callable, Generator, Mapping, Sequence
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
from types import FrameType, MappingProxyType
|
||||
from typing import Final, Protocol, cast
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import click
|
||||
from pydantic import TypeAdapter
|
||||
from uvicorn.main import main as uvicorn_main
|
||||
|
||||
from gateway.launch import uvicorn_argv
|
||||
from litellm.proxy.db.pgbouncer import ManagedPgBouncerRuntime, pgbouncer_startup_signals
|
||||
|
||||
RUST_GATEWAY_FLAG: Final = "LITELLM_RUST_GATEWAY"
|
||||
RUST_GATEWAY_BINARY: Final = "LITELLM_RUST_GATEWAY_BINARY"
|
||||
PYTHON_PORT_ENV: Final = "LITELLM_RUST_GATEWAY_PYTHON_PORT"
|
||||
PYTHON_UPSTREAM_ENV: Final = "LITELLM_PYTHON_UPSTREAM"
|
||||
TRUSTED_PROXIES_ENV: Final = "LITELLM_RUST_GATEWAY_TRUSTED_PROXIES"
|
||||
METRICS_PORT_ENV: Final = "LITELLM_GATEWAY_METRICS_PORT"
|
||||
TERMINATION_GRACE_ENV: Final = "LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS"
|
||||
DEFAULT_RUST_GATEWAY_BINARY: Final = "/usr/local/bin/litellm-gateway"
|
||||
DEFAULT_PRIVATE_PORT: Final = 4002
|
||||
DEFAULT_PYTHON_GRACE_SECONDS: Final = 5
|
||||
DEFAULT_RUST_DRAIN_SECONDS: Final = 15
|
||||
DEFAULT_STARTUP_TIMEOUT_SECONDS: Final = 120
|
||||
DEFAULT_TERMINATION_GRACE_SECONDS: Final = 30
|
||||
TERMINATION_CLEANUP_MARGIN_SECONDS: Final = 10
|
||||
LIVENESS_PATH: Final = "/health/liveliness"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ConfigurationFailure:
|
||||
reason: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SupervisorFailure:
|
||||
reason: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SupervisorConfig:
|
||||
python_command: tuple[str, ...]
|
||||
rust_command: tuple[str, ...]
|
||||
python_environment: Mapping[str, str]
|
||||
rust_environment: Mapping[str, str]
|
||||
public_probe_host: str
|
||||
public_port: int
|
||||
private_port: int
|
||||
startup_timeout: int
|
||||
rust_drain_timeout: int
|
||||
python_grace_timeout: int
|
||||
termination_grace_timeout: int = DEFAULT_TERMINATION_GRACE_SECONDS
|
||||
managed_pooler: ManagedPgBouncerRuntime | None = None
|
||||
|
||||
|
||||
class ChildProcess(Protocol):
|
||||
@property
|
||||
def pid(self) -> int: ...
|
||||
|
||||
def poll(self) -> int | None: ...
|
||||
|
||||
def wait(self, timeout: float | None = None) -> int: ...
|
||||
|
||||
|
||||
class ProcessRunner(Protocol):
|
||||
def start(self, command: tuple[str, ...], environment: Mapping[str, str]) -> ChildProcess: ...
|
||||
|
||||
|
||||
class SubprocessRunner:
|
||||
def start(self, command: tuple[str, ...], environment: Mapping[str, str]) -> ChildProcess:
|
||||
return subprocess.Popen(command, env=environment, start_new_session=True)
|
||||
|
||||
|
||||
def gateway_enabled(environ: Mapping[str, str]) -> bool | ConfigurationFailure:
|
||||
value: Final = environ.get(RUST_GATEWAY_FLAG)
|
||||
if value is None or value == "false":
|
||||
return False
|
||||
if value == "true":
|
||||
return True
|
||||
return ConfigurationFailure(f"{RUST_GATEWAY_FLAG} must be exactly 'true' or 'false'")
|
||||
|
||||
|
||||
def _port_setting(name: str, default: int, environ: Mapping[str, str]) -> int | ConfigurationFailure:
|
||||
value: Final = environ.get(name)
|
||||
if value is None:
|
||||
return default
|
||||
try:
|
||||
port: Final = int(value)
|
||||
except ValueError:
|
||||
return ConfigurationFailure(f"{name} must be an integer port")
|
||||
if not 1 <= port <= 65535:
|
||||
return ConfigurationFailure(f"{name} must be between 1 and 65535")
|
||||
return port
|
||||
|
||||
|
||||
def _timeout_setting(name: str, default: int, environ: Mapping[str, str]) -> int | ConfigurationFailure:
|
||||
value: Final = environ.get(name)
|
||||
if value is None:
|
||||
return default
|
||||
try:
|
||||
timeout: Final = int(value)
|
||||
except ValueError:
|
||||
return ConfigurationFailure(f"{name} must be a positive integer")
|
||||
if timeout < 1:
|
||||
return ConfigurationFailure(f"{name} must be a positive integer")
|
||||
return timeout
|
||||
|
||||
|
||||
def _boolean_setting(name: str, environ: Mapping[str, str]) -> bool | ConfigurationFailure:
|
||||
value: Final = environ.get(name)
|
||||
if value is None:
|
||||
return False
|
||||
normalized: Final = value.casefold()
|
||||
if normalized in ("true", "1", "yes", "y", "on", "t"):
|
||||
return True
|
||||
if normalized in ("false", "0", "no", "n", "off", "f"):
|
||||
return False
|
||||
return ConfigurationFailure(f"{name} must be a boolean")
|
||||
|
||||
|
||||
def _collector_port(environ: Mapping[str, str]) -> int | ConfigurationFailure | None:
|
||||
enabled: Final = _boolean_setting("LITELLM_COLLECTOR_ENABLED", environ)
|
||||
if isinstance(enabled, ConfigurationFailure) or not enabled:
|
||||
return enabled if isinstance(enabled, ConfigurationFailure) else None
|
||||
address: Final = environ.get("LITELLM_COLLECTOR_ADDRESS")
|
||||
if address is None or not address.startswith("tcp://"):
|
||||
return None
|
||||
try:
|
||||
parsed: Final = urlsplit(address)
|
||||
port: Final = parsed.port
|
||||
except ValueError:
|
||||
return ConfigurationFailure("LITELLM_COLLECTOR_ADDRESS must contain a valid TCP port")
|
||||
if port is None:
|
||||
return ConfigurationFailure("LITELLM_COLLECTOR_ADDRESS must contain a valid TCP port")
|
||||
if not 1 <= port <= 65535:
|
||||
return ConfigurationFailure("LITELLM_COLLECTOR_ADDRESS must contain a valid TCP port")
|
||||
return port
|
||||
|
||||
|
||||
def _trusted_proxies(environ: Mapping[str, str]) -> str | ConfigurationFailure:
|
||||
value: Final = environ.get(TRUSTED_PROXIES_ENV, "")
|
||||
if not value:
|
||||
return value
|
||||
try:
|
||||
networks: Final = tuple(ipaddress.ip_network(entry.strip(), strict=False) for entry in value.split(","))
|
||||
except ValueError:
|
||||
return ConfigurationFailure(f"{TRUSTED_PROXIES_ENV} must contain IP addresses or CIDRs")
|
||||
if any(network.prefixlen == 0 for network in networks):
|
||||
return ConfigurationFailure(f"{TRUSTED_PROXIES_ENV} must not trust every address")
|
||||
if any(not entry.strip() for entry in value.split(",")):
|
||||
return ConfigurationFailure(f"{TRUSTED_PROXIES_ENV} contains an empty entry")
|
||||
return value
|
||||
|
||||
|
||||
def _worker_arguments(arguments: tuple[str, ...], private_port: int) -> tuple[str, ...]:
|
||||
value_options: Final = frozenset(("--host", "--port", "--forwarded-allow-ips"))
|
||||
boolean_options: Final = frozenset(("--proxy-headers",))
|
||||
consumed_values: Final = frozenset(
|
||||
index + 1 for index, argument in enumerate(arguments[:-1]) if argument in value_options
|
||||
)
|
||||
without_overrides: Final = tuple(
|
||||
argument
|
||||
for index, argument in enumerate(arguments)
|
||||
if index not in consumed_values
|
||||
and not any(
|
||||
argument == option or argument.startswith(f"{option}=") for option in value_options | boolean_options
|
||||
)
|
||||
)
|
||||
return (
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
str(private_port),
|
||||
"--proxy-headers",
|
||||
"--forwarded-allow-ips",
|
||||
"127.0.0.1",
|
||||
*without_overrides[1:],
|
||||
)
|
||||
|
||||
|
||||
def _context_value(context: click.Context, name: str) -> object:
|
||||
parameters: Final = TypeAdapter(dict[str, object]).validate_python(context.params)
|
||||
value: Final = parameters[name]
|
||||
return value
|
||||
|
||||
|
||||
def _first_port_collision(ports: tuple[tuple[str, int], ...]) -> tuple[str, str] | None:
|
||||
for index, current in enumerate(ports):
|
||||
for other in ports[index + 1 :]:
|
||||
if current[1] == other[1]:
|
||||
return current[0], other[0]
|
||||
return None
|
||||
|
||||
|
||||
def build_configuration(
|
||||
argv: Sequence[str],
|
||||
environ: Mapping[str, str],
|
||||
python_executable: str,
|
||||
) -> SupervisorConfig | ConfigurationFailure:
|
||||
arguments: Final = uvicorn_argv(argv, environ)
|
||||
try:
|
||||
context: Final = uvicorn_main.make_context(
|
||||
"uvicorn",
|
||||
list(arguments),
|
||||
allow_extra_args=True,
|
||||
ignore_unknown_options=True,
|
||||
)
|
||||
except click.ClickException as error:
|
||||
return ConfigurationFailure(f"invalid Uvicorn options: {error.format_message()}")
|
||||
|
||||
if any(argument == "--no-proxy-headers" for argument in arguments):
|
||||
return ConfigurationFailure("--no-proxy-headers is unsupported with the Rust gateway")
|
||||
if any(argument.startswith("--ssl-") for argument in arguments):
|
||||
return ConfigurationFailure("Uvicorn TLS options are unsupported with the Rust gateway")
|
||||
if any(name.startswith("UVICORN_SSL_") for name in environ):
|
||||
return ConfigurationFailure("UVICORN_SSL_* settings are unsupported with the Rust gateway")
|
||||
if _context_value(context, "env_file") is not None:
|
||||
return ConfigurationFailure("--env-file is unsupported with the Rust gateway")
|
||||
if _context_value(context, "uds") is not None or _context_value(context, "fd") is not None:
|
||||
return ConfigurationFailure("Uvicorn Unix socket and file descriptor listeners are unsupported")
|
||||
if _context_value(context, "reload") is True:
|
||||
return ConfigurationFailure("Uvicorn reload is unsupported with the Rust gateway")
|
||||
if _context_value(context, "proxy_headers") is not True:
|
||||
return ConfigurationFailure("Uvicorn proxy headers must be enabled with the Rust gateway")
|
||||
|
||||
protocol: Final = _context_value(context, "http")
|
||||
if not isinstance(protocol, str) or protocol not in ("auto", "h11", "httptools"):
|
||||
return ConfigurationFailure("the Rust gateway supports only Uvicorn HTTP/1.1 protocols")
|
||||
|
||||
host: Final = _context_value(context, "host")
|
||||
port: Final = _context_value(context, "port")
|
||||
if (
|
||||
not isinstance(host, str)
|
||||
or not host
|
||||
or any(character.isspace() or character in "/[]\r\n" for character in host)
|
||||
):
|
||||
return ConfigurationFailure("Uvicorn must have a non-empty public host")
|
||||
if not isinstance(port, int) or isinstance(port, bool) or not 1 <= port <= 65535:
|
||||
return ConfigurationFailure("Uvicorn public port must be between 1 and 65535")
|
||||
if "HOST" in environ and environ["HOST"] != host:
|
||||
return ConfigurationFailure("HOST conflicts with the effective Uvicorn host")
|
||||
if "PORT" in environ and environ["PORT"] != str(port):
|
||||
return ConfigurationFailure("PORT conflicts with the effective Uvicorn port")
|
||||
|
||||
private_port: Final = _port_setting(PYTHON_PORT_ENV, DEFAULT_PRIVATE_PORT, environ)
|
||||
if isinstance(private_port, ConfigurationFailure):
|
||||
return private_port
|
||||
metrics_port: Final = _port_setting(METRICS_PORT_ENV, 0, environ) if METRICS_PORT_ENV in environ else None
|
||||
if isinstance(metrics_port, ConfigurationFailure):
|
||||
return metrics_port
|
||||
pgbouncer_port: Final = (
|
||||
_port_setting("LITELLM_PGBOUNCER_PORT", 6432, environ) if bool(environ.get("DATABASE_URL")) else None
|
||||
)
|
||||
if isinstance(pgbouncer_port, ConfigurationFailure):
|
||||
return pgbouncer_port
|
||||
collector_port: Final = _collector_port(environ)
|
||||
if isinstance(collector_port, ConfigurationFailure):
|
||||
return collector_port
|
||||
startup_timeout: Final = _timeout_setting(
|
||||
"LITELLM_RUST_GATEWAY_STARTUP_TIMEOUT_SECONDS",
|
||||
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
environ,
|
||||
)
|
||||
if isinstance(startup_timeout, ConfigurationFailure):
|
||||
return startup_timeout
|
||||
rust_drain_timeout: Final = _timeout_setting(
|
||||
"LITELLM_RUST_GATEWAY_DRAIN_TIMEOUT_SECONDS",
|
||||
DEFAULT_RUST_DRAIN_SECONDS,
|
||||
environ,
|
||||
)
|
||||
if isinstance(rust_drain_timeout, ConfigurationFailure):
|
||||
return rust_drain_timeout
|
||||
python_grace_timeout: Final = _timeout_setting(
|
||||
"LITELLM_RUST_GATEWAY_PYTHON_GRACE_SECONDS",
|
||||
DEFAULT_PYTHON_GRACE_SECONDS,
|
||||
environ,
|
||||
)
|
||||
if isinstance(python_grace_timeout, ConfigurationFailure):
|
||||
return python_grace_timeout
|
||||
termination_grace_timeout: Final = _timeout_setting(
|
||||
TERMINATION_GRACE_ENV,
|
||||
DEFAULT_TERMINATION_GRACE_SECONDS,
|
||||
environ,
|
||||
)
|
||||
if isinstance(termination_grace_timeout, ConfigurationFailure):
|
||||
return termination_grace_timeout
|
||||
if rust_drain_timeout + python_grace_timeout + TERMINATION_CLEANUP_MARGIN_SECONDS > termination_grace_timeout:
|
||||
return ConfigurationFailure(f"{TERMINATION_GRACE_ENV} must allow Rust drain, Python grace, and cleanup margin")
|
||||
trusted_proxies: Final = _trusted_proxies(environ)
|
||||
if isinstance(trusted_proxies, ConfigurationFailure):
|
||||
return trusted_proxies
|
||||
|
||||
collisions: Final = _first_port_collision(
|
||||
(
|
||||
("public gateway", port),
|
||||
("private Python upstream", private_port),
|
||||
*(() if metrics_port is None else (("metrics server", metrics_port),)),
|
||||
*(() if pgbouncer_port is None else (("PgBouncer", pgbouncer_port),)),
|
||||
*(() if collector_port is None else (("collector", collector_port),)),
|
||||
)
|
||||
)
|
||||
if collisions is not None:
|
||||
return ConfigurationFailure(f"{collisions[0]} and {collisions[1]} use the same TCP port")
|
||||
|
||||
binary: Final = environ.get(RUST_GATEWAY_BINARY, DEFAULT_RUST_GATEWAY_BINARY)
|
||||
binary_path: Final = Path(binary)
|
||||
if not binary_path.is_file() or not os.access(binary_path, os.X_OK):
|
||||
return ConfigurationFailure(f"Rust gateway binary is not executable: {binary}")
|
||||
|
||||
upstream: Final = f"http://127.0.0.1:{private_port}"
|
||||
configured_upstream: Final = environ.get(PYTHON_UPSTREAM_ENV)
|
||||
if configured_upstream is not None and configured_upstream != upstream:
|
||||
return ConfigurationFailure(f"{PYTHON_UPSTREAM_ENV} must equal {upstream}")
|
||||
|
||||
python_environment: Final = MappingProxyType(
|
||||
{
|
||||
**environ,
|
||||
RUST_GATEWAY_FLAG: "false",
|
||||
}
|
||||
)
|
||||
rust_environment: Final = MappingProxyType(
|
||||
{
|
||||
**environ,
|
||||
RUST_GATEWAY_FLAG: "true",
|
||||
PYTHON_UPSTREAM_ENV: upstream,
|
||||
TRUSTED_PROXIES_ENV: trusted_proxies,
|
||||
"HOST": host,
|
||||
"PORT": str(port),
|
||||
}
|
||||
)
|
||||
probe_host: Final = "127.0.0.1" if host in ("0.0.0.0", "*") else "::1" if host == "::" else host
|
||||
python_command: Final = (
|
||||
python_executable,
|
||||
"-m",
|
||||
"gateway.launch",
|
||||
*_worker_arguments(arguments, private_port),
|
||||
)
|
||||
return SupervisorConfig(
|
||||
python_command=python_command,
|
||||
rust_command=(binary,),
|
||||
python_environment=python_environment,
|
||||
rust_environment=rust_environment,
|
||||
public_probe_host=probe_host,
|
||||
public_port=port,
|
||||
private_port=private_port,
|
||||
startup_timeout=startup_timeout,
|
||||
rust_drain_timeout=rust_drain_timeout,
|
||||
python_grace_timeout=python_grace_timeout,
|
||||
termination_grace_timeout=termination_grace_timeout,
|
||||
)
|
||||
|
||||
|
||||
def _enable_subreaper() -> SupervisorFailure | None:
|
||||
if not sys.platform.startswith("linux"):
|
||||
return SupervisorFailure("the Rust gateway supervisor requires Linux")
|
||||
prctl: Final = ctypes.CDLL(None, use_errno=True).prctl
|
||||
if prctl(36, 1, 0, 0, 0) != 0:
|
||||
return SupervisorFailure("could not enable Linux child subreaper mode")
|
||||
return None
|
||||
|
||||
|
||||
def _install_signal_handlers(
|
||||
requested: Event,
|
||||
) -> tuple[tuple[signal.Signals, object], ...]:
|
||||
signals: Final = (signal.SIGTERM, signal.SIGINT)
|
||||
previous: Final = tuple((handled, signal.getsignal(handled)) for handled in signals)
|
||||
|
||||
def request_shutdown(_signum: int, _frame: FrameType | None) -> None:
|
||||
requested.set()
|
||||
|
||||
for handled in signals:
|
||||
signal.signal(handled, request_shutdown)
|
||||
return previous
|
||||
|
||||
|
||||
def _restore_signal_handlers(
|
||||
previous: tuple[tuple[signal.Signals, object], ...],
|
||||
) -> None:
|
||||
for handled, handler in previous:
|
||||
signal.signal(handled, cast(signal.Handlers | Callable[[int, FrameType | None], object] | None, handler))
|
||||
|
||||
|
||||
@contextmanager
|
||||
def startup_scope() -> Generator[None, None, None]:
|
||||
with pgbouncer_startup_signals(True):
|
||||
subreaper_failure: Final = _enable_subreaper()
|
||||
if subreaper_failure is not None:
|
||||
raise SystemExit(subreaper_failure.reason)
|
||||
yield
|
||||
|
||||
|
||||
def _tcp_ready(host: str, port: int) -> bool:
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=0.2):
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _http_ready(host: str, port: int) -> bool:
|
||||
connection: Final = http.client.HTTPConnection(host, port, timeout=0.3)
|
||||
try:
|
||||
connection.request("GET", LIVENESS_PATH)
|
||||
response: Final = connection.getresponse()
|
||||
response.read()
|
||||
return response.status == 200
|
||||
except (http.client.HTTPException, OSError):
|
||||
return False
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _group_exists(process_group: int) -> bool:
|
||||
try:
|
||||
os.killpg(process_group, 0)
|
||||
return True
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
|
||||
|
||||
def _signal_group(process_group: int, requested: signal.Signals) -> None:
|
||||
try:
|
||||
os.killpg(process_group, requested)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def _reap_adopted_children(excluded: frozenset[int]) -> None:
|
||||
children_path: Final = Path(f"/proc/self/task/{os.getpid()}/children")
|
||||
try:
|
||||
children: Final = tuple(int(pid) for pid in children_path.read_text().split() if int(pid) not in excluded)
|
||||
except (OSError, ValueError):
|
||||
return
|
||||
for child in children:
|
||||
try:
|
||||
os.waitpid(child, os.WNOHANG)
|
||||
except ChildProcessError:
|
||||
pass
|
||||
|
||||
|
||||
def _adopted_children(excluded: frozenset[int]) -> tuple[int, ...]:
|
||||
children_path: Final = Path(f"/proc/self/task/{os.getpid()}/children")
|
||||
try:
|
||||
return tuple(int(pid) for pid in children_path.read_text().split() if int(pid) not in excluded)
|
||||
except (OSError, ValueError):
|
||||
return ()
|
||||
|
||||
|
||||
def _signal_adopted_child(child: int, requested: signal.Signals) -> None:
|
||||
process_group: Final = os.getpgid(child)
|
||||
if process_group == os.getpgrp():
|
||||
os.kill(child, requested)
|
||||
else:
|
||||
os.killpg(process_group, requested)
|
||||
|
||||
|
||||
def _reap_supervised_children(
|
||||
excluded: frozenset[int],
|
||||
managed_pooler: ManagedPgBouncerRuntime | None,
|
||||
) -> None:
|
||||
if managed_pooler is None:
|
||||
_reap_adopted_children(excluded)
|
||||
return
|
||||
with managed_pooler.owned_pid_guard() as pooler_pid:
|
||||
protected: Final = frozenset((*excluded, *((pooler_pid,) if pooler_pid is not None else ())))
|
||||
_reap_adopted_children(protected)
|
||||
|
||||
|
||||
def _current_adopted_children(
|
||||
excluded: frozenset[int],
|
||||
managed_pooler: ManagedPgBouncerRuntime | None,
|
||||
) -> tuple[tuple[int, ...], frozenset[int]]:
|
||||
if managed_pooler is None:
|
||||
return _adopted_children(excluded), excluded
|
||||
with managed_pooler.owned_pid_guard() as pooler_pid:
|
||||
protected: Final = frozenset((*excluded, *((pooler_pid,) if pooler_pid is not None else ())))
|
||||
return _adopted_children(protected), protected
|
||||
|
||||
|
||||
def _terminate_adopted_children(
|
||||
excluded: frozenset[int],
|
||||
deadline: float | None = None,
|
||||
managed_pooler: ManagedPgBouncerRuntime | None = None,
|
||||
) -> None:
|
||||
term_children: Final = _current_adopted_children(excluded, managed_pooler)[0]
|
||||
for child in term_children:
|
||||
try:
|
||||
_signal_adopted_child(child, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
term_deadline: Final = time.monotonic() + 1 if deadline is None else min(deadline, time.monotonic() + 1)
|
||||
while time.monotonic() < term_deadline and _current_adopted_children(excluded, managed_pooler)[0]:
|
||||
_reap_supervised_children(excluded, managed_pooler)
|
||||
time.sleep(0.05)
|
||||
kill_children: Final = _current_adopted_children(excluded, managed_pooler)[0]
|
||||
for child in kill_children:
|
||||
try:
|
||||
_signal_adopted_child(child, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
final_deadline: Final = time.monotonic() + 1 if deadline is None else min(deadline, time.monotonic() + 1)
|
||||
while time.monotonic() < final_deadline and _current_adopted_children(excluded, managed_pooler)[0]:
|
||||
_reap_supervised_children(excluded, managed_pooler)
|
||||
time.sleep(0.05)
|
||||
_reap_supervised_children(excluded, managed_pooler)
|
||||
|
||||
|
||||
def _wait_for_group(
|
||||
process: ChildProcess,
|
||||
deadline: float,
|
||||
protected_children: frozenset[int],
|
||||
managed_pooler: ManagedPgBouncerRuntime | None,
|
||||
) -> bool:
|
||||
while time.monotonic() < deadline and _group_exists(process.pid):
|
||||
process.poll()
|
||||
_reap_supervised_children(protected_children, managed_pooler)
|
||||
time.sleep(0.05)
|
||||
process.poll()
|
||||
_reap_supervised_children(protected_children, managed_pooler)
|
||||
return not _group_exists(process.pid)
|
||||
|
||||
|
||||
def _stop_process_group(
|
||||
process: ChildProcess,
|
||||
timeout: int,
|
||||
protected_children: frozenset[int],
|
||||
managed_pooler: ManagedPgBouncerRuntime | None,
|
||||
deadline: float,
|
||||
) -> None:
|
||||
_signal_group(process.pid, signal.SIGTERM)
|
||||
stop_deadline: Final = min(deadline, time.monotonic() + timeout)
|
||||
if not _wait_for_group(process, stop_deadline, protected_children, managed_pooler):
|
||||
_signal_group(process.pid, signal.SIGKILL)
|
||||
_wait_for_group(process, min(deadline, time.monotonic() + 1), protected_children, managed_pooler)
|
||||
try:
|
||||
process.wait(timeout=max(0.0, min(1.0, deadline - time.monotonic())))
|
||||
except subprocess.TimeoutExpired:
|
||||
_signal_group(process.pid, signal.SIGKILL)
|
||||
try:
|
||||
process.wait(timeout=max(0.0, deadline - time.monotonic()))
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
_reap_supervised_children(protected_children, managed_pooler)
|
||||
|
||||
|
||||
def _wait_for_python(
|
||||
process: ChildProcess,
|
||||
port: int,
|
||||
deadline: float,
|
||||
requested: Event,
|
||||
managed_pooler: ManagedPgBouncerRuntime | None,
|
||||
) -> bool | SupervisorFailure:
|
||||
protected_children: Final = frozenset((process.pid,))
|
||||
while not requested.is_set() and time.monotonic() < deadline:
|
||||
_reap_supervised_children(protected_children, managed_pooler)
|
||||
if (status := process.poll()) is not None:
|
||||
return SupervisorFailure(f"Python gateway exited during startup with status {status}")
|
||||
if _tcp_ready("127.0.0.1", port):
|
||||
return True
|
||||
time.sleep(0.05)
|
||||
_reap_supervised_children(protected_children, managed_pooler)
|
||||
return False
|
||||
|
||||
|
||||
def _wait_for_rust(
|
||||
python: ChildProcess,
|
||||
rust: ChildProcess,
|
||||
host: str,
|
||||
port: int,
|
||||
deadline: float,
|
||||
requested: Event,
|
||||
managed_pooler: ManagedPgBouncerRuntime | None,
|
||||
) -> bool | SupervisorFailure:
|
||||
protected_children: Final = frozenset((python.pid, rust.pid))
|
||||
while not requested.is_set() and time.monotonic() < deadline:
|
||||
_reap_supervised_children(protected_children, managed_pooler)
|
||||
if (python_status := python.poll()) is not None:
|
||||
return SupervisorFailure(f"Python gateway exited during startup with status {python_status}")
|
||||
if (rust_status := rust.poll()) is not None:
|
||||
return SupervisorFailure(f"Rust gateway exited during startup with status {rust_status}")
|
||||
if _http_ready(host, port):
|
||||
return True
|
||||
time.sleep(0.05)
|
||||
_reap_supervised_children(protected_children, managed_pooler)
|
||||
return False
|
||||
|
||||
|
||||
def _monitor_children(
|
||||
python: ChildProcess,
|
||||
rust: ChildProcess,
|
||||
requested: Event,
|
||||
managed_pooler: ManagedPgBouncerRuntime | None,
|
||||
) -> SupervisorFailure | None:
|
||||
protected_children: Final = frozenset((python.pid, rust.pid))
|
||||
while not requested.wait(0.1):
|
||||
_reap_supervised_children(protected_children, managed_pooler)
|
||||
if (python_status := python.poll()) is not None:
|
||||
return SupervisorFailure(f"Python gateway exited unexpectedly with status {python_status}")
|
||||
if (rust_status := rust.poll()) is not None:
|
||||
return SupervisorFailure(f"Rust gateway exited unexpectedly with status {rust_status}")
|
||||
_reap_supervised_children(protected_children, managed_pooler)
|
||||
return None
|
||||
|
||||
|
||||
def _shutdown(
|
||||
python: ChildProcess | None,
|
||||
rust: ChildProcess | None,
|
||||
config: SupervisorConfig,
|
||||
) -> None:
|
||||
deadline: Final = time.monotonic() + config.termination_grace_timeout
|
||||
protected: Final = frozenset(process.pid for process in (python, rust) if process is not None)
|
||||
if rust is not None:
|
||||
_stop_process_group(
|
||||
rust,
|
||||
config.rust_drain_timeout,
|
||||
protected,
|
||||
config.managed_pooler,
|
||||
deadline,
|
||||
)
|
||||
if python is not None:
|
||||
_stop_process_group(
|
||||
python,
|
||||
config.python_grace_timeout,
|
||||
protected,
|
||||
config.managed_pooler,
|
||||
deadline,
|
||||
)
|
||||
_terminate_adopted_children(protected, deadline, config.managed_pooler)
|
||||
if config.managed_pooler is not None:
|
||||
config.managed_pooler.stop(deadline)
|
||||
_terminate_adopted_children(frozenset(), deadline)
|
||||
|
||||
|
||||
def _supervise_with_python(
|
||||
config: SupervisorConfig,
|
||||
process_runner: ProcessRunner,
|
||||
requested: Event,
|
||||
python: ChildProcess,
|
||||
) -> SupervisorFailure | None:
|
||||
with ExitStack() as python_cleanup:
|
||||
python_cleanup.callback(_shutdown, python, None, config)
|
||||
startup_deadline: Final = time.monotonic() + config.startup_timeout
|
||||
python_ready: Final = _wait_for_python(
|
||||
python,
|
||||
config.private_port,
|
||||
startup_deadline,
|
||||
requested,
|
||||
config.managed_pooler,
|
||||
)
|
||||
if isinstance(python_ready, SupervisorFailure):
|
||||
return python_ready
|
||||
if not python_ready:
|
||||
return None if requested.is_set() else SupervisorFailure("Python gateway startup timed out")
|
||||
if requested.is_set():
|
||||
return None
|
||||
try:
|
||||
rust: Final = process_runner.start(config.rust_command, config.rust_environment)
|
||||
except OSError as error:
|
||||
return SupervisorFailure(f"could not start Rust gateway: {error}")
|
||||
with ExitStack() as gateway_cleanup:
|
||||
gateway_cleanup.callback(_shutdown, python, rust, config)
|
||||
python_cleanup.pop_all()
|
||||
rust_ready: Final = _wait_for_rust(
|
||||
python,
|
||||
rust,
|
||||
config.public_probe_host,
|
||||
config.public_port,
|
||||
startup_deadline,
|
||||
requested,
|
||||
config.managed_pooler,
|
||||
)
|
||||
if isinstance(rust_ready, SupervisorFailure):
|
||||
return rust_ready
|
||||
if not rust_ready:
|
||||
return None if requested.is_set() else SupervisorFailure("Rust gateway readiness timed out")
|
||||
return _monitor_children(python, rust, requested, config.managed_pooler)
|
||||
|
||||
|
||||
def _supervise(
|
||||
config: SupervisorConfig,
|
||||
process_runner: ProcessRunner,
|
||||
requested: Event,
|
||||
) -> SupervisorFailure | None:
|
||||
try:
|
||||
python: Final = process_runner.start(config.python_command, config.python_environment)
|
||||
except OSError as error:
|
||||
_shutdown(None, None, config)
|
||||
return SupervisorFailure(f"could not start Python gateway: {error}")
|
||||
return _supervise_with_python(config, process_runner, requested, python)
|
||||
|
||||
|
||||
def supervise(config: SupervisorConfig, runner: ProcessRunner | None = None) -> SupervisorFailure | None:
|
||||
subreaper_failure: Final = _enable_subreaper()
|
||||
if subreaper_failure is not None:
|
||||
if config.managed_pooler is not None:
|
||||
config.managed_pooler.stop(time.monotonic() + config.termination_grace_timeout)
|
||||
return subreaper_failure
|
||||
process_runner: Final = runner if runner is not None else SubprocessRunner()
|
||||
requested: Final = Event()
|
||||
previous_handlers: Final = _install_signal_handlers(requested)
|
||||
try:
|
||||
return _supervise(config, process_runner, requested)
|
||||
finally:
|
||||
_restore_signal_handlers(previous_handlers)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit("run the Rust gateway through python -m gateway.launch")
|
||||
|
|
@ -599,6 +599,14 @@ ImplementationSpecific
|
|||
|
||||
{{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}}
|
||||
|
||||
{{- define "litellm.gateway.rustGatewayEnabled" -}}
|
||||
{{- range .Values.gateway.extraEnv -}}
|
||||
{{- if and (eq .name "LITELLM_RUST_GATEWAY") (eq (toString .value) "true") -}}
|
||||
true
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Directory of the collector's unix socket, shared by the gateway and
|
||||
collector containers through an emptyDir. Empty when the sidecar is off
|
||||
|
|
|
|||
|
|
@ -1,4 +1,16 @@
|
|||
{{- if .Values.gateway.enabled }}
|
||||
{{- $rustGatewayEnabled := eq (include "litellm.gateway.rustGatewayEnabled" . | trim) "true" }}
|
||||
{{- $pythonPort := 4002 }}
|
||||
{{- $configuredGrace := .Values.gateway.terminationGracePeriodSeconds }}
|
||||
{{- $gracePeriod := 30 }}
|
||||
{{- if not (or (kindIs "invalid" $configuredGrace) (eq (printf "%v" $configuredGrace) "")) }}
|
||||
{{- $gracePeriod = int $configuredGrace }}
|
||||
{{- end }}
|
||||
{{- range .Values.gateway.extraEnv }}
|
||||
{{- if and (eq .name "LITELLM_RUST_GATEWAY_PYTHON_PORT") .value }}
|
||||
{{- $pythonPort = int .value }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
|
|
@ -79,9 +91,16 @@ spec:
|
|||
{{- if eq (int .Values.gateway.metricsServer.port) 4000 }}
|
||||
{{- fail "gateway.metricsServer.port must differ from the gateway port 4000" }}
|
||||
{{- end }}
|
||||
{{- if and $rustGatewayEnabled (eq (int .Values.gateway.metricsServer.port) $pythonPort) }}
|
||||
{{- fail "gateway.metricsServer.port must differ from the private Python gateway port" }}
|
||||
{{- end }}
|
||||
- name: PROMETHEUS_MULTIPROC_DIR
|
||||
value: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
|
||||
- name: LITELLM_GATEWAY_METRICS_PORT
|
||||
value: {{ .Values.gateway.metricsServer.port | quote }}
|
||||
{{- end }}
|
||||
- name: LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS
|
||||
value: {{ $gracePeriod | quote }}
|
||||
{{- if .Values.gateway.collector.enabled }}
|
||||
{{- include "litellm.gateway.collectorEnv" . | nindent 12 }}
|
||||
{{- end }}
|
||||
|
|
@ -245,8 +264,5 @@ spec:
|
|||
topologySpreadConstraints:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- $gracePeriod := .Values.gateway.terminationGracePeriodSeconds }}
|
||||
{{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }}
|
||||
terminationGracePeriodSeconds: {{ $gracePeriod }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
|
|||
141
helm/litellm/tests/rust_gateway_tests.yaml
Normal file
141
helm/litellm/tests/rust_gateway_tests.yaml
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
suite: test opt-in Rust gateway deployment
|
||||
templates:
|
||||
- gateway/deployment.yaml
|
||||
- gateway/service.yaml
|
||||
- gateway/configmap.yaml
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
tests:
|
||||
- it: leaves HTTP probes and service ports unchanged when disabled
|
||||
set:
|
||||
gateway.metricsServer.enabled: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].livenessProbe.httpGet.path
|
||||
value: /health/liveliness
|
||||
template: gateway/deployment.yaml
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_GATEWAY_METRICS_PORT
|
||||
value: "4001"
|
||||
any: true
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].ports[0].containerPort
|
||||
value: 4000
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.ports[0].port
|
||||
value: 4000
|
||||
template: gateway/service.yaml
|
||||
|
||||
- it: passes enabled gateway settings and keeps the private listener off Services
|
||||
set:
|
||||
gateway.extraEnv:
|
||||
- name: LITELLM_RUST_GATEWAY
|
||||
value: "true"
|
||||
gateway.envConfigMaps:
|
||||
- gateway-runtime
|
||||
gateway.metricsServer.enabled: true
|
||||
gateway.metricsServer.port: 4001
|
||||
gateway.collector.enabled: true
|
||||
gateway.collector.address: tcp://127.0.0.1:4102
|
||||
gateway.terminationGracePeriodSeconds: 30
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_RUST_GATEWAY
|
||||
value: "true"
|
||||
template: gateway/deployment.yaml
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_GATEWAY_METRICS_PORT
|
||||
value: "4001"
|
||||
template: gateway/deployment.yaml
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_COLLECTOR_ADDRESS
|
||||
value: tcp://127.0.0.1:4102
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].envFrom[0].configMapRef.name
|
||||
value: gateway-runtime
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].livenessProbe.httpGet.path
|
||||
value: /health/liveliness
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].readinessProbe.httpGet.path
|
||||
value: /health/readiness
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].ports[0].containerPort
|
||||
value: 4000
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.template.spec.terminationGracePeriodSeconds
|
||||
value: 30
|
||||
template: gateway/deployment.yaml
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS
|
||||
value: "30"
|
||||
template: gateway/deployment.yaml
|
||||
- equal:
|
||||
path: spec.ports[0].port
|
||||
value: 4000
|
||||
template: gateway/service.yaml
|
||||
|
||||
- it: rejects a metrics port that collides with the default private Python listener
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
gateway.extraEnv:
|
||||
- name: LITELLM_RUST_GATEWAY
|
||||
value: "true"
|
||||
gateway.metricsServer.enabled: true
|
||||
gateway.metricsServer.port: 4002
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorPattern: gateway.metricsServer.port must differ from the private Python gateway port
|
||||
|
||||
- it: exports metrics collision metadata when the flag comes from envFrom
|
||||
set:
|
||||
gateway.envConfigMaps:
|
||||
- gateway-runtime
|
||||
gateway.metricsServer.enabled: true
|
||||
gateway.metricsServer.port: 4001
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].envFrom[0].configMapRef.name
|
||||
value: gateway-runtime
|
||||
template: gateway/deployment.yaml
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_GATEWAY_METRICS_PORT
|
||||
value: "4001"
|
||||
template: gateway/deployment.yaml
|
||||
|
||||
- it: preserves an explicit zero termination grace for enabled-mode rejection
|
||||
set:
|
||||
gateway.extraEnv:
|
||||
- name: LITELLM_RUST_GATEWAY
|
||||
value: "true"
|
||||
gateway.terminationGracePeriodSeconds: 0
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.terminationGracePeriodSeconds
|
||||
value: 0
|
||||
template: gateway/deployment.yaml
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS
|
||||
value: "0"
|
||||
template: gateway/deployment.yaml
|
||||
1
litellm-rust/Cargo.lock
generated
1
litellm-rust/Cargo.lock
generated
|
|
@ -3770,6 +3770,7 @@ dependencies = [
|
|||
"http-body-util",
|
||||
"hyper 1.10.1",
|
||||
"hyper-util",
|
||||
"ipnet",
|
||||
"litellm-auth-types",
|
||||
"litellm-config",
|
||||
"litellm-core-utils",
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ envy = "0.4.2"
|
|||
http-body-util = "0.1"
|
||||
hyper = { version = "1", features = ["http1"] }
|
||||
hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] }
|
||||
ipnet = "2.12"
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-inference.workspace = true
|
||||
litellm-gateway-inference.workspace = true
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ pub struct InvalidUpstream {
|
|||
pub struct Forward {
|
||||
client: Arc<Client<HttpConnector, Body>>,
|
||||
authority: Authority,
|
||||
trusted_proxies: Option<crate::proxy_headers::TrustedProxies>,
|
||||
}
|
||||
|
||||
impl Forward {
|
||||
|
|
@ -72,9 +73,15 @@ impl Forward {
|
|||
Ok(Self {
|
||||
client: Arc::new(Client::builder(TokioExecutor::new()).build(HttpConnector::new())),
|
||||
authority,
|
||||
trusted_proxies: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn with_trusted_proxies(mut self, proxies: &str) -> Result<Self, crate::Error> {
|
||||
self.trusted_proxies = Some(crate::proxy_headers::TrustedProxies::parse(proxies)?);
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
pub(super) async fn send(&self, mut request: Request, decline: Decline) -> Response {
|
||||
let websocket = if request.method() == Method::GET
|
||||
&& request.version() == Version::HTTP_11
|
||||
|
|
@ -126,8 +133,14 @@ impl Forward {
|
|||
if websocket.is_some() {
|
||||
websocket_headers(&mut parts.headers);
|
||||
}
|
||||
if let Some(ConnectInfo(client)) = parts.extensions.get::<ConnectInfo<SocketAddr>>() {
|
||||
append_forwarded_for(&mut parts.headers, *client);
|
||||
let peer = parts
|
||||
.extensions
|
||||
.get::<ConnectInfo<SocketAddr>>()
|
||||
.map(|info| info.0);
|
||||
if let Some(policy) = &self.trusted_proxies {
|
||||
policy.apply(&mut parts.headers, peer);
|
||||
} else if let Some(client) = peer {
|
||||
append_forwarded_for(&mut parts.headers, client);
|
||||
}
|
||||
parts
|
||||
.headers
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error(
|
||||
"LITELLM_RUST_GATEWAY_TRUSTED_PROXIES must contain comma-separated IP addresses or CIDRs, not wildcard trust"
|
||||
)]
|
||||
TrustedProxies,
|
||||
#[error(
|
||||
"LITELLM_RUST_GATEWAY=true requires LITELLM_PYTHON_UPSTREAM to be a valid http:// origin"
|
||||
)]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
mod auth;
|
||||
mod dispatch;
|
||||
mod error;
|
||||
mod proxy_headers;
|
||||
mod secrets;
|
||||
mod upgrades;
|
||||
pub use dispatch::{Fallback, Forward, InvalidUpstream};
|
||||
|
|
|
|||
172
litellm-rust/crates/gateway/src/proxy_headers.rs
Normal file
172
litellm-rust/crates/gateway/src/proxy_headers.rs
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
use std::net::{IpAddr, SocketAddr};
|
||||
|
||||
use axum::http::{HeaderMap, HeaderValue};
|
||||
use ipnet::IpNet;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct TrustedProxies(Vec<IpNet>);
|
||||
|
||||
impl TrustedProxies {
|
||||
pub(crate) fn parse(value: &str) -> Result<Self, crate::Error> {
|
||||
if value.trim().is_empty() {
|
||||
return Ok(Self(Vec::new()));
|
||||
}
|
||||
value
|
||||
.split(',')
|
||||
.map(|value| {
|
||||
let value = value.trim();
|
||||
let network = value
|
||||
.parse::<IpNet>()
|
||||
.or_else(|_| value.parse::<IpAddr>().map(IpNet::from))
|
||||
.map_err(|_| crate::Error::TrustedProxies)?;
|
||||
if network.prefix_len() == 0 {
|
||||
return Err(crate::Error::TrustedProxies);
|
||||
}
|
||||
Ok(network)
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map(Self)
|
||||
}
|
||||
|
||||
fn contains(&self, address: IpAddr) -> bool {
|
||||
self.0.iter().any(|network| network.contains(&address))
|
||||
}
|
||||
|
||||
pub(crate) fn apply(&self, headers: &mut HeaderMap, peer: Option<SocketAddr>) {
|
||||
let trusted = peer.is_some_and(|peer| self.contains(peer.ip()));
|
||||
let chain = trusted
|
||||
.then(|| {
|
||||
headers
|
||||
.get_all("x-forwarded-for")
|
||||
.iter()
|
||||
.map(|value| value.to_str().ok())
|
||||
.collect::<Option<Vec<_>>>()
|
||||
.and_then(|values| {
|
||||
values
|
||||
.iter()
|
||||
.flat_map(|value| value.split(','))
|
||||
.map(|value| value.trim().parse::<IpAddr>().ok())
|
||||
.collect::<Option<Vec<_>>>()
|
||||
})
|
||||
})
|
||||
.flatten();
|
||||
let client = chain
|
||||
.as_ref()
|
||||
.and_then(|chain| {
|
||||
chain
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|address| !self.contains(**address))
|
||||
.or_else(|| chain.first())
|
||||
.copied()
|
||||
})
|
||||
.or_else(|| peer.map(|peer| peer.ip()));
|
||||
let scheme = if trusted && headers.get_all("x-forwarded-proto").iter().count() == 1 {
|
||||
match headers
|
||||
.get("x-forwarded-proto")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
{
|
||||
Some("https") => "https",
|
||||
_ => "http",
|
||||
}
|
||||
} else {
|
||||
"http"
|
||||
};
|
||||
for name in [
|
||||
"forwarded",
|
||||
"x-forwarded-for",
|
||||
"x-forwarded-proto",
|
||||
"x-forwarded-host",
|
||||
"x-forwarded-port",
|
||||
"x-forwarded-prefix",
|
||||
"x-real-ip",
|
||||
] {
|
||||
headers.remove(name);
|
||||
}
|
||||
if let Some(client) = client
|
||||
&& let Ok(value) = HeaderValue::from_str(&client.to_string())
|
||||
{
|
||||
headers.insert("x-forwarded-for", value);
|
||||
}
|
||||
headers.insert("x-forwarded-proto", HeaderValue::from_static(scheme));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rstest::rstest;
|
||||
|
||||
#[rstest]
|
||||
#[case::direct("", "127.0.0.2", "198.51.100.7", "https", "127.0.0.2", "http")]
|
||||
#[case::trusted(
|
||||
"127.0.0.2",
|
||||
"127.0.0.2",
|
||||
"198.51.100.7",
|
||||
"https",
|
||||
"198.51.100.7",
|
||||
"https"
|
||||
)]
|
||||
#[case::chain(
|
||||
"10.0.0.0/8",
|
||||
"10.0.0.1",
|
||||
"192.0.2.1, 198.51.100.7, 10.0.0.2",
|
||||
"https",
|
||||
"198.51.100.7",
|
||||
"https"
|
||||
)]
|
||||
#[case::invalid_chain(
|
||||
"10.0.0.0/8",
|
||||
"10.0.0.1",
|
||||
"198.51.100.7, unknown",
|
||||
"https",
|
||||
"10.0.0.1",
|
||||
"https"
|
||||
)]
|
||||
#[case::invalid_scheme(
|
||||
"127.0.0.2",
|
||||
"127.0.0.2",
|
||||
"198.51.100.7",
|
||||
"https,http",
|
||||
"198.51.100.7",
|
||||
"http"
|
||||
)]
|
||||
#[case::ipv6("fd00::/8", "fd00::1", "2001:db8::7", "https", "2001:db8::7", "https")]
|
||||
fn normalizes_proxy_identity(
|
||||
#[case] trusted: &str,
|
||||
#[case] peer: &str,
|
||||
#[case] chain: &str,
|
||||
#[case] proto: &str,
|
||||
#[case] client: &str,
|
||||
#[case] scheme: &str,
|
||||
) {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("host", HeaderValue::from_static("gateway.example"));
|
||||
headers.insert("origin", HeaderValue::from_static("https://app.example"));
|
||||
headers.insert("x-forwarded-for", HeaderValue::from_str(chain).unwrap());
|
||||
headers.insert("x-forwarded-proto", HeaderValue::from_str(proto).unwrap());
|
||||
headers.insert("forwarded", HeaderValue::from_static("proto=https"));
|
||||
TrustedProxies::parse(trusted).unwrap().apply(
|
||||
&mut headers,
|
||||
Some(SocketAddr::new(peer.parse().unwrap(), 1234)),
|
||||
);
|
||||
assert_eq!(headers["x-forwarded-for"], client);
|
||||
assert_eq!(headers["x-forwarded-proto"], scheme);
|
||||
assert_eq!(headers["host"], "gateway.example");
|
||||
assert_eq!(headers["origin"], "https://app.example");
|
||||
assert!(!headers.contains_key("forwarded"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("*")]
|
||||
#[case("0.0.0.0/0")]
|
||||
#[case("::/0")]
|
||||
#[case("127.0.0.1,")]
|
||||
#[case("proxy.example")]
|
||||
fn invalid_policy_fails_closed(#[case] value: &str) {
|
||||
assert!(matches!(
|
||||
TrustedProxies::parse(value),
|
||||
Err(crate::Error::TrustedProxies)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
|
@ -12,6 +12,7 @@ pub(super) struct Settings {
|
|||
pub litellm_config: String,
|
||||
pub litellm_python_upstream: Option<String>,
|
||||
pub litellm_rust_gateway: bool,
|
||||
pub litellm_rust_gateway_trusted_proxies: Option<String>,
|
||||
rust_log: Option<String>,
|
||||
}
|
||||
|
||||
|
|
@ -23,6 +24,7 @@ impl Default for Settings {
|
|||
litellm_config: "config.yaml".into(),
|
||||
litellm_python_upstream: None,
|
||||
litellm_rust_gateway: false,
|
||||
litellm_rust_gateway_trusted_proxies: None,
|
||||
rust_log: None,
|
||||
}
|
||||
}
|
||||
|
|
@ -51,7 +53,11 @@ impl Settings {
|
|||
{
|
||||
return Err(invalid());
|
||||
}
|
||||
litellm_gateway::Forward::new(upstream).map_err(|_| invalid())
|
||||
let forward = litellm_gateway::Forward::new(upstream).map_err(|_| invalid())?;
|
||||
match self.litellm_rust_gateway_trusted_proxies.as_deref() {
|
||||
Some(proxies) => forward.with_trusted_proxies(proxies),
|
||||
None => Ok(forward),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn log_level(&self) -> Level {
|
||||
|
|
@ -192,6 +198,43 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::absent(None)]
|
||||
#[case::empty(Some(""))]
|
||||
#[case::explicit_trust(Some("127.0.0.2"))]
|
||||
fn trusted_proxy_configuration_keeps_absent_and_empty_distinct(#[case] value: Option<&str>) {
|
||||
let settings: Settings = match value {
|
||||
Some(trusted) => parse(&[
|
||||
("LITELLM_PYTHON_UPSTREAM", "http://127.0.0.1:4002"),
|
||||
("LITELLM_RUST_GATEWAY_TRUSTED_PROXIES", trusted),
|
||||
])
|
||||
.unwrap(),
|
||||
None => parse(&[("LITELLM_PYTHON_UPSTREAM", "http://127.0.0.1:4002")]).unwrap(),
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
settings.litellm_rust_gateway_trusted_proxies.as_deref(),
|
||||
value
|
||||
);
|
||||
assert!(settings.forwarder().is_ok());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::wildcard("*")]
|
||||
#[case::default_route("0.0.0.0/0")]
|
||||
fn invalid_trusted_proxy_configuration_fails_forwarder(#[case] value: &str) {
|
||||
let settings: Settings = parse(&[
|
||||
("LITELLM_PYTHON_UPSTREAM", "http://127.0.0.1:4002"),
|
||||
("LITELLM_RUST_GATEWAY_TRUSTED_PROXIES", value),
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
settings.forwarder(),
|
||||
Err(litellm_gateway::Error::TrustedProxies)
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn environment_overrides_startup_settings() {
|
||||
let settings: Settings = parse(&[
|
||||
|
|
|
|||
|
|
@ -0,0 +1,647 @@
|
|||
import http.client
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
import time
|
||||
from collections.abc import Iterator, Mapping
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from gateway.launch import (
|
||||
_pooled_child_environments, # pyright: ignore[reportPrivateUsage] # exercise the child env policy
|
||||
)
|
||||
from gateway.supervisor import ChildProcess, SupervisorConfig, SupervisorFailure, supervise
|
||||
from litellm.proxy.db.pgbouncer import (
|
||||
ManagedPgBouncerRuntime,
|
||||
PgBouncerSettings,
|
||||
start_managed_pgbouncer,
|
||||
)
|
||||
from tests.integration._support.ports import allocate_port
|
||||
|
||||
ROOT: Final = Path(__file__).resolve().parents[3]
|
||||
SERVER_SOURCE: Final = """\
|
||||
import http.server
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
role, port, events, mode, child_pid_file = sys.argv[1:]
|
||||
if role == "python" and mode == "startup-failure":
|
||||
raise SystemExit(17)
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(200)
|
||||
if self.path == "/events":
|
||||
self.send_header("Content-Type", "text/event-stream")
|
||||
self.end_headers()
|
||||
self.wfile.write(b"data: connected\\n\\n" if self.path == "/events" else b"ok")
|
||||
self.wfile.flush()
|
||||
if self.path == "/events":
|
||||
threading.Event().wait()
|
||||
if role == "rust" and mode == "exit":
|
||||
threading.Timer(0.1, server.shutdown).start()
|
||||
|
||||
def log_message(self, *_args):
|
||||
pass
|
||||
|
||||
server = http.server.ThreadingHTTPServer(("127.0.0.1", int(port)), Handler)
|
||||
server.daemon_threads = True
|
||||
if mode == "orphan":
|
||||
child = subprocess.Popen(
|
||||
(
|
||||
sys.executable,
|
||||
"-P",
|
||||
"-c",
|
||||
"import signal,time; signal.signal(signal.SIGTERM, signal.SIG_IGN); time.sleep(60)",
|
||||
),
|
||||
start_new_session=True,
|
||||
)
|
||||
Path(child_pid_file).write_text(str(child.pid))
|
||||
elif mode == "orphan-exit":
|
||||
middle_command = "import subprocess,sys,time; child=subprocess.Popen((sys.executable,'-P','-c','import time; time.sleep(0.35)'), start_new_session=True); open(sys.argv[1],'w').write(str(child.pid)); time.sleep(0.2)"
|
||||
subprocess.Popen((sys.executable, "-P", "-c", middle_command, child_pid_file))
|
||||
|
||||
def stop(_signum, _frame):
|
||||
with Path(events).open("a") as output:
|
||||
output.write(role + "\\n")
|
||||
if mode not in ("ignore", "orphan"):
|
||||
threading.Thread(target=server.shutdown).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
if role == "python" and mode == "exit":
|
||||
threading.Timer(0.5, server.shutdown).start()
|
||||
server.serve_forever()
|
||||
"""
|
||||
LAUNCH_MAIN_SOURCE: Final = """\
|
||||
import os
|
||||
from pathlib import Path
|
||||
from gateway.launch import main
|
||||
|
||||
def serve(_argv):
|
||||
Path(os.environ["TEST_SERVE_MARKER"]).write_text(
|
||||
os.environ["LITELLM_PGBOUNCER_ENABLED"] + "|" + os.environ.get("DATABASE_URL", "")
|
||||
)
|
||||
|
||||
main((), serve)
|
||||
"""
|
||||
SUPERVISOR_SOURCE: Final = """\
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import replace
|
||||
from types import MappingProxyType
|
||||
from pathlib import Path
|
||||
from gateway.supervisor import SupervisorConfig, supervise
|
||||
|
||||
server, private_port, public_port, events, python_mode, rust_mode, child_pid_file, rust_timeout = sys.argv[1:]
|
||||
environment = MappingProxyType(dict(os.environ))
|
||||
python_command = (sys.executable, server, "python", private_port, events, python_mode, child_pid_file)
|
||||
rust_command = (
|
||||
("/missing/rust-gateway",)
|
||||
if rust_mode == "missing"
|
||||
else (sys.executable, server, "rust", public_port, events, rust_mode, child_pid_file)
|
||||
)
|
||||
config = SupervisorConfig(
|
||||
python_command=python_command,
|
||||
rust_command=rust_command,
|
||||
python_environment=environment,
|
||||
rust_environment=environment,
|
||||
public_probe_host="127.0.0.1",
|
||||
public_port=int(public_port),
|
||||
private_port=int(private_port),
|
||||
startup_timeout=5,
|
||||
rust_drain_timeout=int(rust_timeout),
|
||||
python_grace_timeout=2,
|
||||
)
|
||||
if "TEST_MANAGED_PGBOUNCER_BINARY" in os.environ:
|
||||
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, start_managed_pgbouncer
|
||||
|
||||
managed_pooler = start_managed_pgbouncer(
|
||||
PgBouncerSettings(
|
||||
enabled=True,
|
||||
port=int(os.environ["TEST_MANAGED_PGBOUNCER_PORT"]),
|
||||
binary=os.environ["TEST_MANAGED_PGBOUNCER_BINARY"],
|
||||
),
|
||||
"postgresql://litellm:password@127.0.0.1/litellm",
|
||||
)
|
||||
if isinstance(managed_pooler, PgBouncerError):
|
||||
print(managed_pooler.reason, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
pooler_pid_file = Path(os.environ["TEST_MANAGED_PGBOUNCER_PID_FILE"])
|
||||
|
||||
def write_pooler_pid():
|
||||
temporary = pooler_pid_file.with_suffix(".tmp")
|
||||
temporary.write_text(str(managed_pooler.pooler.pid))
|
||||
temporary.replace(pooler_pid_file)
|
||||
|
||||
def record_pooler_pid():
|
||||
while True:
|
||||
write_pooler_pid()
|
||||
time.sleep(0.05)
|
||||
|
||||
write_pooler_pid()
|
||||
threading.Thread(target=record_pooler_pid, daemon=True).start()
|
||||
config = replace(config, managed_pooler=managed_pooler)
|
||||
failure = supervise(config)
|
||||
if failure is not None:
|
||||
print(failure.reason, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _FailAfterPythonStartRunner:
|
||||
python_command: tuple[str, ...]
|
||||
python_pid_file: Path
|
||||
cleanup: ExitStack
|
||||
|
||||
def start(self, command: tuple[str, ...], environment: Mapping[str, str]) -> ChildProcess:
|
||||
if command != self.python_command:
|
||||
raise OSError("injected Rust gateway start failure")
|
||||
process: Final = subprocess.Popen(
|
||||
command,
|
||||
env=environment,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
)
|
||||
self.cleanup.callback(_terminate_if_running, process)
|
||||
self.python_pid_file.write_text(str(process.pid))
|
||||
return process
|
||||
|
||||
|
||||
def _start_supervisor(
|
||||
tmp_path: Path,
|
||||
private_port: int,
|
||||
public_port: int,
|
||||
events: Path,
|
||||
python_mode: str = "normal",
|
||||
rust_mode: str = "normal",
|
||||
rust_timeout: int = 2,
|
||||
managed_pooler: tuple[Path, int, Path] | None = None,
|
||||
) -> subprocess.Popen[bytes]:
|
||||
server: Final = tmp_path / "server.py"
|
||||
launcher: Final = tmp_path / "supervisor.py"
|
||||
child_pid_file: Final = tmp_path / "orphan.pid"
|
||||
server.write_text(SERVER_SOURCE)
|
||||
launcher.write_text(SUPERVISOR_SOURCE)
|
||||
environment: Final = {
|
||||
**os.environ,
|
||||
**(
|
||||
{}
|
||||
if managed_pooler is None
|
||||
else {
|
||||
"TEST_MANAGED_PGBOUNCER_BINARY": str(managed_pooler[0]),
|
||||
"TEST_MANAGED_PGBOUNCER_PORT": str(managed_pooler[1]),
|
||||
"TEST_MANAGED_PGBOUNCER_PID_FILE": str(managed_pooler[2]),
|
||||
}
|
||||
),
|
||||
}
|
||||
return subprocess.Popen(
|
||||
(
|
||||
sys.executable,
|
||||
"-P",
|
||||
str(launcher),
|
||||
str(server),
|
||||
str(private_port),
|
||||
str(public_port),
|
||||
str(events),
|
||||
python_mode,
|
||||
rust_mode,
|
||||
str(child_pid_file),
|
||||
str(rust_timeout),
|
||||
),
|
||||
cwd=ROOT,
|
||||
env=environment,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
)
|
||||
|
||||
|
||||
def _fake_pgbouncer(tmp_path: Path) -> Path:
|
||||
binary: Final = tmp_path / "fake-pgbouncer"
|
||||
binary.write_text(
|
||||
textwrap.dedent(
|
||||
f"""\
|
||||
#!{sys.executable}
|
||||
import configparser, select, socket, sys
|
||||
if sys.argv[1:] == ["--version"]:
|
||||
print("PgBouncer 1.25.2")
|
||||
sys.exit(0)
|
||||
configuration = configparser.ConfigParser()
|
||||
configuration.read(sys.argv[1])
|
||||
port = configuration.getint("pgbouncer", "listen_port")
|
||||
tcp = socket.socket()
|
||||
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
tcp.bind(("127.0.0.1", port))
|
||||
tcp.listen()
|
||||
unix = socket.socket(socket.AF_UNIX)
|
||||
unix.bind(configuration.get("pgbouncer", "unix_socket_dir") + f"/.s.PGSQL.{{port}}")
|
||||
unix.listen()
|
||||
while True:
|
||||
for listener in select.select((tcp, unix), [], [])[0]:
|
||||
listener.accept()[0].close()
|
||||
"""
|
||||
)
|
||||
)
|
||||
binary.chmod(0o700)
|
||||
return binary
|
||||
|
||||
|
||||
def _slow_starting_pgbouncer(tmp_path: Path, pid_file: Path, config_file: Path) -> Path:
|
||||
binary: Final = tmp_path / "slow-pgbouncer"
|
||||
binary.write_text(
|
||||
textwrap.dedent(
|
||||
f"""\
|
||||
#!{sys.executable}
|
||||
import os, signal, sys, time
|
||||
from pathlib import Path
|
||||
if sys.argv[1:] == ["--version"]:
|
||||
print("PgBouncer 1.26.0")
|
||||
sys.exit(0)
|
||||
Path({str(config_file)!r}).write_text(sys.argv[1])
|
||||
Path({str(pid_file)!r}).write_text(str(os.getpid()))
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
time.sleep(60)
|
||||
"""
|
||||
)
|
||||
)
|
||||
binary.chmod(0o700)
|
||||
return binary
|
||||
|
||||
|
||||
def test_managed_pgbouncer_stop_is_bounded_and_guards_its_owned_pid(tmp_path: Path) -> None:
|
||||
port: Final = allocate_port()
|
||||
runtime: Final = start_managed_pgbouncer(
|
||||
PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pgbouncer(tmp_path))),
|
||||
"postgresql://litellm:password@127.0.0.1/litellm",
|
||||
)
|
||||
assert isinstance(runtime, ManagedPgBouncerRuntime), runtime
|
||||
pid: Final = runtime.pooler.pid
|
||||
deadline: Final = time.monotonic() + 2
|
||||
try:
|
||||
with runtime.owned_pid_guard() as guarded_pid:
|
||||
assert guarded_pid == pid
|
||||
finally:
|
||||
runtime.stop(deadline)
|
||||
assert pid is not None
|
||||
assert not Path(f"/proc/{pid}").exists()
|
||||
assert not runtime.runtime_dir.exists()
|
||||
|
||||
|
||||
def test_launch_cleans_up_managed_pooler_when_startup_is_interrupted(tmp_path: Path) -> None:
|
||||
pid_file: Final = tmp_path / "pooler.pid"
|
||||
config_file: Final = tmp_path / "pooler-config"
|
||||
process: Final = subprocess.Popen( # test-quality-ok: child must import the working-tree gateway.launch module
|
||||
(
|
||||
sys.executable,
|
||||
"-m",
|
||||
"gateway.launch",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
str(allocate_port()),
|
||||
),
|
||||
cwd=ROOT,
|
||||
env={
|
||||
**os.environ,
|
||||
"DATABASE_URL": "postgresql://litellm:password@127.0.0.1/litellm",
|
||||
"LITELLM_PGBOUNCER_BINARY": str(_slow_starting_pgbouncer(tmp_path, pid_file, config_file)),
|
||||
"LITELLM_PGBOUNCER_ENABLED": "false",
|
||||
"LITELLM_PGBOUNCER_PORT": str(allocate_port()),
|
||||
"LITELLM_RUST_GATEWAY": "true",
|
||||
"LITELLM_RUST_GATEWAY_BINARY": "/bin/true",
|
||||
},
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
)
|
||||
try:
|
||||
pooler_pid: Final = _wait_for_pid(pid_file)
|
||||
os.kill(process.pid, signal.SIGTERM) # test-quality-ok: Interrupt startup to exercise managed cleanup
|
||||
assert process.wait(timeout=20) == 143
|
||||
assert not Path(f"/proc/{pooler_pid}").exists()
|
||||
runtime_dir: Final = Path(config_file.read_text()).parent
|
||||
assert not runtime_dir.exists()
|
||||
finally:
|
||||
_terminate_if_running(process)
|
||||
|
||||
|
||||
def _wait_for_pid(pid_file: Path) -> int:
|
||||
deadline: Final = time.monotonic() + 5
|
||||
pooler_pid: Final = next(
|
||||
(pid for pid in _recorded_pooler_pids(pid_file, deadline) if Path(f"/proc/{pid}").exists()),
|
||||
None,
|
||||
)
|
||||
if pooler_pid is None:
|
||||
pytest.fail("managed PgBouncer did not record a live process")
|
||||
return pooler_pid
|
||||
|
||||
|
||||
def _recorded_pooler_pids(pid_file: Path, deadline: float) -> Iterator[int]:
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
yield int(pid_file.read_text())
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
def _wait_for_replacement_pooler(pid_file: Path, original_pid: int) -> int | None:
|
||||
deadline: Final = time.monotonic() + 5
|
||||
return next(
|
||||
(
|
||||
pid
|
||||
for pid in _recorded_pooler_pids(pid_file, deadline)
|
||||
if pid != original_pid and Path(f"/proc/{pid}").exists()
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def test_database_free_python_child_runs_launch_with_pooling_disabled(tmp_path: Path) -> None:
|
||||
serve_marker: Final = tmp_path / "serve"
|
||||
inherited_environment: Final = {
|
||||
name: value
|
||||
for name, value in os.environ.items()
|
||||
if not name.startswith("DATABASE_") and name not in ("DIRECT_URL", "LITELLM_PGBOUNCER_POOLED_DATABASE_URL")
|
||||
}
|
||||
rust_environment: Final = {
|
||||
**inherited_environment,
|
||||
"LITELLM_RUST_GATEWAY": "false",
|
||||
"LITELLM_PGBOUNCER_ENABLED": "true",
|
||||
"TEST_SERVE_MARKER": str(serve_marker),
|
||||
}
|
||||
python_environment, _ = _pooled_child_environments(rust_environment, {}, None)
|
||||
child: Final = subprocess.run(
|
||||
(sys.executable, "-P", "-c", LAUNCH_MAIN_SOURCE),
|
||||
cwd=ROOT,
|
||||
env=python_environment,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
)
|
||||
assert child.returncode == 0, child.stderr.decode(errors="replace")
|
||||
assert serve_marker.read_text() == "false|"
|
||||
|
||||
|
||||
def _wait_for_gateway(process: subprocess.Popen[bytes], port: int) -> None:
|
||||
deadline: Final = time.monotonic() + 10
|
||||
while time.monotonic() < deadline:
|
||||
if process.poll() is not None:
|
||||
pytest.fail(f"supervisor exited early with status {process.returncode}")
|
||||
if _gateway_readiness_status(port) == 200:
|
||||
return
|
||||
time.sleep(0.05)
|
||||
pytest.fail("supervisor did not start both child listeners")
|
||||
|
||||
|
||||
def _gateway_readiness_status(port: int) -> int | None:
|
||||
connection: Final = http.client.HTTPConnection("127.0.0.1", port, timeout=0.2)
|
||||
try:
|
||||
connection.request("GET", "/health/readiness")
|
||||
response: Final = connection.getresponse()
|
||||
response.read()
|
||||
return response.status
|
||||
except OSError:
|
||||
return None
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _terminate_if_running(process: subprocess.Popen[bytes]) -> None:
|
||||
if process.poll() is not None:
|
||||
return
|
||||
os.killpg(process.pid, signal.SIGKILL) # test-quality-ok: Cleanup terminates only this test-owned supervisor group
|
||||
process.wait(timeout=5)
|
||||
|
||||
|
||||
def test_supervisor_drains_rust_before_python(tmp_path: Path) -> None:
|
||||
private_port: Final = allocate_port()
|
||||
public_port: Final = allocate_port()
|
||||
events: Final = tmp_path / "signals"
|
||||
process: Final = _start_supervisor(tmp_path, private_port, public_port, events)
|
||||
try:
|
||||
_wait_for_gateway(process, public_port)
|
||||
os.kill(process.pid, signal.SIGTERM) # test-quality-ok: The signal is the ordered-shutdown event under test
|
||||
assert process.wait(timeout=10) == 0
|
||||
assert events.read_text().splitlines() == ["rust", "python"]
|
||||
finally:
|
||||
_terminate_if_running(process)
|
||||
|
||||
|
||||
def test_supervisor_fails_when_python_cannot_start(tmp_path: Path) -> None:
|
||||
process: Final = _start_supervisor(
|
||||
tmp_path,
|
||||
allocate_port(),
|
||||
allocate_port(),
|
||||
tmp_path / "signals",
|
||||
python_mode="startup-failure",
|
||||
)
|
||||
try:
|
||||
assert process.wait(timeout=10) == 1
|
||||
finally:
|
||||
_terminate_if_running(process)
|
||||
|
||||
|
||||
def test_supervisor_cleans_up_python_when_rust_cannot_start(tmp_path: Path) -> None:
|
||||
events: Final = tmp_path / "signals"
|
||||
process: Final = _start_supervisor(tmp_path, allocate_port(), allocate_port(), events, rust_mode="missing")
|
||||
try:
|
||||
assert process.wait(timeout=10) == 1
|
||||
assert events.read_text().splitlines() == ["python"]
|
||||
finally:
|
||||
_terminate_if_running(process)
|
||||
|
||||
|
||||
def test_supervisor_cleans_children_and_managed_pooler_when_rust_start_raises(tmp_path: Path) -> None:
|
||||
server: Final = tmp_path / "server.py"
|
||||
server.write_text(SERVER_SOURCE)
|
||||
python_pid_file: Final = tmp_path / "python.pid"
|
||||
python_command: Final = (
|
||||
sys.executable,
|
||||
"-P",
|
||||
str(server),
|
||||
"python",
|
||||
str(allocate_port()),
|
||||
str(tmp_path / "events"),
|
||||
"normal",
|
||||
str(tmp_path / "orphan.pid"),
|
||||
)
|
||||
pooler: Final = start_managed_pgbouncer(
|
||||
PgBouncerSettings(enabled=True, port=allocate_port(), binary=str(_fake_pgbouncer(tmp_path))),
|
||||
"postgresql://litellm:password@127.0.0.1/litellm",
|
||||
)
|
||||
assert isinstance(pooler, ManagedPgBouncerRuntime), pooler
|
||||
pooler_pid: Final = pooler.pooler.pid
|
||||
config: Final = SupervisorConfig(
|
||||
python_command=python_command,
|
||||
rust_command=("/injected/rust-gateway",),
|
||||
python_environment={},
|
||||
rust_environment={},
|
||||
public_probe_host="127.0.0.1",
|
||||
public_port=allocate_port(),
|
||||
private_port=int(python_command[4]),
|
||||
startup_timeout=5,
|
||||
rust_drain_timeout=1,
|
||||
python_grace_timeout=1,
|
||||
termination_grace_timeout=5,
|
||||
managed_pooler=pooler,
|
||||
)
|
||||
try:
|
||||
with ExitStack() as child_cleanup:
|
||||
failure: Final = supervise(
|
||||
config,
|
||||
runner=_FailAfterPythonStartRunner(python_command, python_pid_file, child_cleanup),
|
||||
)
|
||||
assert isinstance(failure, SupervisorFailure)
|
||||
assert "injected Rust gateway start failure" in failure.reason
|
||||
python_pid: Final = int(python_pid_file.read_text())
|
||||
assert not Path(f"/proc/{python_pid}").exists()
|
||||
assert pooler_pid is not None
|
||||
assert not Path(f"/proc/{pooler_pid}").exists()
|
||||
assert not pooler.runtime_dir.exists()
|
||||
finally:
|
||||
pooler.stop(time.monotonic() + 2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exiting_child", ("python", "rust"))
|
||||
def test_supervisor_fails_when_either_child_exits(tmp_path: Path, exiting_child: str) -> None:
|
||||
events: Final = tmp_path / "signals"
|
||||
process: Final = _start_supervisor(
|
||||
tmp_path,
|
||||
allocate_port(),
|
||||
allocate_port(),
|
||||
events,
|
||||
python_mode="exit" if exiting_child == "python" else "normal",
|
||||
rust_mode="exit" if exiting_child == "rust" else "normal",
|
||||
)
|
||||
try:
|
||||
assert process.wait(timeout=10) == 1
|
||||
finally:
|
||||
_terminate_if_running(process)
|
||||
|
||||
|
||||
def test_supervisor_kills_unresponsive_rust_group_and_reaps_orphan(tmp_path: Path) -> None:
|
||||
private_port: Final = allocate_port()
|
||||
public_port: Final = allocate_port()
|
||||
events: Final = tmp_path / "signals"
|
||||
child_pid_file: Final = tmp_path / "orphan.pid"
|
||||
process: Final = _start_supervisor(
|
||||
tmp_path,
|
||||
private_port,
|
||||
public_port,
|
||||
events,
|
||||
rust_mode="orphan",
|
||||
rust_timeout=1,
|
||||
)
|
||||
try:
|
||||
_wait_for_gateway(process, public_port)
|
||||
deadline: Final = time.monotonic() + 5
|
||||
while not child_pid_file.is_file() and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
assert child_pid_file.is_file()
|
||||
child_pid: Final = int(child_pid_file.read_text())
|
||||
os.kill(process.pid, signal.SIGTERM) # test-quality-ok: The signal is the orphan-cleanup event under test
|
||||
assert process.wait(timeout=10) == 0
|
||||
assert events.read_text().splitlines() == ["rust", "python"]
|
||||
assert not Path(f"/proc/{child_pid}").exists()
|
||||
finally:
|
||||
_terminate_if_running(process)
|
||||
|
||||
|
||||
def test_supervisor_reaps_adopted_child_while_gateway_is_ready(tmp_path: Path) -> None:
|
||||
public_port: Final = allocate_port()
|
||||
child_pid_file: Final = tmp_path / "orphan.pid"
|
||||
pooler_pid_file: Final = tmp_path / "pooler.pid"
|
||||
process: Final = _start_supervisor(
|
||||
tmp_path,
|
||||
allocate_port(),
|
||||
public_port,
|
||||
tmp_path / "signals",
|
||||
rust_mode="orphan-exit",
|
||||
managed_pooler=(_fake_pgbouncer(tmp_path), allocate_port(), pooler_pid_file),
|
||||
)
|
||||
try:
|
||||
_wait_for_gateway(process, public_port)
|
||||
pooler_pid: Final = _wait_for_pid(pooler_pid_file)
|
||||
deadline: Final = time.monotonic() + 5
|
||||
while not child_pid_file.is_file() and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
assert child_pid_file.is_file()
|
||||
child_pid: Final = int(child_pid_file.read_text())
|
||||
child_deadline: Final = time.monotonic() + 5
|
||||
while Path(f"/proc/{child_pid}").exists() and time.monotonic() < child_deadline:
|
||||
time.sleep(0.05)
|
||||
assert not Path(f"/proc/{child_pid}").exists()
|
||||
assert process.poll() is None
|
||||
assert Path(f"/proc/{pooler_pid}").exists()
|
||||
os.kill(pooler_pid, signal.SIGKILL) # test-quality-ok: The signal exercises supervised PgBouncer restart
|
||||
replacement_pid: Final = _wait_for_replacement_pooler(pooler_pid_file, pooler_pid)
|
||||
assert replacement_pid is not None
|
||||
assert replacement_pid != pooler_pid
|
||||
_wait_for_gateway(process, public_port)
|
||||
os.kill(process.pid, signal.SIGTERM) # test-quality-ok: Signal shutdown after observing the adopted child reap
|
||||
assert process.wait(timeout=10) == 0
|
||||
assert not Path(f"/proc/{replacement_pid}").exists()
|
||||
finally:
|
||||
_terminate_if_running(process)
|
||||
|
||||
|
||||
def test_supervisor_stops_managed_pooler_when_python_startup_fails(tmp_path: Path) -> None:
|
||||
pooler_pid_file: Final = tmp_path / "pooler.pid"
|
||||
process: Final = _start_supervisor(
|
||||
tmp_path,
|
||||
allocate_port(),
|
||||
allocate_port(),
|
||||
tmp_path / "signals",
|
||||
python_mode="startup-failure",
|
||||
managed_pooler=(_fake_pgbouncer(tmp_path), allocate_port(), pooler_pid_file),
|
||||
)
|
||||
try:
|
||||
assert process.wait(timeout=10) == 1
|
||||
pooler_pid: Final = int(pooler_pid_file.read_text())
|
||||
assert not Path(f"/proc/{pooler_pid}").exists()
|
||||
finally:
|
||||
_terminate_if_running(process)
|
||||
|
||||
|
||||
def test_supervisor_bounds_shutdown_with_an_open_sse_response(tmp_path: Path) -> None:
|
||||
public_port: Final = allocate_port()
|
||||
events: Final = tmp_path / "signals"
|
||||
process: Final = _start_supervisor(
|
||||
tmp_path,
|
||||
allocate_port(),
|
||||
public_port,
|
||||
events,
|
||||
rust_mode="ignore",
|
||||
rust_timeout=1,
|
||||
)
|
||||
connection: Final = http.client.HTTPConnection("127.0.0.1", public_port, timeout=2)
|
||||
try:
|
||||
_wait_for_gateway(process, public_port)
|
||||
connection.request("GET", "/events")
|
||||
response: Final = connection.getresponse()
|
||||
assert response.status == 200
|
||||
assert response.getheader("Content-Type") == "text/event-stream"
|
||||
assert response.readline() == b"data: connected\n"
|
||||
os.kill(process.pid, signal.SIGTERM) # test-quality-ok: The signal starts the bounded SSE drain under test
|
||||
assert process.wait(timeout=10) == 0
|
||||
assert events.read_text().splitlines() == ["rust", "python"]
|
||||
finally:
|
||||
connection.close()
|
||||
_terminate_if_running(process)
|
||||
|
|
@ -0,0 +1,183 @@
|
|||
import http.client
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from tests.integration._support.ports import allocate_port
|
||||
|
||||
ROOT: Final = Path(__file__).resolve().parents[3]
|
||||
DEFAULT_BINARY: Final = ROOT / "litellm-rust" / "target" / "debug" / "litellm-gateway"
|
||||
HEADER_OBSERVATION: Final = TypeAdapter(dict[str, str | None])
|
||||
APP_SOURCE: Final = """\
|
||||
from fastapi import FastAPI, Request
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
@app.get("/health/readiness")
|
||||
async def readiness():
|
||||
return {"status": "healthy"}
|
||||
|
||||
@app.get("/headers")
|
||||
async def headers(request: Request):
|
||||
return {
|
||||
"client": request.client.host if request.client is not None else None,
|
||||
"host": request.headers.get("host"),
|
||||
"origin": request.headers.get("origin"),
|
||||
"xff": request.headers.get("x-forwarded-for"),
|
||||
"proto": request.headers.get("x-forwarded-proto"),
|
||||
"scheme": request.url.scheme,
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def _request(address: str, port: int, source: str | None = None) -> dict[str, str | None]:
|
||||
client: Final = http.client.HTTPConnection(
|
||||
address,
|
||||
port,
|
||||
timeout=1,
|
||||
source_address=None if source is None else (source, 0),
|
||||
)
|
||||
try:
|
||||
client.request(
|
||||
"GET",
|
||||
"/headers",
|
||||
headers={
|
||||
"Host": "gateway.example",
|
||||
"Origin": "https://ui.example",
|
||||
"X-Forwarded-For": "198.51.100.7",
|
||||
"X-Forwarded-Proto": "https",
|
||||
"Connection": "close",
|
||||
},
|
||||
)
|
||||
response: Final = client.getresponse()
|
||||
body: Final = response.read()
|
||||
assert response.status == 200, body
|
||||
return HEADER_OBSERVATION.validate_json(body)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
def _wait_until_ready(address: str, port: int, process: subprocess.Popen[bytes]) -> None:
|
||||
deadline: Final = time.monotonic() + 10
|
||||
while time.monotonic() < deadline:
|
||||
if process.poll() is not None:
|
||||
_stdout, stderr = process.communicate()
|
||||
pytest.fail(
|
||||
f"Rust gateway exited during startup with status {process.returncode}: "
|
||||
f"{(stderr or b'').decode(errors='replace')}"
|
||||
)
|
||||
connection: Final = http.client.HTTPConnection(address, port, timeout=0.2)
|
||||
try:
|
||||
connection.request("GET", "/health/readiness")
|
||||
response: Final = connection.getresponse()
|
||||
response.read()
|
||||
if response.status == 200:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
connection.close()
|
||||
time.sleep(0.05)
|
||||
pytest.fail("Rust gateway did not forward readiness to the Uvicorn fixture")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("trusted_proxies", "expected_client", "expected_proto"),
|
||||
(
|
||||
("", "127.0.0.2", "http"),
|
||||
("127.0.0.2", "198.51.100.7", "https"),
|
||||
),
|
||||
)
|
||||
def test_rust_gateway_canonicalizes_headers_before_real_uvicorn(
|
||||
tmp_path: Path,
|
||||
trusted_proxies: str,
|
||||
expected_client: str,
|
||||
expected_proto: str,
|
||||
) -> None:
|
||||
binary: Final = Path(os.environ.get("LITELLM_RUST_GATEWAY_BINARY", str(DEFAULT_BINARY)))
|
||||
if not binary.is_file():
|
||||
pytest.skip("build litellm-rust/crates/gateway before running the real Uvicorn trust fixture")
|
||||
|
||||
python_port: Final = allocate_port()
|
||||
rust_port: Final = allocate_port()
|
||||
app_path: Final = tmp_path / "fixture.py"
|
||||
app_path.write_text(APP_SOURCE)
|
||||
python_environment: Final = {
|
||||
**os.environ,
|
||||
"PYTHONPATH": str(tmp_path),
|
||||
"FORWARDED_ALLOW_IPS": "127.0.0.1",
|
||||
}
|
||||
python: Final = subprocess.Popen(
|
||||
(
|
||||
sys.executable,
|
||||
"-P",
|
||||
"-m",
|
||||
"uvicorn",
|
||||
"fixture:app",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
str(python_port),
|
||||
"--proxy-headers",
|
||||
"--forwarded-allow-ips",
|
||||
"127.0.0.1",
|
||||
),
|
||||
env=python_environment,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
start_new_session=True,
|
||||
)
|
||||
_wait_until_ready("127.0.0.1", python_port, python)
|
||||
direct_python: Final = _request("127.0.0.1", python_port, "127.0.0.2")
|
||||
assert direct_python == {
|
||||
"client": "127.0.0.2",
|
||||
"host": "gateway.example",
|
||||
"origin": "https://ui.example",
|
||||
"xff": "198.51.100.7",
|
||||
"proto": "https",
|
||||
"scheme": "http",
|
||||
}
|
||||
rust_environment: Final[Mapping[str, str]] = {
|
||||
**os.environ,
|
||||
"HOST": "127.0.0.1",
|
||||
"PORT": str(rust_port),
|
||||
"LITELLM_RUST_GATEWAY": "true",
|
||||
"LITELLM_PYTHON_UPSTREAM": f"http://127.0.0.1:{python_port}",
|
||||
"LITELLM_RUST_GATEWAY_TRUSTED_PROXIES": trusted_proxies,
|
||||
}
|
||||
rust: Final = subprocess.Popen(
|
||||
(str(binary),),
|
||||
env=rust_environment,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
start_new_session=True,
|
||||
)
|
||||
try:
|
||||
_wait_until_ready("127.0.0.1", rust_port, rust)
|
||||
response: Final = _request("127.0.0.1", rust_port, "127.0.0.2")
|
||||
|
||||
assert response == {
|
||||
"client": expected_client,
|
||||
"host": "gateway.example",
|
||||
"origin": "https://ui.example",
|
||||
"xff": expected_client,
|
||||
"proto": expected_proto,
|
||||
"scheme": expected_proto,
|
||||
}
|
||||
finally:
|
||||
for process in (rust, python):
|
||||
if process.poll() is None:
|
||||
os.killpg(process.pid, signal.SIGTERM) # test-quality-ok: Signal only this fixture's process group
|
||||
try:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL) # test-quality-ok: Kill only this fixture's process group
|
||||
process.wait(timeout=5)
|
||||
|
|
@ -3,6 +3,7 @@ import os
|
|||
import socket
|
||||
import sys
|
||||
import textwrap
|
||||
import time
|
||||
import urllib.parse
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
|
@ -13,9 +14,27 @@ import pytest
|
|||
from uvicorn.importer import import_from_string
|
||||
from uvicorn.main import main as uvicorn_main
|
||||
|
||||
from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv
|
||||
from gateway.launch import (
|
||||
GATEWAY_APP,
|
||||
_pooled_child_environments,
|
||||
main,
|
||||
managed_pool_database_url,
|
||||
pool_database_url,
|
||||
uvicorn_argv,
|
||||
)
|
||||
from gateway.supervisor import (
|
||||
RUST_GATEWAY_BINARY,
|
||||
ConfigurationFailure,
|
||||
build_configuration,
|
||||
gateway_enabled,
|
||||
)
|
||||
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
||||
from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings
|
||||
from litellm.proxy.db.pgbouncer import (
|
||||
PGBOUNCER_POOLED_ENV_VAR,
|
||||
ManagedPgBouncerRuntime,
|
||||
PgBouncerError,
|
||||
PgBouncerSettings,
|
||||
)
|
||||
from litellm.proxy.proxy_server import app as proxy_app
|
||||
|
||||
DB_ENV: Final = {
|
||||
|
|
@ -127,6 +146,50 @@ class TestUvicornArgv:
|
|||
|
||||
|
||||
class TestPoolDatabaseUrl:
|
||||
def test_rust_mode_without_a_writer_stays_database_free(self, password_env: dict[str, str]) -> None:
|
||||
assert (
|
||||
managed_pool_database_url(
|
||||
DatabaseURLSettings.from_env(),
|
||||
PgBouncerSettings(enabled=True),
|
||||
{"DATABASE_URL": ""},
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
def test_rust_managed_pooler_is_forced_for_a_writer_when_the_flag_is_off(
|
||||
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
port: Final = _free_port()
|
||||
binary: Final = _fake_pooler(tmp_path)
|
||||
monkeypatch.setenv("DATABASE_URL", "postgresql://litellm_pool:p%40ss@db.internal:5432/litellm")
|
||||
outcome: Final = managed_pool_database_url(
|
||||
DatabaseURLSettings.from_env(),
|
||||
PgBouncerSettings(enabled=False, port=port, binary=str(binary)),
|
||||
os.environ,
|
||||
)
|
||||
assert isinstance(outcome, ManagedPgBouncerRuntime), outcome
|
||||
try:
|
||||
assert urllib.parse.urlsplit(outcome.pooled_url).port == port
|
||||
configuration: Final = build_configuration(
|
||||
("--host", "0.0.0.0", "--port", "4000"),
|
||||
{RUST_GATEWAY_BINARY: "/bin/true"},
|
||||
sys.executable,
|
||||
)
|
||||
assert not isinstance(configuration, ConfigurationFailure)
|
||||
python_environment, rust_environment = _pooled_child_environments(
|
||||
configuration.python_environment,
|
||||
configuration.rust_environment,
|
||||
outcome,
|
||||
)
|
||||
assert python_environment["DATABASE_URL"] == rust_environment["DATABASE_URL"] == outcome.pooled_url
|
||||
assert python_environment[PGBOUNCER_POOLED_ENV_VAR] == rust_environment[PGBOUNCER_POOLED_ENV_VAR] == "true"
|
||||
assert python_environment["LITELLM_PGBOUNCER_ENABLED"] == "false"
|
||||
assert rust_environment.get("LITELLM_PGBOUNCER_ENABLED") == configuration.rust_environment.get(
|
||||
"LITELLM_PGBOUNCER_ENABLED"
|
||||
)
|
||||
finally:
|
||||
outcome.stop(time.monotonic() + 2)
|
||||
|
||||
def test_a_disabled_pooler_yields_no_url_to_install(self, password_env: dict[str, str]):
|
||||
settings: Final = DatabaseURLSettings.from_env()
|
||||
settings.apply_to_env()
|
||||
|
|
@ -158,6 +221,149 @@ class TestPoolDatabaseUrl:
|
|||
assert "MINTED_TOKEN" not in outcome
|
||||
|
||||
|
||||
class TestRustGatewayConfiguration:
|
||||
@pytest.fixture
|
||||
def rust_binary(self, tmp_path: Path) -> Path:
|
||||
binary: Final = tmp_path / "litellm-gateway"
|
||||
binary.write_text("#!/bin/sh\nexit 0\n")
|
||||
binary.chmod(0o700)
|
||||
return binary
|
||||
|
||||
@pytest.fixture
|
||||
def clear_uvicorn_environment(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
for name in (
|
||||
"UVICORN_HOST",
|
||||
"UVICORN_PORT",
|
||||
"UVICORN_UDS",
|
||||
"UVICORN_FD",
|
||||
"UVICORN_RELOAD",
|
||||
"UVICORN_PROXY_HEADERS",
|
||||
"UVICORN_SSL_KEYFILE",
|
||||
):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
|
||||
def test_flag_requires_exact_lowercase_values(self) -> None:
|
||||
assert gateway_enabled({}) is False
|
||||
assert gateway_enabled({"LITELLM_RUST_GATEWAY": "true"}) is True
|
||||
assert isinstance(gateway_enabled({"LITELLM_RUST_GATEWAY": "TRUE"}), ConfigurationFailure)
|
||||
|
||||
def test_private_worker_keeps_config_rootpath_keepalive_and_unknown_options(
|
||||
self,
|
||||
rust_binary: Path,
|
||||
clear_uvicorn_environment: None,
|
||||
) -> None:
|
||||
environ: Final = {
|
||||
RUST_GATEWAY_BINARY: str(rust_binary),
|
||||
"CONFIG_FILE_PATH": "/app/config/config.yaml",
|
||||
"KEEPALIVE_TIMEOUT": "41",
|
||||
}
|
||||
configuration: Final = build_configuration(
|
||||
(
|
||||
"--workers",
|
||||
"3",
|
||||
"--proxy-headers",
|
||||
"--host",
|
||||
"0.0.0.0",
|
||||
"--port",
|
||||
"4000",
|
||||
"--root-path",
|
||||
"/gateway",
|
||||
"--unknown-option",
|
||||
"preserved",
|
||||
"--proxy-headers",
|
||||
),
|
||||
environ,
|
||||
sys.executable,
|
||||
)
|
||||
|
||||
assert not isinstance(configuration, ConfigurationFailure)
|
||||
assert configuration.python_command == (
|
||||
sys.executable,
|
||||
"-m",
|
||||
"gateway.launch",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
"4002",
|
||||
"--proxy-headers",
|
||||
"--forwarded-allow-ips",
|
||||
"127.0.0.1",
|
||||
"--workers",
|
||||
"3",
|
||||
"--root-path",
|
||||
"/gateway",
|
||||
"--unknown-option",
|
||||
"preserved",
|
||||
"--timeout-keep-alive",
|
||||
"41",
|
||||
)
|
||||
assert configuration.python_environment["CONFIG_FILE_PATH"] == "/app/config/config.yaml"
|
||||
assert configuration.python_environment["LITELLM_RUST_GATEWAY"] == "false"
|
||||
assert configuration.rust_environment["LITELLM_RUST_GATEWAY"] == "true"
|
||||
assert configuration.rust_environment["LITELLM_PYTHON_UPSTREAM"] == "http://127.0.0.1:4002"
|
||||
assert configuration.rust_environment["HOST"] == "0.0.0.0"
|
||||
assert configuration.rust_environment["PORT"] == "4000"
|
||||
|
||||
def test_database_writer_reserves_pgbouncer_port_when_the_python_only_flag_is_false(
|
||||
self, rust_binary: Path, clear_uvicorn_environment: None
|
||||
) -> None:
|
||||
configuration: Final = build_configuration(
|
||||
("--host", "0.0.0.0", "--port", "4000"),
|
||||
{
|
||||
RUST_GATEWAY_BINARY: str(rust_binary),
|
||||
"DATABASE_URL": "postgresql://app:pw@db/litellm",
|
||||
"LITELLM_PGBOUNCER_ENABLED": "false",
|
||||
"LITELLM_PGBOUNCER_PORT": "4000",
|
||||
},
|
||||
sys.executable,
|
||||
)
|
||||
assert isinstance(configuration, ConfigurationFailure)
|
||||
assert "PgBouncer" in configuration.reason
|
||||
assert "same TCP port" in configuration.reason
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("arguments", "extra_environment", "reason"),
|
||||
(
|
||||
(("--no-proxy-headers",), {}, "--no-proxy-headers"),
|
||||
(("--ssl-certfile", "/tmp/cert.pem"), {}, "TLS"),
|
||||
(("--env-file", "/tmp/uvicorn.env"), {}, "--env-file"),
|
||||
(("--uds", "/tmp/gateway.sock"), {}, "Unix socket"),
|
||||
(("--fd", "5"), {}, "file descriptor"),
|
||||
(("--http", "h2"), {}, "HTTP/1.1"),
|
||||
(("--host", "0.0.0.0", "--port", "4000"), {"PORT": "4001"}, "PORT conflicts"),
|
||||
(("--host", "0.0.0.0", "--port", "4000"), {"LITELLM_GATEWAY_METRICS_PORT": "4002"}, "same TCP port"),
|
||||
(
|
||||
("--host", "0.0.0.0", "--port", "4000"),
|
||||
{"LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS": "20"},
|
||||
"must allow Rust drain",
|
||||
),
|
||||
(
|
||||
("--host", "0.0.0.0", "--port", "4000"),
|
||||
{"LITELLM_RUST_GATEWAY_TERMINATION_GRACE_SECONDS": "0"},
|
||||
"must be a positive integer",
|
||||
),
|
||||
(
|
||||
("--host", "0.0.0.0", "--port", "4000"),
|
||||
{"LITELLM_COLLECTOR_ENABLED": "true", "LITELLM_COLLECTOR_ADDRESS": "tcp://127.0.0.1:4002"},
|
||||
"same TCP port",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_rejects_unsupported_or_conflicting_configuration(
|
||||
self,
|
||||
rust_binary: Path,
|
||||
clear_uvicorn_environment: None,
|
||||
arguments: tuple[str, ...],
|
||||
extra_environment: dict[str, str],
|
||||
reason: str,
|
||||
) -> None:
|
||||
environ: Final = {RUST_GATEWAY_BINARY: str(rust_binary), **extra_environment}
|
||||
outcome: Final = build_configuration(arguments, environ, sys.executable)
|
||||
|
||||
assert isinstance(outcome, ConfigurationFailure)
|
||||
assert reason in outcome.reason
|
||||
|
||||
|
||||
class TestMain:
|
||||
def test_workers_inherit_the_loopback_url_the_supervisor_installed(
|
||||
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue