scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 101

Merge pull request #40482 from BerriAI/litellm_e2e_redis_timeout

Browse source 
test(load): add a Redis timeout chaos load test
This commit is contained in:
kerry-berri 2026-09-11 16:58:52 -07:00 • committed by GitHub
commit 8e4f2abb40
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 1299 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

103
.github/workflows/test-e2e-redis-chaos.yml vendored Normal file
View file

@ -0,0 +1,103 @@
name: "Redis Chaos E2E"
on:
workflow_dispatch:
workflow_call:
inputs:
ref:
description: "Commit SHA or ref to test. Defaults to the ref the workflow was triggered on"
required: false
type: string
permissions:
contents: read
jobs:
redis-chaos-e2e:
runs-on: ubuntu-latest-16-cores
timeout-minutes: 30
services:
postgres:
image: postgres:16.6@sha256:557fea37a744d5f4c8faab304b0a90858b53ab119735a88c131fd19dab802f36
env:
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U llmproxy"
--health-interval 5s
--health-timeout 5s
--health-retries 10
valkey:
image: valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa
ports:
- 6379:6379
options: >-
--health-cmd "valkey-cli ping"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
LITELLM_MASTER_KEY: sk-redis-chaos-e2e
LITELLM_LOG: WARNING
JSON_LOGS: "true"
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
ref: ${{ inputs.ref || github.sha }}
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --group e2e-dev --extra proxy
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Start a multi-worker proxy on the chaos config
run: |
nohup uv run --no-sync litellm --config tests/e2e/gateway/redis_chaos_ci_config.yml --port 4000 --num_workers 4 > proxy.log 2>&1 &
echo "E2E_PROXY_PID=$!" >> "$GITHUB_ENV"
echo "E2E_PROXY_LOG=$(pwd)/proxy.log" >> "$GITHUB_ENV"
for _ in $(seq 1 90); do
if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
exit 0
fi
sleep 2
done
echo "proxy never became live"
tail -n 100 proxy.log
exit 1
- name: Run the Redis chaos load test
env:
E2E_REDIS_CHAOS: "1"
LITELLM_PROXY_URL: http://localhost:4000
REDIS_HOST: 127.0.0.1
REDIS_PORT: "6379"
run: |
uv run --no-sync pytest tests/e2e/load/test_redis_chaos_e2e.py -v --tb=short -rA -s
- name: Show proxy log on failure
if: failure()
run: tail -n 300 proxy.log

1
pyproject.toml
View file

@ -220,6 +220,7 @@ e2e-dev = [
"playwright==1.61.0",
"websockets>=15.0.1,<16.0",
"locust==2.45.0",
"psutil==7.2.2",
"mcp>=1.28.1,<2.0",
]
proxy-dev = [

2
tests/e2e/CLAUDE.md
View file

@ -18,7 +18,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
- `logging/` - logging-integration delivery (datadog and friends)
- `security/` - secret handling and log-leak protection
- `router/` - routing and reliability behavior (fallbacks, cooldowns)
- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What remains here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`) and markerless harness unit tests for the Locust/session-anomaly aggregation logic
- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic
- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite
- `gateway/` - proxy configuration only (`litellm-config.yml`); no tests
- `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke

2
tests/e2e/CONTRIBUTING.md
View file

@ -65,7 +65,7 @@ A couple of logging destinations are configured on the proxy rather than by the
### The pull request check
Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start
Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set
Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch

9
tests/e2e/conftest.py
View file

@ -20,16 +20,14 @@ from datetime import datetime, timezone
import pytest
import requests
from e2e_config import CONTROL_PLANE_BASE_URL, FIXTURE_DIR, FIXTURE_MODE_RAW, PROXY_BASE_URL
from e2e_db import RESET_OPT_IN_ENV, reset_spend_logs, run_spend_log_cleanup
from fixture_mode import fixture_mode_collection_error, fixture_report_lines
from provider_edge import replay_leftover_error
from junit_properties import attach_result_properties
from lifecycle import ProxyClientProvider, ResourceManager
from provider_edge import replay_leftover_error
from proxy_client import ProxyClient, build_proxy_client
_E2E_TEST_RAN = pytest.StashKey[bool]()
_CALL_PASSED = pytest.StashKey[bool]()
@ -60,6 +58,11 @@ def pytest_configure(config: pytest.Config) -> None:
"markers",
"managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set",
)
config.addinivalue_line(
"markers",
"redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from "
"gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set",
)
def pytest_sessionstart(session: pytest.Session) -> None:

1
tests/e2e/coverage_registry/reliability.yaml
View file

@ -31,6 +31,7 @@
- {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"}
- {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix"}
- {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"}
- {id: reliability.circuit_breaker.redis_timeout.stays_responsive, module: reliability, tier: P1, behavior: circuit_breaker, variant: redis_timeout, assertions: [stays_responsive], exercised_on: [chat_completions, messages], source: "litellm/proxy/hooks/proxy_track_cost_callback.py:386", fail_before_fix: proven, rationale: "Under locust load split round robin over /chat/completions and /v1/messages with every request retrying through failing mock deployments, holding Redis in CLIENT PAUSE ALL for the phase trips the breaker and every request still succeeds, with latency, RSS, and CPU reported as p50/p90/p99 against the pre-pause baseline; on v1.100.0 the failed-tracking alert body doubled per request until the worker OOMed (LIT-6780)"}
- {id: reliability.timeout.request_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: request_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions, messages], source: "litellm/router.py:545-551", rationale: "Per-request timeout raises Timeout"}
- {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"}
- {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"}

5
tests/e2e/e2e_config.py
View file

@ -13,7 +13,6 @@ from pathlib import Path
from typing import Final
from dotenv import load_dotenv
from fixture_mode import deterministic_marker, parse_fixture_mode
from provider_edge import provider_edge_api_base
@ -144,6 +143,7 @@ LOAD_MIN_CONCURRENCY_EFFICIENCY = float(os.environ.get("E2E_LOAD_MIN_CONCURRENCY
WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY"
MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK"
REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS"
ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6"))
ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6"))
ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3"))
@ -181,8 +181,7 @@ def datadog_mcp_url(*, toolsets: str = "core") -> str:
site = (
os.environ.get("DD_SITE", DD_SITE) or "datadoghq.com"
).strip().removeprefix("https://").removeprefix("http://").rstrip("/")
if site.startswith("app."):
site = site[len("app.") :]
site = site.removeprefix("app.")
host = "mcp.datadoghq.com" if site in ("", "datadoghq.com") else f"mcp.{site}"
base = f"https://{host}/v1/mcp"
return f"{base}?toolsets={toolsets}" if toolsets else base

19
tests/e2e/gateway/redis_chaos_ci_config.yml Normal file
View file

@ -0,0 +1,19 @@
general_settings:
master_key: os.environ/LITELLM_MASTER_KEY
store_model_in_db: true
use_redis_transaction_buffer: true
litellm_settings:
callbacks: ["prometheus"]
require_auth_for_metrics_endpoint: false
enable_redis_auth_cache: true
cache: true
cache_params:
type: redis
host: 127.0.0.1
port: 6379
socket_timeout: 0.1
router_settings:
num_retries: 2
disable_cooldowns: true

23
tests/e2e/load/conftest.py
View file

@ -3,26 +3,23 @@ from __future__ import annotations
import os
import pytest
from e2e_config import WEEKLY_ANOMALY_OPT_IN_ENV
from e2e_config import REDIS_CHAOS_OPT_IN_ENV, WEEKLY_ANOMALY_OPT_IN_ENV
from load_client import LoadClient, build_client
from proxy_client import ProxyClient
_OPT_IN_MARKERS = (
("weekly", WEEKLY_ANOMALY_OPT_IN_ENV),
("redis_chaos", REDIS_CHAOS_OPT_IN_ENV),
)
def pytest_collection_modifyitems(
config: pytest.Config, items: list[pytest.Item]
) -> None:
if os.environ.get(WEEKLY_ANOMALY_OPT_IN_ENV):
return
deselected = [
item for item in items if item.get_closest_marker("weekly") is not None
]
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
opted_out = {marker for marker, opt_in_env in _OPT_IN_MARKERS if not os.environ.get(opt_in_env)}
deselected = [item for item in items if any(item.get_closest_marker(marker) is not None for marker in opted_out)]
if not deselected:
return
config.hook.pytest_deselected(items=deselected)
items[:] = [
item for item in items if item.get_closest_marker("weekly") is None
]
items[:] = [item for item in items if item not in deselected]
@pytest.fixture(scope="session")

150
tests/e2e/load/locust_load.py
View file

@ -1,17 +1,26 @@
from __future__ import annotations
import csv
import os
import subprocess
import sys
import tempfile
from collections.abc import Sequence
from dataclasses import dataclass
from itertools import accumulate
from pathlib import Path
from typing import Final
from pydantic import BaseModel, TypeAdapter
_LOCUSTFILE = Path(__file__).with_name("locustfile.py")
_CSV_PREFIX = "locust"
_GENERATOR_SATURATION_MARKER = "CPU usage above"
_MAX_REPORTED_ERRORS = 5
class LocustStatEntry(BaseModel):
name: str
num_requests: int
num_failures: int
start_time: float
@ -29,12 +38,25 @@ class LoadError:
occurrences: int
@dataclass(frozen=True, slots=True)
class EndpointLoad:
"""One route's share of a phase, so a run that silently drove only one of them is visible."""
name: str
requests: int
failures: int
p50_seconds: float
@dataclass(frozen=True, slots=True)
class LoadResult:
requests: int
failures: int
requests_per_second: float
median_response_seconds: float
p50_seconds: float
p90_seconds: float
p99_seconds: float
endpoints: tuple[EndpointLoad, ...]
errors: tuple[LoadError, ...]
generator_warnings: tuple[str, ...]
@ -53,33 +75,65 @@ class LoadResult:
lines.append("locust recorded no error breakdown")
return "; ".join((*lines, *self.generator_warnings))
def latency_summary(self) -> str:
return f"p50 {self.p50_seconds:.3f}s, p90 {self.p90_seconds:.3f}s, p99 {self.p99_seconds:.3f}s"
def median_seconds(entries: list[LocustStatEntry]) -> float:
samples = sorted(
(milliseconds, count) for entry in entries for milliseconds, count in entry.response_times.items()
)
def endpoint_summary(self) -> str:
return ", ".join(
f"{endpoint.name} {endpoint.requests} requests, {endpoint.failures} failures, "
f"p50 {endpoint.p50_seconds:.3f}s"
for endpoint in self.endpoints
)
def percentile_seconds(entries: Sequence[LocustStatEntry], fraction: float) -> float:
"""The response time at `fraction` of the merged histograms, in seconds.
Locust buckets response times by millisecond, so this reads the first bucket whose
running count reaches the rank, the same lower-sample convention locust's own
percentiles use.
"""
samples = sorted((milliseconds, count) for entry in entries for milliseconds, count in entry.response_times.items())
total = sum(count for _, count in samples)
if total == 0:
return 0.0
running = accumulate(count for _, count in samples)
return next(
milliseconds for (milliseconds, _), seen in zip(samples, running) if seen >= total / 2
) / 1000.0
rank: Final = total * fraction
return next(milliseconds for (milliseconds, _), seen in zip(samples, running) if seen >= rank) / 1000.0
def per_endpoint(entries: Sequence[LocustStatEntry]) -> tuple[EndpointLoad, ...]:
"""Each locust request name's own totals, in the order the names first appear."""
names: Final = tuple(dict.fromkeys(entry.name for entry in entries))
grouped: Final = ((name, tuple(entry for entry in entries if entry.name == name)) for name in names)
return tuple(
EndpointLoad(
name=name,
requests=sum(entry.num_requests for entry in group),
failures=sum(entry.num_failures for entry in group),
p50_seconds=percentile_seconds(group, 0.5),
)
for name, group in grouped
)
def aggregate_stats(
entries: list[LocustStatEntry],
entries: Sequence[LocustStatEntry],
errors: tuple[LoadError, ...],
generator_warnings: tuple[str, ...],
) -> LoadResult:
requests = sum(entry.num_requests for entry in entries)
failures = sum(entry.num_failures for entry in entries)
endpoints = per_endpoint(entries)
if not entries or requests == 0:
return LoadResult(
requests=requests,
failures=failures,
requests_per_second=0.0,
median_response_seconds=0.0,
p50_seconds=0.0,
p90_seconds=0.0,
p99_seconds=0.0,
endpoints=endpoints,
errors=errors,
generator_warnings=generator_warnings,
)
@ -88,7 +142,10 @@ def aggregate_stats(
requests=requests,
failures=failures,
requests_per_second=requests / elapsed if elapsed > 0 else 0.0,
median_response_seconds=median_seconds(entries),
p50_seconds=percentile_seconds(entries, 0.5),
p90_seconds=percentile_seconds(entries, 0.9),
p99_seconds=percentile_seconds(entries, 0.99),
endpoints=endpoints,
errors=errors,
generator_warnings=generator_warnings,
)
@ -121,3 +178,74 @@ def read_generator_warnings(stderr: str) -> tuple[str, ...]:
if _GENERATOR_SATURATION_MARKER in line
)
return tuple(dict.fromkeys(saturated))
def run_gateway_load(
*,
base_url: str,
api_keys: tuple[str, ...],
model: str,
endpoints: tuple[str, ...],
users: int,
spawn_rate: float,
duration_seconds: float,
) -> LoadResult:
"""Drive `endpoints` from headless locust and aggregate what it reported.
Each simulated user picks one of `api_keys`, so auth and budget lookups spread over a
pool of virtual keys instead of keeping one key's cache entry permanently warm, and one
of `endpoints` round robin, so the run covers every route the caller asked for.
"""
with tempfile.TemporaryDirectory(prefix="e2e-load-") as report_dir:
csv_prefix = Path(report_dir) / _CSV_PREFIX
completed = subprocess.run(
[
sys.executable,
"-m",
"locust",
"--headless",
"--json",
"--csv",
str(csv_prefix),
"--locustfile",
str(_LOCUSTFILE),
"--host",
base_url,
"--users",
str(users),
"--spawn-rate",
str(spawn_rate),
"--run-time",
f"{int(duration_seconds)}s",
"--exit-code-on-error",
"0",
],
env={
**os.environ,
"LOAD_API_KEYS": ",".join(api_keys),
"LOAD_MODEL": model,
"LOAD_ENDPOINTS": ",".join(endpoints),
},
capture_output=True,
text=True,
timeout=duration_seconds + 120,
check=False,
)
if completed.returncode != 0:
raise RuntimeError(
f"locust exited {completed.returncode} before it could report throughput "
f"(a startup failure, not request failures, which are folded into the JSON summary via "
f"--exit-code-on-error 0):\n{completed.stderr}"
)
try:
entries = _STATS_ADAPTER.validate_json(completed.stdout)
except ValueError as exc:
raise RuntimeError(
f"locust exited 0 but did not print a parseable --json throughput summary on stdout; "
f"got stdout={completed.stdout!r}, stderr={completed.stderr!r}"
) from exc
return aggregate_stats(
entries,
read_errors(csv_prefix.with_name(f"{_CSV_PREFIX}_failures.csv")),
read_generator_warnings(completed.stderr),
)

52
tests/e2e/load/locustfile.py Normal file
View file

@ -0,0 +1,52 @@
from __future__ import annotations
import os
import random
import uuid
from itertools import cycle
from typing import Final
from locust import FastHttpUser, constant, task
_MODEL: Final = os.environ["LOAD_MODEL"]
_API_KEYS: Final = tuple(os.environ["LOAD_API_KEYS"].split(","))
_NEXT_ENDPOINT: Final = cycle(os.environ["LOAD_ENDPOINTS"].split(","))
_FILLER: Final = "x" * 40_000
def _payload() -> dict[str, object]:
"""A prompt no other request sent, so the response cache never answers for the deployment.
Both endpoints take the same body: /v1/messages requires max_tokens, which /chat/completions
also accepts, so one payload serves the whole round robin. Padded to tens of KB so a
per-request bookkeeping cost that scales with body size (string formatting, hashing) shows
up in the CPU and log-size budgets instead of hiding behind a 40-byte prompt.
"""
return {
"model": _MODEL,
"messages": [{"role": "user", "content": f"load test ping {uuid.uuid4().hex} {_FILLER}"}],
"max_tokens": 16,
}
class GatewayUser(FastHttpUser):
"""One simulated user, pinned to one endpoint for its lifetime.
Endpoints are handed out round robin as users spawn, so a run spreads evenly over them
while each user's traffic stays on a single route, the way a real client behaves.
"""
wait_time = constant(0)
def on_start(self) -> None:
self.headers = {"Authorization": f"Bearer {random.choice(_API_KEYS)}"}
self.endpoint = next(_NEXT_ENDPOINT)
@task
def call(self) -> None:
self.client.post( # pyright: ignore[reportUnknownMemberType] # locust FastHttpSession.post types json/**kwargs as Any
self.endpoint,
json=_payload(),
headers=self.headers,
name=self.endpoint,
)

81
tests/e2e/load/phase_budget.py Normal file
View file

@ -0,0 +1,81 @@
"""Comparing one load phase against another, for tests that degrade a dependency mid-run.
Two shapes of ceiling, because the metrics divide into two kinds. RSS and CPU are
machine-shaped: RSS scales with worker count and CPU with core count, so an absolute number
calibrated on one runner means nothing on the next, and what travels is the ratio against a
healthy phase measured on the same machine in the same run. Latency and log volume are not:
a ratio there is actively misleading, because a dependency that fails fast once its breaker
opens can make the degraded phase look cheaper than the healthy one while still being far
slower or noisier than a user should ever see. Those get a flat ceiling, which is the promise
the test is actually making.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Final, TypeAlias
def _rendered(value: float, unit: str, decimals: int) -> str:
return f"{value:.{decimals}f}{unit}"
@dataclass(frozen=True, slots=True)
class RatioBudget:
"""One metric's healthy value, its degraded value, and how much growth is allowed."""
name: str
baseline: float
degraded: float
ratio_ceiling: float
unit: str
decimals: int = 1
@property
def ratio(self) -> float | None:
"""How many times the baseline the degraded value is, or None if there is no baseline."""
return self.degraded / self.baseline if self.baseline > 0 else None
def violation(self) -> str | None:
"""Why this metric fails its budget, or None if it passes."""
ratio: Final = self.ratio
if ratio is None:
return (
f"{self.name} measured {_rendered(self.baseline, self.unit, self.decimals)} in the healthy phase, "
f"so there is nothing to compare the degraded phase against; the measurement did not happen"
)
if ratio > self.ratio_ceiling:
return (
f"{self.name} went from {_rendered(self.baseline, self.unit, self.decimals)} healthy to "
f"{_rendered(self.degraded, self.unit, self.decimals)} degraded, {ratio:.1f}x the baseline and past "
f"the {self.ratio_ceiling:.1f}x allowed"
)
return None
@dataclass(frozen=True, slots=True)
class AbsoluteBudget:
"""One metric's degraded value against a flat ceiling, for metrics a ratio cannot bound."""
name: str
measured: float
ceiling: float
unit: str
decimals: int = 1
def violation(self) -> str | None:
"""Why this metric fails its budget, or None if it passes."""
if self.measured > self.ceiling:
return (
f"{self.name} measured {_rendered(self.measured, self.unit, self.decimals)} in the degraded phase, "
f"past the {_rendered(self.ceiling, self.unit, self.decimals)} allowed"
)
return None
Budget: TypeAlias = RatioBudget | AbsoluteBudget
def violations(budgets: tuple[Budget, ...]) -> tuple[str, ...]:
"""Every budget the run blew, so one failure reports all of them instead of the first."""
return tuple(violation for budget in budgets if (violation := budget.violation()) is not None)

164
tests/e2e/load/proxy_usage.py Normal file
View file

@ -0,0 +1,164 @@
"""Resident memory and CPU of the proxy process tree, sampled on a background thread.
The proxy under load runs several worker processes, and `/metrics` cannot report their
memory: litellm sets PROMETHEUS_MULTIPROC_DIR when num_workers > 1, and the multiprocess
collector drops the process collector's `process_resident_memory_bytes` /
`process_cpu_seconds_total` entirely. So the test measures the tree itself through psutil,
which needs the proxy to run on the same host as the test.
"""
from __future__ import annotations
import math
import threading
import time
from dataclasses import dataclass
from typing import Final
import psutil
from pydantic import BaseModel, ConfigDict
class _MemoryInfo(BaseModel):
model_config = ConfigDict(from_attributes=True)
rss: int
@dataclass(frozen=True, slots=True)
class UsageSample:
elapsed_seconds: float
rss_bytes: int
cpu_seconds: float
@dataclass(frozen=True, slots=True)
class UsageWindow:
"""The samples taken across one phase, plus what they say about that phase."""
samples: tuple[UsageSample, ...]
def rss_percentile(self, fraction: float) -> int:
if not self.samples:
return 0
ordered: Final = sorted(sample.rss_bytes for sample in self.samples)
return ordered[_rank(len(ordered), fraction)]
def cpu_seconds_consumed(self) -> float:
"""CPU seconds the tree burned across the window, from its monotonic counter."""
if len(self.samples) < 2:
return 0.0
return self.samples[-1].cpu_seconds - self.samples[0].cpu_seconds
def cpu_seconds_per_request(self, requests: int) -> float:
"""CPU seconds the tree spent per request served.
The portable cost figure: cores-busy saturates at the worker count under enough load,
so it reads the same whether a request costs 10 ms of CPU or 40 ms. This does not.
"""
return self.cpu_seconds_consumed() / requests if requests else 0.0
def cpu_utilization_percentiles(self) -> tuple[float, float, float]:
"""Per-interval CPU utilization (cores busy) at p50, p90 and p99.
Derived from consecutive samples of the cumulative counter rather than
psutil's own cpu_percent, so it covers every process in the tree including
workers that came and went between samples.
"""
rates: Final = sorted(
(later.cpu_seconds - earlier.cpu_seconds) / (later.elapsed_seconds - earlier.elapsed_seconds)
for earlier, later in zip(self.samples, self.samples[1:])
if later.elapsed_seconds > earlier.elapsed_seconds
)
if not rates:
return 0.0, 0.0, 0.0
return (
rates[_rank(len(rates), 0.5)],
rates[_rank(len(rates), 0.9)],
rates[_rank(len(rates), 0.99)],
)
def summary(self) -> str:
p50_cpu, p90_cpu, p99_cpu = self.cpu_utilization_percentiles()
return (
f"RSS p50 {self.rss_percentile(0.5) / 2**20:.0f} MB, "
f"p90 {self.rss_percentile(0.9) / 2**20:.0f} MB, "
f"p99 {self.rss_percentile(0.99) / 2**20:.0f} MB; "
f"CPU cores busy p50 {p50_cpu:.2f}, p90 {p90_cpu:.2f}, p99 {p99_cpu:.2f}; "
f"{self.cpu_seconds_consumed():.1f} CPU seconds consumed"
)
def _rank(count: int, fraction: float) -> int:
"""Index of the sample at `fraction`, the same lower-sample convention as locust's percentiles."""
return min(count - 1, max(0, math.ceil(count * fraction) - 1))
def _read_process(process: psutil.Process) -> tuple[int, float] | None:
try:
with process.oneshot():
memory: Final = _MemoryInfo.model_validate(process.memory_info())
times: Final = process.cpu_times()
return memory.rss, times.user + times.system
except (psutil.NoSuchProcess, psutil.AccessDenied):
return None
class ProxyUsageSampler:
"""Samples the proxy process tree every `interval_seconds` until stopped.
`split()` returns the samples taken so far and starts a new window, so one sampler
covers a baseline phase and a chaos phase without a gap between them.
"""
def __init__(self, pid: int, interval_seconds: float = 1.0) -> None:
self._process: Final = psutil.Process(pid)
self._interval: Final = interval_seconds
self._stop: Final = threading.Event()
self._lock: Final = threading.Lock()
self._samples: list[UsageSample] = [] # mutable-ok: a sampling buffer the reader drains under a lock
self._started: Final = time.monotonic()
self._thread: Final = threading.Thread(target=self._run, name="proxy-usage-sampler", daemon=True)
def __enter__(self) -> ProxyUsageSampler:
self._thread.start()
return self
def __exit__(self, *_: object) -> None:
self._stop.set()
self._thread.join(timeout=self._interval * 5)
def _tree(self) -> tuple[psutil.Process, ...]:
try:
return (self._process, *self._process.children(recursive=True))
except psutil.NoSuchProcess:
return ()
def _sample(self) -> UsageSample | None:
readings: Final = tuple(reading for process in self._tree() if (reading := _read_process(process)) is not None)
if not readings:
return None
return UsageSample(
elapsed_seconds=time.monotonic() - self._started,
rss_bytes=sum(rss for rss, _ in readings),
cpu_seconds=sum(cpu for _, cpu in readings),
)
def _run(self) -> None:
while not self._stop.is_set():
sample = self._sample()
if sample is not None:
with self._lock:
self._samples.append(sample)
self._stop.wait(self._interval)
def split(self) -> UsageWindow:
"""The window that ends now; the next one starts from this window's last sample.
The boundary sample is carried into the next window so its CPU counter has a
starting point, which is what makes the two windows' utilization comparable.
"""
with self._lock:
taken = tuple(self._samples)
self._samples = [taken[-1]] if taken else [] # rebind-ok: drains the buffer under the lock
return UsageWindow(samples=taken)

91
tests/e2e/load/test_locust_load.py
View file

@ -1,13 +1,14 @@
from __future__ import annotations
from pathlib import Path
from typing import Final
from locust_load import (
LoadError,
LoadResult,
LocustStatEntry,
aggregate_stats,
median_seconds,
percentile_seconds,
read_errors,
read_generator_warnings,
)
@ -18,12 +19,14 @@ _FAILURES_HEADER = "Method,Name,Error,Occurrences,First Seen,Last Seen\n"
def _entry(
*,
num_requests: int,
name: str = "/chat/completions",
num_failures: int = 0,
start_time: float = 1000.0,
last_request_timestamp: float = 1010.0,
response_times: dict[int, int] | None = None,
) -> LocustStatEntry:
return LocustStatEntry(
name=name,
num_requests=num_requests,
num_failures=num_failures,
start_time=start_time,
@ -41,35 +44,48 @@ def _result(
requests=10,
failures=10,
requests_per_second=1.0,
median_response_seconds=0.05,
p50_seconds=0.05,
p90_seconds=0.08,
p99_seconds=0.1,
endpoints=(),
errors=errors,
generator_warnings=generator_warnings,
)
class TestSerialLatency:
class TestPercentiles:
def test_median_is_the_middle_sample_not_the_mean_a_slow_tail_would_drag(self) -> None:
# Nine fast requests and one very slow one: the mean is 1.99s, the median is 20ms.
entry = _entry(num_requests=10, response_times={20: 9, 20000: 1})
assert median_seconds([entry]) == 0.02
assert percentile_seconds([entry], 0.5) == 0.02
def test_median_merges_the_histograms_of_every_stats_entry(self) -> None:
def test_the_tail_percentiles_reach_the_slow_samples_the_median_hides(self) -> None:
# 100 samples: 89 fast, 10 slow, 1 very slow. p50 sits in the fast bucket, p90 in the
# slow one, and p99 lands on the single very slow sample.
entry = _entry(num_requests=100, response_times={20: 89, 500: 10, 20000: 1})
assert percentile_seconds([entry], 0.5) == 0.02
assert percentile_seconds([entry], 0.9) == 0.5
assert percentile_seconds([entry], 0.99) == 0.5
assert percentile_seconds([entry], 1.0) == 20.0
def test_percentiles_merge_the_histograms_of_every_stats_entry(self) -> None:
# Per entry the median would be 10ms and 90ms; merged, the middle of the five samples is 90ms.
entries = [
_entry(num_requests=2, response_times={10: 2}),
_entry(num_requests=3, response_times={90: 3}),
]
assert median_seconds(entries) == 0.09
assert percentile_seconds(entries, 0.5) == 0.09
def test_an_even_split_takes_the_lower_middle_sample_as_locust_itself_does(self) -> None:
entry = _entry(num_requests=4, response_times={10: 2, 90: 2})
assert median_seconds([entry]) == 0.01
assert percentile_seconds([entry], 0.5) == 0.01
def test_no_samples_reports_zero_rather_than_dividing_by_an_empty_histogram(self) -> None:
assert median_seconds([]) == 0.0
assert percentile_seconds([], 0.5) == 0.0
class TestAggregate:
@ -84,9 +100,20 @@ class TestAggregate:
result = aggregate_stats([entry], (), ())
assert result.requests_per_second == 3.0
assert result.median_response_seconds == 0.057
assert result.p50_seconds == 0.057
assert result.p99_seconds == 0.057
assert result.failure_ratio == 0.0
def test_tail_percentiles_come_from_the_slow_end_of_the_histogram(self) -> None:
entry = _entry(num_requests=100, response_times={20: 89, 500: 10, 3000: 1})
result = aggregate_stats([entry], (), ())
assert result.p50_seconds == 0.02
assert result.p90_seconds == 0.5
assert result.p99_seconds == 0.5
assert result.latency_summary() == "p50 0.020s, p90 0.500s, p99 0.500s"
def test_throughput_spans_from_the_earliest_start_when_locust_reports_several_entries(self) -> None:
entries = [
_entry(num_requests=60, start_time=1000.0, last_request_timestamp=1030.0),
@ -103,6 +130,49 @@ class TestAggregate:
assert result.requests == 0
assert result.requests_per_second == 0.0
assert result.failure_ratio == 1.0
assert result.endpoints == ()
class TestPerEndpoint:
def test_each_route_keeps_its_own_requests_failures_and_median(self) -> None:
entries: Final = (
_entry(name="/chat/completions", num_requests=100, response_times={20: 100}),
_entry(name="/v1/messages", num_requests=40, num_failures=3, response_times={900: 40}),
)
result: Final = aggregate_stats(entries, (), ())
assert tuple((one.name, one.requests, one.failures, one.p50_seconds) for one in result.endpoints) == (
("/chat/completions", 100, 0, 0.02),
("/v1/messages", 40, 3, 0.9),
)
def test_several_stats_entries_for_one_route_fold_into_a_single_row(self) -> None:
entries: Final = (
_entry(name="/v1/messages", num_requests=10, response_times={30: 10}),
_entry(name="/v1/messages", num_requests=30, num_failures=1, response_times={30: 30}),
)
result: Final = aggregate_stats(entries, (), ())
assert tuple((one.name, one.requests, one.failures) for one in result.endpoints) == (("/v1/messages", 40, 1),)
def test_a_route_that_never_ran_is_absent_so_a_one_sided_run_cannot_pass_unnoticed(self) -> None:
result: Final = aggregate_stats((_entry(name="/chat/completions", num_requests=10),), (), ())
assert tuple(one.name for one in result.endpoints) == ("/chat/completions",)
def test_the_summary_names_every_route_with_its_counts(self) -> None:
entries: Final = (
_entry(name="/chat/completions", num_requests=2, response_times={20: 2}),
_entry(name="/v1/messages", num_requests=1, num_failures=1, response_times={500: 1}),
)
result: Final = aggregate_stats(entries, (), ())
assert result.endpoint_summary() == (
"/chat/completions 2 requests, 0 failures, p50 0.020s, /v1/messages 1 requests, 1 failures, p50 0.500s"
)
class TestErrorBreakdown:
@ -133,8 +203,7 @@ class TestErrorBreakdown:
def test_diagnosis_caps_the_list_and_says_how_many_it_left_out(self) -> None:
result = _result(
errors=tuple(
LoadError(name="/chat/completions", error=f"error-{index}", occurrences=index)
for index in range(1, 9)
LoadError(name="/chat/completions", error=f"error-{index}", occurrences=index) for index in range(1, 9)
)
)

105
tests/e2e/load/test_phase_budget.py Normal file
View file

@ -0,0 +1,105 @@
from __future__ import annotations
from typing import Final
from phase_budget import AbsoluteBudget, RatioBudget, violations
def _budget(*, baseline: float, degraded: float, ceiling: float = 2.0) -> RatioBudget:
return RatioBudget(
name="p99 RSS", baseline=baseline, degraded=degraded, ratio_ceiling=ceiling, unit=" MB", decimals=0
)
class TestRatioBudget:
def test_growth_within_the_ceiling_is_not_a_violation(self) -> None:
assert _budget(baseline=100, degraded=199).violation() is None
def test_growth_exactly_at_the_ceiling_is_allowed(self) -> None:
assert _budget(baseline=100, degraded=200).violation() is None
def test_growth_past_the_ceiling_reports_both_values_and_the_ratio(self) -> None:
violation: Final = _budget(baseline=100, degraded=250).violation()
assert violation is not None
assert "100 MB" in violation
assert "250 MB" in violation
assert "2.5x" in violation
assert "2.0x allowed" in violation
def test_shrinking_is_never_a_violation(self) -> None:
assert _budget(baseline=100, degraded=10).violation() is None
def test_a_missing_baseline_is_a_violation_rather_than_a_silent_pass(self) -> None:
# The trap this guards: 0 as a baseline would make every ratio a division by zero, and
# treating it as "no growth" would pass a run that measured nothing at all.
violation: Final = _budget(baseline=0, degraded=4000).violation()
assert violation is not None
assert "nothing to compare" in violation
def test_the_unit_and_decimals_carry_into_the_message(self) -> None:
violation: Final = RatioBudget(
name="p99 latency", baseline=0.16, degraded=9.5, ratio_ceiling=8.0, unit="s", decimals=3
).violation()
assert violation is not None
assert "0.160s" in violation
assert "9.500s" in violation
class TestAbsoluteBudget:
def test_a_value_under_the_ceiling_is_not_a_violation(self) -> None:
assert AbsoluteBudget(name="p99 latency", measured=1.2, ceiling=5.0, unit="s", decimals=3).violation() is None
def test_a_value_exactly_at_the_ceiling_is_allowed(self) -> None:
assert AbsoluteBudget(name="p99 latency", measured=5.0, ceiling=5.0, unit="s", decimals=3).violation() is None
def test_a_value_past_the_ceiling_reports_the_measurement_and_the_ceiling(self) -> None:
violation: Final = AbsoluteBudget(
name="p99 latency", measured=9.5, ceiling=5.0, unit="s", decimals=3
).violation()
assert violation is not None
assert "9.500s" in violation
assert "5.000s allowed" in violation
def test_a_flat_ceiling_fails_a_degraded_phase_that_is_cheaper_than_its_baseline(self) -> None:
# The whole reason this shape exists: once the breaker opens, requests skip Redis instead
# of waiting on its socket timeout, so the chaos phase can measure faster than the healthy
# one. A ratio against that baseline passes; the user still waited 9.5s.
assert _budget(baseline=20.0, degraded=9.5, ceiling=2.0).violation() is None
assert AbsoluteBudget(name="p99 latency", measured=9.5, ceiling=5.0, unit="s").violation() is not None
def test_a_zero_measurement_is_not_a_violation(self) -> None:
assert AbsoluteBudget(name="log bytes per request", measured=0, ceiling=12_000, unit=" B").violation() is None
class TestViolations:
def test_every_blown_budget_is_reported_not_just_the_first(self) -> None:
blown: Final = violations(
(
_budget(baseline=100, degraded=500),
_budget(baseline=100, degraded=120),
RatioBudget(name="CPU per request", baseline=10, degraded=90, ratio_ceiling=6.0, unit=" ms"),
)
)
assert len(blown) == 2
assert blown[0].startswith("p99 RSS")
assert blown[1].startswith("CPU per request")
def test_both_budget_shapes_report_together(self) -> None:
blown: Final = violations(
(
_budget(baseline=100, degraded=500),
AbsoluteBudget(name="p99 latency", measured=9.5, ceiling=5.0, unit="s", decimals=3),
)
)
assert len(blown) == 2
assert blown[0].startswith("p99 RSS")
assert blown[1].startswith("p99 latency")
def test_a_run_inside_every_budget_reports_nothing(self) -> None:
assert violations((_budget(baseline=100, degraded=150),)) == ()

71
tests/e2e/load/test_proxy_usage.py Normal file
View file

@ -0,0 +1,71 @@
from __future__ import annotations
from typing import Final
from proxy_usage import UsageSample, UsageWindow
_MB: Final = 2**20
def _window(*points: tuple[float, int, float]) -> UsageWindow:
return UsageWindow(
samples=tuple(
UsageSample(elapsed_seconds=elapsed, rss_bytes=rss, cpu_seconds=cpu) for elapsed, rss, cpu in points
)
)
class TestRssPercentiles:
def test_the_tail_percentiles_reach_the_peak_the_median_hides(self) -> None:
# 100 one-second samples: 89 flat, 10 elevated, 1 spike. The median stays flat, p90 sees the
# elevated plateau, and only the max reaches the spike.
window: Final = _window(
*((float(i), 100 * _MB, float(i)) for i in range(89)),
*((float(89 + i), 300 * _MB, float(89 + i)) for i in range(10)),
(99.0, 900 * _MB, 99.0),
)
assert window.rss_percentile(0.5) == 100 * _MB
assert window.rss_percentile(0.9) == 300 * _MB
assert window.rss_percentile(0.99) == 300 * _MB
assert window.rss_percentile(1.0) == 900 * _MB
def test_an_empty_window_reports_zero_rather_than_indexing_nothing(self) -> None:
assert _window().rss_percentile(0.5) == 0
class TestCpuUtilization:
def test_utilization_is_the_counter_delta_over_the_interval_not_the_counter_itself(self) -> None:
# The counter climbs 0.5 CPU seconds per second, then 4.0 per second: half a core, then four.
window: Final = _window((0.0, _MB, 0.0), (1.0, _MB, 0.5), (2.0, _MB, 1.0), (3.0, _MB, 5.0))
p50, p90, p99 = window.cpu_utilization_percentiles()
assert (p50, p90, p99) == (0.5, 4.0, 4.0)
assert window.cpu_seconds_consumed() == 5.0
def test_a_single_sample_has_no_interval_and_reports_zero(self) -> None:
window: Final = _window((0.0, _MB, 3.0))
assert window.cpu_utilization_percentiles() == (0.0, 0.0, 0.0)
assert window.cpu_seconds_consumed() == 0.0
def test_cost_per_request_separates_runs_that_cores_busy_reports_identically(self) -> None:
# Both windows pin 4 cores for 10 seconds, so utilization cannot tell them apart. The
# second one served a tenth of the traffic for the same CPU, which is the regression shape.
window: Final = _window(*((float(i), _MB, 4.0 * i) for i in range(11)))
assert window.cpu_utilization_percentiles()[0] == 4.0
assert window.cpu_seconds_per_request(4000) == 0.01
assert window.cpu_seconds_per_request(400) == 0.1
def test_no_requests_reports_zero_cost_rather_than_dividing_by_zero(self) -> None:
assert _window((0.0, _MB, 0.0), (1.0, _MB, 1.0)).cpu_seconds_per_request(0) == 0.0
def test_summary_reports_every_percentile_in_human_units(self) -> None:
window: Final = _window((0.0, 200 * _MB, 0.0), (1.0, 200 * _MB, 1.5), (2.0, 200 * _MB, 3.0))
assert window.summary() == (
"RSS p50 200 MB, p90 200 MB, p99 200 MB; "
"CPU cores busy p50 1.50, p90 1.50, p99 1.50; 3.0 CPU seconds consumed"
)

454
tests/e2e/load/test_redis_chaos_e2e.py Normal file
View file

@ -0,0 +1,454 @@
"""Live e2e: the proxy under load keeps serving every request while Redis is down entirely.
Runs against a proxy booted from tests/e2e/gateway/redis_chaos_ci_config.yml, which points
cache_params at a real Redis with litellm's default socket_timeout. That one client backs all
three Redis touchpoints on the request path: the virtual-key auth cache, the response cache,
and the cross-pod spend counter the cost-tracking callback awaits.
The load runs in two phases against one model group of three mock deployments. The two at
order 1 raise InternalServerError and the one at order 2 serves, so every request burns its
retries on the failing pair (a 500 is retryable, so retries keep re-picking inside the lowest
order) and the router's order-based fallback then re-targets order 2. Every request is expected
to succeed, and each one carries retry breadcrumbs into cost tracking.
Traffic is split round robin between /chat/completions and /v1/messages, one endpoint per
simulated user: the Redis touchpoints and the cost-tracking callback are shared by both, but
the Anthropic Messages route reaches them through its own request path, so a regression that
only shows up there would not surface from chat completions alone.
Phase A is a baseline with Redis healthy; phase B holds Redis in CLIENT PAUSE ALL for the
length of the phase, simulating Redis being down outright rather than merely slow to write.
Every touchpoint times out: the auth cache read falls back to Postgres, the response cache
read and write both fail, and the spend counter increment times out and the callback
stringifies the request metadata, breadcrumbs included, into a failed-tracking alert. On
v1.100.0 that string doubled per request until the worker hung (LIT-6780), which is what the
per-phase RSS, CPU, and log-bytes budgets are here to catch.
Needs the proxy on the same host, since RSS and CPU come from psutil on its process tree:
a multi-worker proxy serves /metrics from the prometheus multiprocess collector, which drops
the process collector's memory and CPU series. Log bytes are read from the file the proxy's
stdout/stderr was redirected to, so the same host requirement covers that too. Deselected
unless E2E_REDIS_CHAOS is set.
"""
from __future__ import annotations
import os
import re
import time
from collections.abc import Iterator
from dataclasses import dataclass
from itertools import pairwise
from pathlib import Path
from typing import Final
import pytest
import redis
from e2e_config import PROXY_BASE_URL, unique_marker
from e2e_http import NoBody
from lifecycle import ResourceManager
from load_client import LoadClient
from locust_load import LoadResult, run_gateway_load
from models import KeyGenerateBody, LiteLLMParamsBody
from phase_budget import AbsoluteBudget, Budget, RatioBudget, violations
from proxy_client import ProxyClient
from proxy_usage import ProxyUsageSampler, UsageWindow
pytestmark: Final = pytest.mark.e2e
MODEL_GROUP: Final = f"redis-chaos-fable-{unique_marker()}"
MOCK_MODEL: Final = "anthropic/claude-fable-5-1"
FAILING_DEPLOYMENTS: Final = 2
SERVING_DEPLOYMENTS: Final = 1
FAILING_ORDER: Final = 1
SERVING_ORDER: Final = 2
KEY_POOL_SIZE: Final = 8
LOAD_ENDPOINTS: Final = ("/chat/completions", "/v1/messages")
LOCUST_USERS: Final = 50
LOCUST_SPAWN_RATE: Final = 50.0
BASELINE_SECONDS: Final = 60.0
CHAOS_SECONDS: Final = 90.0
REDIS_PAUSE_MS: Final = int(CHAOS_SECONDS * 1000)
# RSS and CPU are budgeted as a multiple of the same metric in the baseline phase, because both
# are machine-shaped: RSS scales with worker count and CPU with core count, so a number
# calibrated on one runner means nothing on another. RSS moved 0.91x-1.40x across three otherwise
# identical local runs, so it stays loose; CPU per request held steady at 1.33x-1.36x across the
# same runs, so it sits close to what is actually measured. That makes CPU the likeliest of these
# to flake first on a runner whose core count shifts how much of baseline CPU is fixed per-request
# work: loosen it rather than widening the others if a CI run trips it without a real cause.
CHAOS_RSS_RATIO_CEILING: Final = 2.0
CHAOS_CPU_PER_REQUEST_RATIO_CEILING: Final = 2.0
# Latency and log volume get flat ceilings instead, because a ratio cannot bound either one. Once
# the breaker opens, a request skips Redis rather than waiting on its socket timeout, so the chaos
# phase can come in faster than baseline (local runs measured p90 at 0.61x) and a ratio passes on a
# phase that was never slow. What a user actually cares about is the wall-clock number, which these
# hold directly. Calibrated from local runs whose worst chaos phase was p50 0.19s, p90 0.23s, p99
# 0.69s and 3.5 KB of log per request, with several times that left as slack for a shared CI runner.
CHAOS_P50_LATENCY_CEILING_SECONDS: Final = 1.0
CHAOS_P90_LATENCY_CEILING_SECONDS: Final = 2.0
CHAOS_P99_LATENCY_CEILING_SECONDS: Final = 3.0
CHAOS_LOG_BYTES_PER_REQUEST_CEILING: Final = 10_000.0
DRAIN_TIMEOUT_SECONDS: Final = 30.0
DRAIN_POLL_SECONDS: Final = 1.0
TIMEOUT_FAILURES_RE: Final = re.compile(
r'^litellm_redis_circuit_breaker_failures_total\{failure_class="timeout"\} ([0-9.e+]+)$', re.M
)
# The state gauge carries a pid label under the multiprocess collector, one series per worker,
# so this matches any label order rather than a bare {state="open"} that never appears.
BREAKER_OPEN_RE: Final = re.compile(
r'^litellm_redis_circuit_breaker_state\{[^}]*state="open"[^}]*\} ([0-9.e+]+)$', re.M
)
BREAKER_TRANSITIONS_RE: Final = re.compile(
r'^litellm_redis_circuit_breaker_transitions_total\{state="[a-z_]+"\} ([0-9.e+]+)$', re.M
)
def _deployment_metric_re(name: str, model_ids: tuple[str, ...]) -> re.Pattern[str]:
"""A per-deployment counter, narrowed to the deployments one run registered, so traffic
anything else sends the same proxy during the run cannot pad the retry count."""
ids: Final = "|".join(re.escape(model_id) for model_id in model_ids)
return re.compile(rf'^litellm_{name}\{{[^}}]*model_id="(?:{ids})"[^}}]*\}} ([0-9.e+]+)$', re.M)
@dataclass(frozen=True, slots=True)
class Phase:
"""One load phase's traffic and what the proxy's process tree did during it."""
name: str
load: LoadResult
usage: UsageWindow
redis_timeouts: float
log_bytes: int
@property
def timeouts_per_request(self) -> float:
return self.redis_timeouts / self.load.requests if self.load.requests else 0.0
@property
def cpu_seconds_per_request(self) -> float:
return self.usage.cpu_seconds_per_request(self.load.requests)
@property
def log_bytes_per_request(self) -> float:
return self.log_bytes / self.load.requests if self.load.requests else 0.0
def report(self) -> str:
return (
f"{self.name}: {self.load.requests} requests, {self.load.failures} failures, "
f"{self.load.requests_per_second:.0f} rps, {self.load.latency_summary()}; {self.usage.summary()}; "
f"{self.cpu_seconds_per_request * 1000:.1f} ms CPU per request; "
f"{self.log_bytes_per_request:.0f} log bytes per request; "
f"{self.timeouts_per_request:.2f} Redis timeouts per request; "
f"by endpoint: {self.load.endpoint_summary()}"
)
def _failing_params() -> LiteLLMParamsBody:
return LiteLLMParamsBody(
model=MOCK_MODEL,
api_key="sk-redis-chaos-not-used",
mock_response="litellm.InternalServerError",
order=FAILING_ORDER,
)
def _serving_params() -> LiteLLMParamsBody:
return LiteLLMParamsBody(
model=MOCK_MODEL,
api_key="sk-redis-chaos-not-used",
mock_response="redis chaos ok",
order=SERVING_ORDER,
)
@pytest.fixture
def proxy_pid() -> int:
"""The proxy's PID, which the workflow exports after starting it.
Required rather than discovered: picking a process out of the table by name would be
ambiguous on a developer machine running more than one proxy.
"""
pid: Final = os.environ.get("E2E_PROXY_PID")
assert pid and pid.isdigit(), (
"E2E_PROXY_PID must hold the PID of the proxy under test; RSS and CPU are read from "
"its process tree because a multi-worker proxy does not report them on /metrics"
)
return int(pid)
@pytest.fixture
def proxy_log() -> Path:
"""Path to the proxy's stdout/stderr log, which the workflow captures to a file.
Required rather than discovered for the same reason as proxy_pid: a developer machine may
have more than one proxy log around.
"""
path: Final = os.environ.get("E2E_PROXY_LOG")
assert path, "E2E_PROXY_LOG must hold the path the proxy's stdout/stderr was redirected to"
return Path(path)
def _log_bytes(path: Path) -> int:
return path.stat().st_size
@pytest.fixture
def redis_control() -> Iterator[redis.Redis[bytes]]:
"""A control connection to the proxy's Redis, which unpauses it in teardown as a safety net.
CLIENT PAUSE ALL freezes every connection including this one, so REDIS_PAUSE_MS is sized
to the chaos phase: by the time teardown runs, the pause has
already lapsed on its own and CLIENT UNPAUSE here returns immediately. It only actually
waits out a lapsed pause if the chaos phase itself overran that duration.
"""
host: Final = os.environ.get("REDIS_HOST")
port: Final = os.environ.get("REDIS_PORT")
assert host and port, "REDIS_HOST and REDIS_PORT must name the Redis the proxy under test uses"
control: Final = redis.Redis(host=host, port=int(port), socket_timeout=5)
try:
yield control
finally:
control.client_unpause() # pyright: ignore[reportUnknownMemberType] # redis-py stubs return Any
control.close()
def _scrape(proxy: ProxyClient) -> str:
"""One /metrics body, read once per checkpoint so every counter comes from the same instant."""
scrape: Final = proxy.probe("/metrics", params=NoBody())
assert scrape.status_code == 200, (
f"/metrics did not answer ({scrape.status_code}: {scrape.body[:200]}), so no counter can be read; "
f"a silent 0 here would turn every before-and-after difference negative"
)
return scrape.body
def _metric(scrape: str, pattern: re.Pattern[str]) -> float:
return sum(float(match.group(1)) for match in pattern.finditer(scrape))
def _scrape_after_drain(proxy: ProxyClient, pattern: re.Pattern[str]) -> str:
"""A /metrics body taken once `pattern`'s count has stopped moving.
`set_llm_deployment_failure_metrics` runs from the async logging callback queue, so a load
generator that just stopped sending traffic can still have thousands of failure increments
in flight, and a scrape taken the instant load stops undercounts them. Settling on the
counter rather than sleeping a fixed duration keeps the wait proportional to how backed up
the queue actually is.
"""
deadline: Final = time.monotonic() + DRAIN_TIMEOUT_SECONDS
def scrapes() -> Iterator[str]:
yield _scrape(proxy)
while time.monotonic() < deadline:
time.sleep(DRAIN_POLL_SECONDS)
yield _scrape(proxy)
settled: Final = next(
(later for earlier, later in pairwise(scrapes()) if _metric(earlier, pattern) == _metric(later, pattern)),
None,
)
return settled if settled is not None else _scrape(proxy)
def _register_deployments(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, ...]:
"""The model ids this run registered, which scope its per-deployment metric reads."""
params: Final = (
*(_failing_params() for _ in range(FAILING_DEPLOYMENTS)),
*(_serving_params() for _ in range(SERVING_DEPLOYMENTS)),
)
model_ids: Final = tuple(proxy.create_model(MODEL_GROUP, one) for one in params)
for model_id in model_ids:
resources.defer(lambda doomed=model_id: proxy.delete_model(doomed))
return model_ids
def _generate_key_pool(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, ...]:
"""A pool of virtual keys so auth and budget lookups are not one permanently warm
cache entry; each locust user picks one, so Redis auth reads actually happen."""
keys: Final = tuple(
proxy.generate_key(
KeyGenerateBody(models=[MODEL_GROUP], key_alias=f"e2e-redis-chaos-{unique_marker()}-{index}")
)
for index in range(KEY_POOL_SIZE)
)
for key in keys:
resources.defer(lambda doomed=key: proxy.delete_key(doomed))
return keys
def _drive(keys: tuple[str, ...], seconds: float) -> LoadResult:
return run_gateway_load(
base_url=PROXY_BASE_URL,
api_keys=keys,
model=MODEL_GROUP,
endpoints=LOAD_ENDPOINTS,
users=LOCUST_USERS,
spawn_rate=LOCUST_SPAWN_RATE,
duration_seconds=seconds,
)
def _latency_budget(percentile: str, measured: float, ceiling: float) -> Budget:
return AbsoluteBudget(name=f"{percentile} latency", measured=measured, ceiling=ceiling, unit="s", decimals=3)
def _rss_budget(percentile: str, baseline: UsageWindow, degraded: UsageWindow, fraction: float) -> Budget:
return RatioBudget(
name=f"{percentile} RSS",
baseline=baseline.rss_percentile(fraction) / 2**20,
degraded=degraded.rss_percentile(fraction) / 2**20,
ratio_ceiling=CHAOS_RSS_RATIO_CEILING,
unit=" MB",
decimals=0,
)
def _chaos_budgets(baseline: Phase, chaos: Phase) -> tuple[Budget, ...]:
"""What a Redis outage is allowed to cost.
Every request still succeeding is the headline assertion, but a proxy can answer every
request while leaking: the v1.100.0 regression (LIT-6780) served traffic the whole way up
to a 61 GB worker. These bound the cost of serving it. RSS and CPU are bounded against the
same run's healthy phase, latency and log bytes against a flat ceiling; see phase_budget
for why the two kinds of metric cannot share one shape.
Latency and RSS are budgeted at p50, p90 and p99 so a regression that only shows up in the
tail (or only in the median) cannot hide behind the other. RSS gets the tightest bound: the
failure path has no business allocating more per request. CPU and log bytes are each budgeted
once, as an amount per request rather than per percentile: cores-busy saturates at the worker
count under load, so its percentiles read the same whether a request costs 10 ms of CPU or
40, and cannot budget anything; per-request is the figure that actually moves. Log bytes
isolates the cost of the failed-tracking alert's own noisy error handling from the CPU it
burns doing useful retry work, since the two would otherwise be indistinguishable in one
CPU number.
"""
return (
_latency_budget("p50", chaos.load.p50_seconds, CHAOS_P50_LATENCY_CEILING_SECONDS),
_latency_budget("p90", chaos.load.p90_seconds, CHAOS_P90_LATENCY_CEILING_SECONDS),
_latency_budget("p99", chaos.load.p99_seconds, CHAOS_P99_LATENCY_CEILING_SECONDS),
_rss_budget("p50", baseline.usage, chaos.usage, 0.5),
_rss_budget("p90", baseline.usage, chaos.usage, 0.9),
_rss_budget("p99", baseline.usage, chaos.usage, 0.99),
RatioBudget(
name="CPU per request",
baseline=baseline.cpu_seconds_per_request * 1000,
degraded=chaos.cpu_seconds_per_request * 1000,
ratio_ceiling=CHAOS_CPU_PER_REQUEST_RATIO_CEILING,
unit=" ms",
),
AbsoluteBudget(
name="log bytes per request",
measured=chaos.log_bytes_per_request,
ceiling=CHAOS_LOG_BYTES_PER_REQUEST_CEILING,
unit=" B",
decimals=0,
),
)
@pytest.mark.redis_chaos
class TestRedisChaos:
@pytest.mark.covers(
"reliability.circuit_breaker.redis_timeout.stays_responsive",
exercised_on=("chat_completions", "messages"),
)
def test_load_survives_redis_being_down(
self,
client: LoadClient,
resources: ResourceManager,
proxy_pid: int,
proxy_log: Path,
redis_control: redis.Redis[bytes],
) -> None:
proxy: Final = client.proxy
model_ids: Final = _register_deployments(proxy, resources)
keys: Final = _generate_key_pool(proxy, resources)
retries_re: Final = _deployment_metric_re("deployment_failure_responses_total", model_ids)
cooldown_re: Final = _deployment_metric_re("deployment_cooled_down_total", model_ids)
at_start: Final = _scrape(proxy)
log_at_start: Final = _log_bytes(proxy_log)
with ProxyUsageSampler(proxy_pid) as sampler:
baseline_load: Final = _drive(keys, BASELINE_SECONDS)
baseline_usage: Final = sampler.split()
after_baseline: Final = _scrape(proxy)
log_after_baseline: Final = _log_bytes(proxy_log)
redis_control.client_pause(REDIS_PAUSE_MS, all=True) # pyright: ignore[reportUnknownMemberType] # redis-py stubs return Any
chaos_load: Final = _drive(keys, CHAOS_SECONDS)
chaos_usage: Final = sampler.split()
at_end: Final = _scrape_after_drain(proxy, retries_re)
log_at_end: Final = _log_bytes(proxy_log)
baseline: Final = Phase(
name="baseline",
load=baseline_load,
usage=baseline_usage,
redis_timeouts=_metric(after_baseline, TIMEOUT_FAILURES_RE) - _metric(at_start, TIMEOUT_FAILURES_RE),
log_bytes=log_after_baseline - log_at_start,
)
chaos: Final = Phase(
name="chaos",
load=chaos_load,
usage=chaos_usage,
redis_timeouts=_metric(at_end, TIMEOUT_FAILURES_RE) - _metric(after_baseline, TIMEOUT_FAILURES_RE),
log_bytes=log_at_end - log_after_baseline,
)
report: Final = f"{baseline.report()} | {chaos.report()}"
for phase in (baseline, chaos):
assert phase.load.requests > 0, (
f"{phase.name} drove no traffic at all, so it proved nothing: {phase.load.diagnosis()}. {report}"
)
assert frozenset(endpoint.name for endpoint in phase.load.endpoints) == frozenset(LOAD_ENDPOINTS), (
f"{phase.name} drove {tuple(endpoint.name for endpoint in phase.load.endpoints)} rather than every "
f"endpoint in {LOAD_ENDPOINTS}; the round robin hands one endpoint to each simulated user, so a "
f"missing one means a route never ran and its request path was never exercised. {report}"
)
assert phase.load.failures == 0, (
f"{phase.name} had {phase.load.failures} of {phase.load.requests} requests fail. Every request "
f"must succeed: the failing deployments sit at order {FAILING_ORDER} and the serving one at order "
f"{SERVING_ORDER}, so once the retries on order {FAILING_ORDER} are spent the order-based fallback "
f"lands on the serving deployment. Failures mean it was cooled down, the fallback did not run, or "
f"a Redis failure reached the response path. {phase.load.diagnosis()}. {report}"
)
cooldowns: Final = _metric(at_end, cooldown_re) - _metric(at_start, cooldown_re)
assert cooldowns == 0, (
f"{cooldowns:.0f} deployments were cooled down during the run; the failing deployments are supposed "
f"to stay in rotation so every request keeps exercising the retry path. {report}"
)
retries: Final = _metric(at_end, retries_re) - _metric(at_start, retries_re)
assert retries >= baseline.load.requests + chaos.load.requests, (
f"only {retries:.0f} deployment failures were counted across "
f"{baseline.load.requests + chaos.load.requests} requests; the mock deployments did not fail, so no "
f"request carried retry breadcrumbs into cost tracking and the regression path was never entered. "
f"{report}"
)
transitions: Final = _metric(at_end, BREAKER_TRANSITIONS_RE) - _metric(after_baseline, BREAKER_TRANSITIONS_RE)
breaker_open: Final = _metric(at_end, BREAKER_OPEN_RE) >= 1
assert transitions >= 1 or breaker_open, (
f"pausing Redis produced no circuit breaker state transitions and it ended closed; nothing on the "
f"request path ever saw Redis fail, so this run proved nothing. {report}"
)
blown: Final = violations(_chaos_budgets(baseline, chaos))
assert not blown, (
f"pausing Redis cost the proxy more than a Redis outage is allowed to: {'; '.join(blown)}. {report}"
)
rows: Final = proxy.poll_logs_for_key(keys[0], min_rows=1)
assert rows, (
f"no spend rows landed for the first key in the pool; a Redis outage must not cost the proxy its "
f"spend logs, which are written to Postgres through a queue rather than through Redis. {report}"
)
print(f"\nredis chaos load: {report}") # noqa: T201 # the numbers this test exists to report, read off the CI log

3
tests/e2e/models.py
View file

@ -922,10 +922,11 @@ class LiteLLMParamsBody(BaseModel):
auto_router_default_model: str | None = None
auto_router_embedding_model: str | None = None
tags: list[str] | None = None
mock_response: str | None = None
mock_response: str | list[float] | None = None
timeout: float | None = None
tpm: int | None = None
weight: int | None = None
order: int | None = None
ModelMode = Literal["batch", "realtime", "image_generation"]

1
tests/e2e/pytest.ini
View file

@ -9,3 +9,4 @@ markers =
load: heavy throughput/load test; collected last so it never perturbs latency-sensitive suites
weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set
managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set
redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set

9
tests/e2e/router/conftest.py
View file

@ -13,10 +13,7 @@ from __future__ import annotations
from collections.abc import Iterator
import pytest
from requests import RequestException
from complexity_router_client import ComplexityRouterClient, build_client
from proxy_client import ProxyClient
from e2e_http import NoBody, Success
from lifecycle import ResourceManager
from models import (
@ -26,6 +23,8 @@ from models import (
LiteLLMParamsBody,
ModelsListResponse,
)
from proxy_client import ProxyClient
from requests import RequestException
ROUTER_MODEL = "complexity-smart-router"
ROUTER_PARAMS = LiteLLMParamsBody(
@ -120,8 +119,6 @@ def _ensure_complexity_smart_router( # pyright: ignore[reportUnusedFunction] #
@pytest.fixture
def complexity_key(resources: ResourceManager, client: ComplexityRouterClient) -> str:
"""Per-test key allowed to call the complexity router and its tier backends."""
key = client.proxy.generate_key(
KeyGenerateBody(models=ROUTER_KEY_MODELS, user_id="e2e-complexity-router")
)
key = client.proxy.generate_key(KeyGenerateBody(models=ROUTER_KEY_MODELS, user_id="e2e-complexity-router"))
resources.defer(lambda: client.proxy.delete_key(key))
return key

4
uv.lock generated
View file

@ -10,7 +10,7 @@ resolution-markers = [
]
[options]
exclude-newer = "2026-09-07T23:09:03.362777Z"
exclude-newer = "2026-09-08T03:56:24.358378Z"
exclude-newer-span = "P3D"
[manifest]
@ -4559,6 +4559,7 @@ e2e-dev = [
{ name = "locust" },
{ name = "mcp" },
{ name = "playwright" },
{ name = "psutil" },
{ name = "websockets" },
]
healthcheck = [
@ -4747,6 +4748,7 @@ e2e-dev = [
{ name = "locust", specifier = "==2.45.0" },
{ name = "mcp", specifier = ">=1.28.1,<2.0" },
{ name = "playwright", specifier = "==1.61.0" },
{ name = "psutil", specifier = "==7.2.2" },
{ name = "websockets", specifier = ">=15.0.1,<16.0" },
]
healthcheck = [