From 058ff8c63c222d52392814b0fda84d5b030a84ec Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 9 Sep 2026 15:33:44 -0700 Subject: [PATCH 01/27] test(e2e): keep answering while every Redis command times out Add tests/e2e/router/test_redis_timeout_e2e.py against a proxy booted from tests/e2e/gateway/redis_timeout_ci_config.yml: a real Redis with socket_timeout 0.001, so every command times out and the circuit breaker opens, plus a primary deployment that always fails and falls back to a healthy one, so every request carries retry breadcrumbs into cost tracking. The test drives twenty chat requests through the proxy and asserts each answers within ten seconds, the last third is no slower than the first, /health/liveliness stays fast, and every request still reaches the spend log. Gate it behind the redis_timeout marker and E2E_REDIS_TIMEOUT, exclude it from the per-PR e2e-changed selector, register the reliability.circuit_breaker.redis_timeout.stays_responsive cell, and run it as its own job in the weekly load anomaly workflow with a Postgres and Valkey service. Against a v1.100.0 proxy the run wedges the worker: requests time out and liveliness stops answering (LIT-6780). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014ZDULyJPp17ZFiJenRxs2T --- .github/e2e-stack/select_tests.py | 1 + .github/workflows/weekly_load_anomaly.yml | 84 +++++++++++++++++ tests/e2e/CLAUDE.md | 2 +- tests/e2e/CONTRIBUTING.md | 2 +- tests/e2e/conftest.py | 9 +- tests/e2e/coverage_registry/reliability.yaml | 1 + tests/e2e/e2e_config.py | 5 +- tests/e2e/gateway/redis_timeout_ci_config.yml | 29 ++++++ tests/e2e/pytest.ini | 1 + tests/e2e/router/conftest.py | 21 +++-- tests/e2e/router/test_redis_timeout_e2e.py | 93 +++++++++++++++++++ 11 files changed, 234 insertions(+), 14 deletions(-) create mode 100644 tests/e2e/gateway/redis_timeout_ci_config.yml create mode 100644 tests/e2e/router/test_redis_timeout_e2e.py diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py index a62358f81ff..10f26bd3b8a 100644 --- a/.github/e2e-stack/select_tests.py +++ b/.github/e2e-stack/select_tests.py @@ -8,6 +8,7 @@ UNSUPPORTED: Final = re.compile( r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$" r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$" r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$" + r"|^tests/e2e/router/test_redis_timeout_e2e\.py$" ) HARNESS: Final = re.compile( r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" diff --git a/.github/workflows/weekly_load_anomaly.yml b/.github/workflows/weekly_load_anomaly.yml index 3e1fca89645..6e564cf7e78 100644 --- a/.github/workflows/weekly_load_anomaly.yml +++ b/.github/workflows/weekly_load_anomaly.yml @@ -83,3 +83,87 @@ jobs: - name: Show proxy log on failure if: failure() run: tail -n 300 proxy.log + + redis-timeout-e2e: + if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + postgres: + image: postgres:16.6 + env: + POSTGRES_USER: llmproxy + POSTGRES_PASSWORD: dbpassword9090 + POSTGRES_DB: litellm + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U llmproxy" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + valkey: + image: valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa + ports: + - 6379:6379 + options: >- + --health-cmd "valkey-cli ping" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm + LITELLM_MASTER_KEY: sk-redis-timeout-e2e + LITELLM_LOG: WARNING + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build + + - name: Install dependencies + run: | + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy + + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + + - name: Generate Prisma client + run: | + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + - name: Start the proxy with a Redis that times out on every command + run: | + nohup uv run --no-sync litellm --config tests/e2e/gateway/redis_timeout_ci_config.yml --port 4000 > proxy.log 2>&1 & + for _ in $(seq 1 90); do + if curl -fs http://localhost:4000/health/liveliness > /dev/null; then + exit 0 + fi + sleep 2 + done + echo "proxy never became live" + tail -n 100 proxy.log + exit 1 + + - name: Run the Redis timeout e2e test + env: + E2E_REDIS_TIMEOUT: "1" + LITELLM_PROXY_URL: http://localhost:4000 + run: | + uv run --no-sync pytest tests/e2e/router/test_redis_timeout_e2e.py -v --tb=short -rA + + - name: Show proxy log on failure + if: failure() + run: tail -n 300 proxy.log diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 89c04208d65..53342f66b67 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -17,7 +17,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server (see "MCP suite: real Datadog only" below); plus the gateway-managed OAuth (authorization_code) path exercised through `/chat/completions`, the one behavior Datadog's static-header auth cannot reach, seeding the per-user upstream token via the interactive authorize dance driven with the mcp SDK's own OAuth client (headless-browser consent from a saved session) and asserting the completion lists and executes the server's tools with the stored per-user token - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection -- `router/` - routing and reliability behavior (fallbacks, cooldowns) +- `router/` - routing and reliability behavior (fallbacks, cooldowns). Also holds `test_redis_timeout_e2e.py`, which needs a proxy booted from `gateway/redis_timeout_ci_config.yml` (real Redis, `socket_timeout: 0.001`, so every command times out); it is marked `redis_timeout`, deselected unless `E2E_REDIS_TIMEOUT` is set, excluded from the per-PR check, and driven by `.github/workflows/weekly_load_anomaly.yml` - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What remains here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`) and markerless harness unit tests for the Locust/session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 1183096b81e..ccd27df6f46 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -65,7 +65,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, `guardrails/test_presidio_masking_e2e.py`, and `router/test_redis_timeout_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start, and the Redis timeout test needs a proxy whose Redis times out on every command (`gateway/redis_timeout_ci_config.yml`), which the weekly load workflow boots Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index e1b987cbfd9..ff3d2dcc6bd 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -20,16 +20,14 @@ from datetime import datetime, timezone import pytest import requests - from e2e_config import CONTROL_PLANE_BASE_URL, FIXTURE_DIR, FIXTURE_MODE_RAW, PROXY_BASE_URL from e2e_db import RESET_OPT_IN_ENV, reset_spend_logs, run_spend_log_cleanup from fixture_mode import fixture_mode_collection_error, fixture_report_lines -from provider_edge import replay_leftover_error from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager +from provider_edge import replay_leftover_error from proxy_client import ProxyClient, build_proxy_client - _E2E_TEST_RAN = pytest.StashKey[bool]() _CALL_PASSED = pytest.StashKey[bool]() @@ -60,6 +58,11 @@ def pytest_configure(config: pytest.Config) -> None: "markers", "managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set", ) + config.addinivalue_line( + "markers", + "redis_timeout: needs a proxy booted from gateway/redis_timeout_ci_config.yml whose Redis times out on every " + "command; deselected unless E2E_REDIS_TIMEOUT is set", + ) def pytest_sessionstart(session: pytest.Session) -> None: diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 6b69677d490..d62f0105931 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -31,6 +31,7 @@ - {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"} - {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix"} - {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"} +- {id: reliability.circuit_breaker.redis_timeout.stays_responsive, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis_timeout, assertions: [stays_responsive], exercised_on: [chat_completions], source: "litellm/proxy/hooks/proxy_track_cost_callback.py:386", fail_before_fix: proven, rationale: "With every Redis command timing out and every request retrying then falling back, per-request latency stays flat, liveliness stays fast, and spend rows still land; on v1.100.0 the failed-tracking alert body doubled per request until the worker OOMed (LIT-6780)"} - {id: reliability.timeout.request_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: request_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions, messages], source: "litellm/router.py:545-551", rationale: "Per-request timeout raises Timeout"} - {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"} - {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 691335ffdd5..6cb59ea4e90 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -13,7 +13,6 @@ from pathlib import Path from typing import Final from dotenv import load_dotenv - from fixture_mode import deterministic_marker, parse_fixture_mode from provider_edge import provider_edge_api_base @@ -144,6 +143,7 @@ LOAD_MIN_CONCURRENCY_EFFICIENCY = float(os.environ.get("E2E_LOAD_MIN_CONCURRENCY WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY" MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK" +REDIS_TIMEOUT_OPT_IN_ENV = "E2E_REDIS_TIMEOUT" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) @@ -181,8 +181,7 @@ def datadog_mcp_url(*, toolsets: str = "core") -> str: site = ( os.environ.get("DD_SITE", DD_SITE) or "datadoghq.com" ).strip().removeprefix("https://").removeprefix("http://").rstrip("/") - if site.startswith("app."): - site = site[len("app.") :] + site = site.removeprefix("app.") host = "mcp.datadoghq.com" if site in ("", "datadoghq.com") else f"mcp.{site}" base = f"https://{host}/v1/mcp" return f"{base}?toolsets={toolsets}" if toolsets else base diff --git a/tests/e2e/gateway/redis_timeout_ci_config.yml b/tests/e2e/gateway/redis_timeout_ci_config.yml new file mode 100644 index 00000000000..69ab2ee14dc --- /dev/null +++ b/tests/e2e/gateway/redis_timeout_ci_config.yml @@ -0,0 +1,29 @@ +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + store_model_in_db: true + +litellm_settings: + cache: true + cache_params: + type: redis + host: 127.0.0.1 + port: 6379 + socket_timeout: 0.001 + +router_settings: + num_retries: 1 + fallbacks: + - redis-timeout-primary: + - redis-timeout-backup + +model_list: + - model_name: redis-timeout-primary + litellm_params: + model: openai/gpt-5-mini + api_key: sk-redis-timeout-primary-not-used + mock_response: "litellm.InternalServerError" + - model_name: redis-timeout-backup + litellm_params: + model: openai/gpt-5-mini + api_key: sk-redis-timeout-backup-not-used + mock_response: "ok" diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index c3f8865f218..f69d5d058f3 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -9,3 +9,4 @@ markers = load: heavy throughput/load test; collected last so it never perturbs latency-sensitive suites weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set + redis_timeout: needs a proxy booted from gateway/redis_timeout_ci_config.yml whose Redis times out on every command; deselected unless E2E_REDIS_TIMEOUT is set diff --git a/tests/e2e/router/conftest.py b/tests/e2e/router/conftest.py index 98501f9bd7c..57fde729f41 100644 --- a/tests/e2e/router/conftest.py +++ b/tests/e2e/router/conftest.py @@ -10,13 +10,12 @@ proxy does not already list it (compose has it in static config; stage does not) from __future__ import annotations +import os from collections.abc import Iterator import pytest -from requests import RequestException - from complexity_router_client import ComplexityRouterClient, build_client -from proxy_client import ProxyClient +from e2e_config import REDIS_TIMEOUT_OPT_IN_ENV from e2e_http import NoBody, Success from lifecycle import ResourceManager from models import ( @@ -26,6 +25,8 @@ from models import ( LiteLLMParamsBody, ModelsListResponse, ) +from proxy_client import ProxyClient +from requests import RequestException ROUTER_MODEL = "complexity-smart-router" ROUTER_PARAMS = LiteLLMParamsBody( @@ -45,6 +46,16 @@ ROUTER_PARAMS = LiteLLMParamsBody( ROUTER_KEY_MODELS = [ROUTER_MODEL, "gpt-5.5", "claude-haiku-4-5"] +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + if os.environ.get(REDIS_TIMEOUT_OPT_IN_ENV): + return + deselected = [item for item in items if item.get_closest_marker("redis_timeout") is not None] + if not deselected: + return + config.hook.pytest_deselected(items=deselected) + items[:] = [item for item in items if item.get_closest_marker("redis_timeout") is None] + + @pytest.fixture(scope="session") def client(proxy: ProxyClient) -> ComplexityRouterClient: return build_client(proxy) @@ -120,8 +131,6 @@ def _ensure_complexity_smart_router( # pyright: ignore[reportUnusedFunction] # @pytest.fixture def complexity_key(resources: ResourceManager, client: ComplexityRouterClient) -> str: """Per-test key allowed to call the complexity router and its tier backends.""" - key = client.proxy.generate_key( - KeyGenerateBody(models=ROUTER_KEY_MODELS, user_id="e2e-complexity-router") - ) + key = client.proxy.generate_key(KeyGenerateBody(models=ROUTER_KEY_MODELS, user_id="e2e-complexity-router")) resources.defer(lambda: client.proxy.delete_key(key)) return key diff --git a/tests/e2e/router/test_redis_timeout_e2e.py b/tests/e2e/router/test_redis_timeout_e2e.py new file mode 100644 index 00000000000..eeb5224e051 --- /dev/null +++ b/tests/e2e/router/test_redis_timeout_e2e.py @@ -0,0 +1,93 @@ +"""Live e2e: the proxy keeps answering while every Redis command times out. + +Runs only against a proxy booted from tests/e2e/gateway/redis_timeout_ci_config.yml, which +points cache_params at a real Redis with socket_timeout 0.001 so every command times out and +the circuit breaker opens. Each request fails its primary deployment, retries, falls back to the +backup and succeeds, so it carries retry breadcrumbs; its cost tracking then fails on the spend +counter increment and stringifies the request metadata into a failed-tracking alert. On v1.100.0 +that string doubled per request until the worker hung (LIT-6780). Deselected unless +E2E_REDIS_TIMEOUT is set, since it needs that dedicated proxy. +""" + +from __future__ import annotations + +import time +from typing import Final + +import pytest +from complexity_router_client import ComplexityRouterClient +from e2e_config import unique_marker +from e2e_http import NoBody, Success +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, ChatResponse, KeyGenerateBody + +pytestmark = [pytest.mark.e2e, pytest.mark.redis_timeout] + +PRIMARY_MODEL: Final = "redis-timeout-primary" +BACKUP_MODEL: Final = "redis-timeout-backup" +REQUESTS: Final = 20 +MAX_SECONDS_PER_REQUEST: Final = 10.0 +MAX_LATENCY_GROWTH_RATIO: Final = 3.0 +MAX_LIVELINESS_SECONDS: Final = 2.0 + + +class TestRedisTimeout: + @pytest.mark.covers( + "reliability.circuit_breaker.redis_timeout.stays_responsive", + exercised_on=["chat_completions"], + ) + def test_retries_under_redis_timeouts_keep_answering( + self, client: ComplexityRouterClient, resources: ResourceManager + ) -> None: + proxy = client.proxy + key = proxy.generate_key( + KeyGenerateBody(models=[PRIMARY_MODEL, BACKUP_MODEL], key_alias=f"e2e-redis-timeout-{unique_marker()}") + ) + resources.defer(lambda: proxy.delete_key(key)) + + latencies: list[float] = [] + for request_number in range(1, REQUESTS + 1): + started = time.monotonic() + result = proxy.transport.post( + "/chat/completions", + headers=proxy.transport.bearer(key), + json=ChatBody( + model=PRIMARY_MODEL, + messages=[ChatMessage(role="user", content=f"redis timeout {unique_marker()} {request_number}")], + max_tokens=5, + ), + response_type=ChatResponse, + timeout=MAX_SECONDS_PER_REQUEST, + ) + elapsed = time.monotonic() - started + assert isinstance(result, Success), ( + f"request {request_number} failed after {elapsed:.1f}s with Redis timing out: {result}; " + f"earlier requests took {[round(seconds, 2) for seconds in latencies]}" + ) + assert result.data.choices, f"request {request_number}: fallback to {BACKUP_MODEL} returned no choices" + assert elapsed < MAX_SECONDS_PER_REQUEST, ( + f"request {request_number} took {elapsed:.1f}s with Redis timing out; " + f"earlier requests took {[round(seconds, 2) for seconds in latencies]}" + ) + latencies.append(elapsed) + + third = REQUESTS // 3 + early = sum(latencies[:third]) / third + late = sum(latencies[-third:]) / third + assert late <= max(early * MAX_LATENCY_GROWTH_RATIO, 0.5), ( + f"per-request latency grew from {early:.2f}s to {late:.2f}s across {REQUESTS} requests " + "while Redis timed out; the proxy is paying more for each failed request" + ) + + started = time.monotonic() + probe = proxy.transport.probe("/health/liveliness", params=NoBody()) + liveliness_seconds = time.monotonic() - started + assert probe.healthy, f"/health/liveliness returned {probe.status_code} after the Redis timeout loop" + assert liveliness_seconds < MAX_LIVELINESS_SECONDS, ( + f"/health/liveliness took {liveliness_seconds:.1f}s after the loop; the worker is stalled" + ) + + rows = proxy.poll_logs_for_key(key, min_rows=REQUESTS) + assert len(rows) >= REQUESTS, ( + f"only {len(rows)} of {REQUESTS} requests reached the spend log; a Redis outage must not lose spend rows" + ) From d9e822d2562ddd716d8f6a5e86d3d8fedcb30df1 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 9 Sep 2026 15:51:11 -0700 Subject: [PATCH 02/27] ci: run the Redis timeout e2e test from its own weekly workflow It is a functional e2e test, not a load test, so give it its own workflow instead of a job inside the load anomaly run. It keeps the Saturday 12:00 UTC cadence and manual dispatch, and boots the timeout-config proxy with Postgres and Valkey services exactly as before. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014ZDULyJPp17ZFiJenRxs2T --- .github/workflows/test-e2e-redis-timeout.yml | 94 ++++++++++++++++++++ .github/workflows/weekly_load_anomaly.yml | 84 ----------------- tests/e2e/CLAUDE.md | 2 +- tests/e2e/CONTRIBUTING.md | 2 +- 4 files changed, 96 insertions(+), 86 deletions(-) create mode 100644 .github/workflows/test-e2e-redis-timeout.yml diff --git a/.github/workflows/test-e2e-redis-timeout.yml b/.github/workflows/test-e2e-redis-timeout.yml new file mode 100644 index 00000000000..a956ed79590 --- /dev/null +++ b/.github/workflows/test-e2e-redis-timeout.yml @@ -0,0 +1,94 @@ +name: "Weekly Redis Timeout E2E" + +on: + schedule: + - cron: "0 12 * * 6" + workflow_dispatch: + +permissions: + contents: read + +jobs: + redis-timeout-e2e: + if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + postgres: + image: postgres:16.6 + env: + POSTGRES_USER: llmproxy + POSTGRES_PASSWORD: dbpassword9090 + POSTGRES_DB: litellm + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U llmproxy" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + valkey: + image: valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa + ports: + - 6379:6379 + options: >- + --health-cmd "valkey-cli ping" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm + LITELLM_MASTER_KEY: sk-redis-timeout-e2e + LITELLM_LOG: WARNING + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build + + - name: Install dependencies + run: | + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy + + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + + - name: Generate Prisma client + run: | + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + - name: Start the proxy with a Redis that times out on every command + run: | + nohup uv run --no-sync litellm --config tests/e2e/gateway/redis_timeout_ci_config.yml --port 4000 > proxy.log 2>&1 & + for _ in $(seq 1 90); do + if curl -fs http://localhost:4000/health/liveliness > /dev/null; then + exit 0 + fi + sleep 2 + done + echo "proxy never became live" + tail -n 100 proxy.log + exit 1 + + - name: Run the Redis timeout e2e test + env: + E2E_REDIS_TIMEOUT: "1" + LITELLM_PROXY_URL: http://localhost:4000 + run: | + uv run --no-sync pytest tests/e2e/router/test_redis_timeout_e2e.py -v --tb=short -rA + + - name: Show proxy log on failure + if: failure() + run: tail -n 300 proxy.log diff --git a/.github/workflows/weekly_load_anomaly.yml b/.github/workflows/weekly_load_anomaly.yml index 6e564cf7e78..3e1fca89645 100644 --- a/.github/workflows/weekly_load_anomaly.yml +++ b/.github/workflows/weekly_load_anomaly.yml @@ -83,87 +83,3 @@ jobs: - name: Show proxy log on failure if: failure() run: tail -n 300 proxy.log - - redis-timeout-e2e: - if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm' - runs-on: ubuntu-latest - timeout-minutes: 30 - services: - postgres: - image: postgres:16.6 - env: - POSTGRES_USER: llmproxy - POSTGRES_PASSWORD: dbpassword9090 - POSTGRES_DB: litellm - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U llmproxy" - --health-interval 5s - --health-timeout 5s - --health-retries 10 - valkey: - image: valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa - ports: - - 6379:6379 - options: >- - --health-cmd "valkey-cli ping" - --health-interval 5s - --health-timeout 5s - --health-retries 10 - env: - DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm - LITELLM_MASTER_KEY: sk-redis-timeout-e2e - LITELLM_LOG: WARNING - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Set up uv - uses: ./.github/actions/setup-uv-with-retries - with: - version: "0.10.9" - - - name: Cache the Rust build - uses: ./.github/actions/cache-cargo-build - - - name: Install dependencies - run: | - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy - - - name: Cache Prisma binaries - uses: ./.github/actions/cache-prisma-binaries - - - name: Generate Prisma client - run: | - uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - - - name: Start the proxy with a Redis that times out on every command - run: | - nohup uv run --no-sync litellm --config tests/e2e/gateway/redis_timeout_ci_config.yml --port 4000 > proxy.log 2>&1 & - for _ in $(seq 1 90); do - if curl -fs http://localhost:4000/health/liveliness > /dev/null; then - exit 0 - fi - sleep 2 - done - echo "proxy never became live" - tail -n 100 proxy.log - exit 1 - - - name: Run the Redis timeout e2e test - env: - E2E_REDIS_TIMEOUT: "1" - LITELLM_PROXY_URL: http://localhost:4000 - run: | - uv run --no-sync pytest tests/e2e/router/test_redis_timeout_e2e.py -v --tb=short -rA - - - name: Show proxy log on failure - if: failure() - run: tail -n 300 proxy.log diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 53342f66b67..76e8550cb9a 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -17,7 +17,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server (see "MCP suite: real Datadog only" below); plus the gateway-managed OAuth (authorization_code) path exercised through `/chat/completions`, the one behavior Datadog's static-header auth cannot reach, seeding the per-user upstream token via the interactive authorize dance driven with the mcp SDK's own OAuth client (headless-browser consent from a saved session) and asserting the completion lists and executes the server's tools with the stored per-user token - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection -- `router/` - routing and reliability behavior (fallbacks, cooldowns). Also holds `test_redis_timeout_e2e.py`, which needs a proxy booted from `gateway/redis_timeout_ci_config.yml` (real Redis, `socket_timeout: 0.001`, so every command times out); it is marked `redis_timeout`, deselected unless `E2E_REDIS_TIMEOUT` is set, excluded from the per-PR check, and driven by `.github/workflows/weekly_load_anomaly.yml` +- `router/` - routing and reliability behavior (fallbacks, cooldowns). Also holds `test_redis_timeout_e2e.py`, which needs a proxy booted from `gateway/redis_timeout_ci_config.yml` (real Redis, `socket_timeout: 0.001`, so every command times out); it is marked `redis_timeout`, deselected unless `E2E_REDIS_TIMEOUT` is set, excluded from the per-PR check, and driven weekly by `.github/workflows/test-e2e-redis-timeout.yml` - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What remains here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`) and markerless harness unit tests for the Locust/session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index ccd27df6f46..7a859965c80 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -65,7 +65,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, `guardrails/test_presidio_masking_e2e.py`, and `router/test_redis_timeout_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start, and the Redis timeout test needs a proxy whose Redis times out on every command (`gateway/redis_timeout_ci_config.yml`), which the weekly load workflow boots +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, `guardrails/test_presidio_masking_e2e.py`, and `router/test_redis_timeout_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start, and the Redis timeout test needs a proxy whose Redis times out on every command (`gateway/redis_timeout_ci_config.yml`), which `.github/workflows/test-e2e-redis-timeout.yml` boots on a weekly schedule Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch From 02351da51f14d644660c4a5583968389752af46b Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 9 Sep 2026 15:51:35 -0700 Subject: [PATCH 03/27] test(e2e): register the Redis timeout cell at tier P1 Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014ZDULyJPp17ZFiJenRxs2T --- tests/e2e/coverage_registry/reliability.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index d62f0105931..38427b647b9 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -31,7 +31,7 @@ - {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"} - {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix"} - {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"} -- {id: reliability.circuit_breaker.redis_timeout.stays_responsive, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis_timeout, assertions: [stays_responsive], exercised_on: [chat_completions], source: "litellm/proxy/hooks/proxy_track_cost_callback.py:386", fail_before_fix: proven, rationale: "With every Redis command timing out and every request retrying then falling back, per-request latency stays flat, liveliness stays fast, and spend rows still land; on v1.100.0 the failed-tracking alert body doubled per request until the worker OOMed (LIT-6780)"} +- {id: reliability.circuit_breaker.redis_timeout.stays_responsive, module: reliability, tier: P1, behavior: circuit_breaker, variant: redis_timeout, assertions: [stays_responsive], exercised_on: [chat_completions], source: "litellm/proxy/hooks/proxy_track_cost_callback.py:386", fail_before_fix: proven, rationale: "With every Redis command timing out and every request retrying then falling back, per-request latency stays flat, liveliness stays fast, and spend rows still land; on v1.100.0 the failed-tracking alert body doubled per request until the worker OOMed (LIT-6780)"} - {id: reliability.timeout.request_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: request_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions, messages], source: "litellm/router.py:545-551", rationale: "Per-request timeout raises Timeout"} - {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"} - {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"} From 939ac04a932baae63f2f8b9440ee43ed70570763 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 9 Sep 2026 16:03:36 -0700 Subject: [PATCH 04/27] test(e2e): cover /v1/responses in the Redis timeout test and fail the primary for real The Responses path returns a mock for any mock_response string, so the InternalServerError sentinel never failed there. Point the primary deployment's api_base at a closed port instead, which fails every endpoint the same way, then parametrize the test over /chat/completions and /v1/responses and register both on the coverage cell. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014ZDULyJPp17ZFiJenRxs2T --- tests/e2e/coverage_registry/reliability.yaml | 2 +- tests/e2e/gateway/redis_timeout_ci_config.yml | 2 +- tests/e2e/router/test_redis_timeout_e2e.py | 105 +++++++++++++----- 3 files changed, 81 insertions(+), 28 deletions(-) diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 38427b647b9..3b378b56870 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -31,7 +31,7 @@ - {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"} - {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix"} - {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"} -- {id: reliability.circuit_breaker.redis_timeout.stays_responsive, module: reliability, tier: P1, behavior: circuit_breaker, variant: redis_timeout, assertions: [stays_responsive], exercised_on: [chat_completions], source: "litellm/proxy/hooks/proxy_track_cost_callback.py:386", fail_before_fix: proven, rationale: "With every Redis command timing out and every request retrying then falling back, per-request latency stays flat, liveliness stays fast, and spend rows still land; on v1.100.0 the failed-tracking alert body doubled per request until the worker OOMed (LIT-6780)"} +- {id: reliability.circuit_breaker.redis_timeout.stays_responsive, module: reliability, tier: P1, behavior: circuit_breaker, variant: redis_timeout, assertions: [stays_responsive], exercised_on: [chat_completions, responses], source: "litellm/proxy/hooks/proxy_track_cost_callback.py:386", fail_before_fix: proven, rationale: "With every Redis command timing out and every request retrying then falling back, per-request latency stays flat, liveliness stays fast, and spend rows still land; on v1.100.0 the failed-tracking alert body doubled per request until the worker OOMed (LIT-6780)"} - {id: reliability.timeout.request_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: request_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions, messages], source: "litellm/router.py:545-551", rationale: "Per-request timeout raises Timeout"} - {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"} - {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"} diff --git a/tests/e2e/gateway/redis_timeout_ci_config.yml b/tests/e2e/gateway/redis_timeout_ci_config.yml index 69ab2ee14dc..735285adb36 100644 --- a/tests/e2e/gateway/redis_timeout_ci_config.yml +++ b/tests/e2e/gateway/redis_timeout_ci_config.yml @@ -21,7 +21,7 @@ model_list: litellm_params: model: openai/gpt-5-mini api_key: sk-redis-timeout-primary-not-used - mock_response: "litellm.InternalServerError" + api_base: http://127.0.0.1:1 - model_name: redis-timeout-backup litellm_params: model: openai/gpt-5-mini diff --git a/tests/e2e/router/test_redis_timeout_e2e.py b/tests/e2e/router/test_redis_timeout_e2e.py index eeb5224e051..d10f32b6482 100644 --- a/tests/e2e/router/test_redis_timeout_e2e.py +++ b/tests/e2e/router/test_redis_timeout_e2e.py @@ -1,25 +1,29 @@ """Live e2e: the proxy keeps answering while every Redis command times out. Runs only against a proxy booted from tests/e2e/gateway/redis_timeout_ci_config.yml, which -points cache_params at a real Redis with socket_timeout 0.001 so every command times out and -the circuit breaker opens. Each request fails its primary deployment, retries, falls back to the -backup and succeeds, so it carries retry breadcrumbs; its cost tracking then fails on the spend -counter increment and stringifies the request metadata into a failed-tracking alert. On v1.100.0 -that string doubled per request until the worker hung (LIT-6780). Deselected unless -E2E_REDIS_TIMEOUT is set, since it needs that dedicated proxy. +points cache_params at a real Redis with socket_timeout 0.001 so commands time out and the +circuit breaker opens. Each request fails its primary deployment, whose api_base is a closed +port, retries, falls back to the backup and succeeds, so it carries retry breadcrumbs; its cost +tracking then fails on the spend counter increment and stringifies the request metadata into a +failed-tracking alert. On v1.100.0 that string doubled per request until the worker hung +(LIT-6780). Deselected unless E2E_REDIS_TIMEOUT is set, since it needs that dedicated proxy. """ from __future__ import annotations import time +from collections.abc import Callable +from dataclasses import dataclass from typing import Final import pytest from complexity_router_client import ComplexityRouterClient from e2e_config import unique_marker -from e2e_http import NoBody, Success +from e2e_http import NoBody, Result, Success from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatResponse, KeyGenerateBody +from proxy_client import ProxyClient +from pydantic import BaseModel pytestmark = [pytest.mark.e2e, pytest.mark.redis_timeout] @@ -31,42 +35,90 @@ MAX_LATENCY_GROWTH_RATIO: Final = 3.0 MAX_LIVELINESS_SECONDS: Final = 2.0 +class ResponsesBody(BaseModel): + model: str + input: str + max_output_tokens: int = 5 + + +class ResponsesObject(BaseModel): + id: str | None = None + status: str | None = None + output: list[object] = [] + + +@dataclass(frozen=True, slots=True) +class Endpoint: + name: str + send: Callable[[ProxyClient, str, str], Result[BaseModel]] + served: Callable[[BaseModel], bool] + + +def _send_chat(proxy: ProxyClient, key: str, marker: str) -> Result[BaseModel]: + return proxy.transport.post( + "/chat/completions", + headers=proxy.transport.bearer(key), + json=ChatBody(model=PRIMARY_MODEL, messages=[ChatMessage(role="user", content=marker)], max_tokens=5), + response_type=ChatResponse, + timeout=MAX_SECONDS_PER_REQUEST, + ) + + +def _send_responses(proxy: ProxyClient, key: str, marker: str) -> Result[BaseModel]: + return proxy.transport.post( + "/v1/responses", + headers=proxy.transport.bearer(key), + json=ResponsesBody(model=PRIMARY_MODEL, input=marker), + response_type=ResponsesObject, + timeout=MAX_SECONDS_PER_REQUEST, + ) + + +ENDPOINTS: Final = ( + Endpoint( + name="chat_completions", + send=_send_chat, + served=lambda data: isinstance(data, ChatResponse) and bool(data.choices), + ), + Endpoint( + name="responses", + send=_send_responses, + served=lambda data: isinstance(data, ResponsesObject) and bool(data.output), + ), +) + + class TestRedisTimeout: + @pytest.mark.parametrize("endpoint", ENDPOINTS, ids=[endpoint.name for endpoint in ENDPOINTS]) @pytest.mark.covers( "reliability.circuit_breaker.redis_timeout.stays_responsive", - exercised_on=["chat_completions"], + exercised_on=["chat_completions", "responses"], ) def test_retries_under_redis_timeouts_keep_answering( - self, client: ComplexityRouterClient, resources: ResourceManager + self, client: ComplexityRouterClient, resources: ResourceManager, endpoint: Endpoint ) -> None: proxy = client.proxy key = proxy.generate_key( - KeyGenerateBody(models=[PRIMARY_MODEL, BACKUP_MODEL], key_alias=f"e2e-redis-timeout-{unique_marker()}") + KeyGenerateBody( + models=[PRIMARY_MODEL, BACKUP_MODEL], key_alias=f"e2e-redis-timeout-{endpoint.name}-{unique_marker()}" + ) ) resources.defer(lambda: proxy.delete_key(key)) latencies: list[float] = [] for request_number in range(1, REQUESTS + 1): started = time.monotonic() - result = proxy.transport.post( - "/chat/completions", - headers=proxy.transport.bearer(key), - json=ChatBody( - model=PRIMARY_MODEL, - messages=[ChatMessage(role="user", content=f"redis timeout {unique_marker()} {request_number}")], - max_tokens=5, - ), - response_type=ChatResponse, - timeout=MAX_SECONDS_PER_REQUEST, - ) + result = endpoint.send(proxy, key, f"redis timeout {unique_marker()} {request_number}") elapsed = time.monotonic() - started assert isinstance(result, Success), ( - f"request {request_number} failed after {elapsed:.1f}s with Redis timing out: {result}; " + f"{endpoint.name} request {request_number} failed after {elapsed:.1f}s with Redis timing out: {result}; " f"earlier requests took {[round(seconds, 2) for seconds in latencies]}" ) - assert result.data.choices, f"request {request_number}: fallback to {BACKUP_MODEL} returned no choices" + assert endpoint.served(result.data), ( + f"{endpoint.name} request {request_number}: fallback to {BACKUP_MODEL} returned no output" + ) assert elapsed < MAX_SECONDS_PER_REQUEST, ( - f"request {request_number} took {elapsed:.1f}s with Redis timing out; " + f"{endpoint.name} request {request_number} took {elapsed:.1f}s with Redis timing out; " f"earlier requests took {[round(seconds, 2) for seconds in latencies]}" ) latencies.append(elapsed) @@ -75,7 +127,7 @@ class TestRedisTimeout: early = sum(latencies[:third]) / third late = sum(latencies[-third:]) / third assert late <= max(early * MAX_LATENCY_GROWTH_RATIO, 0.5), ( - f"per-request latency grew from {early:.2f}s to {late:.2f}s across {REQUESTS} requests " + f"{endpoint.name} per-request latency grew from {early:.2f}s to {late:.2f}s across {REQUESTS} requests " "while Redis timed out; the proxy is paying more for each failed request" ) @@ -89,5 +141,6 @@ class TestRedisTimeout: rows = proxy.poll_logs_for_key(key, min_rows=REQUESTS) assert len(rows) >= REQUESTS, ( - f"only {len(rows)} of {REQUESTS} requests reached the spend log; a Redis outage must not lose spend rows" + f"only {len(rows)} of {REQUESTS} {endpoint.name} requests reached the spend log; " + "a Redis outage must not lose spend rows" ) From 2d7e5999b78f726c97a7142231de566d788d4fc0 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 9 Sep 2026 16:26:03 -0700 Subject: [PATCH 05/27] test(e2e): pause Redis writes for the run and prove the timeouts from /metrics A loopback Redis answers many commands inside the 1 ms socket timeout, so nothing guaranteed the failure path ran. The test now holds the proxy's Redis in CLIENT PAUSE WRITE for its duration, so every write the proxy sends, the spend counter increment included, outlives the timeout, and lifts the pause in teardown. Reads stay live so the control connection can do that. Enable the prometheus callback in the gateway config and assert from /metrics that the proxy counted at least the breaker's five timeouts and that, during each case, it saw fresh timeouts, a breaker transition, or an open breaker rejecting every call. The open breaker is the state a customer's worker sits in, and cost tracking fails on every request either way. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014ZDULyJPp17ZFiJenRxs2T --- .github/workflows/test-e2e-redis-timeout.yml | 2 + tests/e2e/gateway/redis_timeout_ci_config.yml | 2 + tests/e2e/router/test_redis_timeout_e2e.py | 60 +++++++++++++++++-- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-e2e-redis-timeout.yml b/.github/workflows/test-e2e-redis-timeout.yml index a956ed79590..9c318a1346c 100644 --- a/.github/workflows/test-e2e-redis-timeout.yml +++ b/.github/workflows/test-e2e-redis-timeout.yml @@ -86,6 +86,8 @@ jobs: env: E2E_REDIS_TIMEOUT: "1" LITELLM_PROXY_URL: http://localhost:4000 + REDIS_HOST: 127.0.0.1 + REDIS_PORT: "6379" run: | uv run --no-sync pytest tests/e2e/router/test_redis_timeout_e2e.py -v --tb=short -rA diff --git a/tests/e2e/gateway/redis_timeout_ci_config.yml b/tests/e2e/gateway/redis_timeout_ci_config.yml index 735285adb36..375a51f6127 100644 --- a/tests/e2e/gateway/redis_timeout_ci_config.yml +++ b/tests/e2e/gateway/redis_timeout_ci_config.yml @@ -3,6 +3,8 @@ general_settings: store_model_in_db: true litellm_settings: + callbacks: ["prometheus"] + require_auth_for_metrics_endpoint: false cache: true cache_params: type: redis diff --git a/tests/e2e/router/test_redis_timeout_e2e.py b/tests/e2e/router/test_redis_timeout_e2e.py index d10f32b6482..e99bd2fbe80 100644 --- a/tests/e2e/router/test_redis_timeout_e2e.py +++ b/tests/e2e/router/test_redis_timeout_e2e.py @@ -1,8 +1,10 @@ """Live e2e: the proxy keeps answering while every Redis command times out. Runs only against a proxy booted from tests/e2e/gateway/redis_timeout_ci_config.yml, which -points cache_params at a real Redis with socket_timeout 0.001 so commands time out and the -circuit breaker opens. Each request fails its primary deployment, whose api_base is a closed +points cache_params at a real Redis with socket_timeout 0.001. The test holds that Redis in +CLIENT PAUSE WRITE for its duration, so every write the proxy sends, the spend counter increment +included, hangs past the timeout, and it proves the degradation was real from the breaker metrics on /metrics: fresh timeouts, a breaker transition, +or an already-open breaker rejecting every call, which is the state a customer's worker sits in. Each request fails its primary deployment, whose api_base is a closed port, retries, falls back to the backup and succeeds, so it carries retry breadcrumbs; its cost tracking then fails on the spend counter increment and stringifies the request metadata into a failed-tracking alert. On v1.100.0 that string doubled per request until the worker hung @@ -11,12 +13,15 @@ failed-tracking alert. On v1.100.0 that string doubled per request until the wor from __future__ import annotations +import os +import re import time -from collections.abc import Callable +from collections.abc import Callable, Iterator from dataclasses import dataclass from typing import Final import pytest +import redis from complexity_router_client import ComplexityRouterClient from e2e_config import unique_marker from e2e_http import NoBody, Result, Success @@ -33,6 +38,11 @@ REQUESTS: Final = 20 MAX_SECONDS_PER_REQUEST: Final = 10.0 MAX_LATENCY_GROWTH_RATIO: Final = 3.0 MAX_LIVELINESS_SECONDS: Final = 2.0 +REDIS_PAUSE_MS: Final = 600_000 +BREAKER_FAILURE_THRESHOLD: Final = 5 +TIMEOUT_FAILURES_RE: Final = re.compile( + r'^litellm_redis_circuit_breaker_failures_total\{failure_class="timeout"\} ([0-9.e+]+)$', re.M +) class ResponsesBody(BaseModel): @@ -86,6 +96,32 @@ ENDPOINTS: Final = ( served=lambda data: isinstance(data, ResponsesObject) and bool(data.output), ), ) +BREAKER_OPEN_RE: Final = re.compile(r'^litellm_redis_circuit_breaker_state\{state="open"\} ([0-9.e+]+)$', re.M) +BREAKER_TRANSITIONS_RE: Final = re.compile( + r'^litellm_redis_circuit_breaker_transitions_total\{state="[a-z_]+"\} ([0-9.e+]+)$', re.M +) + + +@pytest.fixture +def paused_redis() -> Iterator[None]: + """Hold the proxy's Redis in CLIENT PAUSE WRITE so every write it sends outlives the 1 ms socket + timeout. A loopback Redis otherwise answers many commands inside that budget. Reads stay live so + this control connection can lift the pause in teardown.""" + host = os.environ.get("REDIS_HOST") + port = os.environ.get("REDIS_PORT") + assert host and port, "REDIS_HOST and REDIS_PORT must name the Redis the proxy under test uses" + control = redis.Redis(host=host, port=int(port), socket_timeout=5) + control.client_pause(REDIS_PAUSE_MS, all=False) # pyright: ignore[reportUnknownMemberType] # redis-py stubs return Any + try: + yield + finally: + control.client_unpause() # pyright: ignore[reportUnknownMemberType] # redis-py stubs return Any + control.close() + + +def _metric(proxy: ProxyClient, pattern: re.Pattern[str]) -> float: + body = proxy.probe("/metrics", params=NoBody()).body + return sum(float(match.group(1)) for match in pattern.finditer(body)) class TestRedisTimeout: @@ -95,9 +131,11 @@ class TestRedisTimeout: exercised_on=["chat_completions", "responses"], ) def test_retries_under_redis_timeouts_keep_answering( - self, client: ComplexityRouterClient, resources: ResourceManager, endpoint: Endpoint + self, client: ComplexityRouterClient, resources: ResourceManager, endpoint: Endpoint, paused_redis: None ) -> None: proxy = client.proxy + timeouts_before = _metric(proxy, TIMEOUT_FAILURES_RE) + transitions_before = _metric(proxy, BREAKER_TRANSITIONS_RE) key = proxy.generate_key( KeyGenerateBody( models=[PRIMARY_MODEL, BACKUP_MODEL], key_alias=f"e2e-redis-timeout-{endpoint.name}-{unique_marker()}" @@ -139,6 +177,20 @@ class TestRedisTimeout: f"/health/liveliness took {liveliness_seconds:.1f}s after the loop; the worker is stalled" ) + timeouts_total = _metric(proxy, TIMEOUT_FAILURES_RE) + timeouts = timeouts_total - timeouts_before + transitions = _metric(proxy, BREAKER_TRANSITIONS_RE) - transitions_before + breaker_open = _metric(proxy, BREAKER_OPEN_RE) >= 1 + assert timeouts_total >= BREAKER_FAILURE_THRESHOLD, ( + f"the proxy counted only {timeouts_total:.0f} Redis timeouts in its lifetime; the write-paused Redis " + "never made its spend counter writes time out, so this run proved nothing" + ) + assert timeouts >= REQUESTS or transitions >= 1 or breaker_open, ( + f"during {REQUESTS} {endpoint.name} requests the breaker counted {timeouts:.0f} new timeouts, " + f"{transitions:.0f} state transitions, and ended {'open' if breaker_open else 'closed'}; " + "Redis was healthy for this case, so it proved nothing" + ) + rows = proxy.poll_logs_for_key(key, min_rows=REQUESTS) assert len(rows) >= REQUESTS, ( f"only {len(rows)} of {REQUESTS} {endpoint.name} requests reached the spend log; " From 2e4461ec458756c940f3a8863ea805bc44c33907 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 9 Sep 2026 16:33:18 -0700 Subject: [PATCH 06/27] test(e2e): register the Redis timeout deployments through /model/new The e2e directive has every test create its deployments through the management API and delete them on teardown. Drop the static model_list from the gateway config; the test now registers the closed-port primary and the mock backup itself, and the fallback map stays in router_settings where proxy-level config belongs. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014ZDULyJPp17ZFiJenRxs2T --- tests/e2e/gateway/redis_timeout_ci_config.yml | 12 ----------- tests/e2e/router/test_redis_timeout_e2e.py | 20 ++++++++++++++++--- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/tests/e2e/gateway/redis_timeout_ci_config.yml b/tests/e2e/gateway/redis_timeout_ci_config.yml index 375a51f6127..9bcf65162d6 100644 --- a/tests/e2e/gateway/redis_timeout_ci_config.yml +++ b/tests/e2e/gateway/redis_timeout_ci_config.yml @@ -17,15 +17,3 @@ router_settings: fallbacks: - redis-timeout-primary: - redis-timeout-backup - -model_list: - - model_name: redis-timeout-primary - litellm_params: - model: openai/gpt-5-mini - api_key: sk-redis-timeout-primary-not-used - api_base: http://127.0.0.1:1 - - model_name: redis-timeout-backup - litellm_params: - model: openai/gpt-5-mini - api_key: sk-redis-timeout-backup-not-used - mock_response: "ok" diff --git a/tests/e2e/router/test_redis_timeout_e2e.py b/tests/e2e/router/test_redis_timeout_e2e.py index e99bd2fbe80..58d10f1d8ea 100644 --- a/tests/e2e/router/test_redis_timeout_e2e.py +++ b/tests/e2e/router/test_redis_timeout_e2e.py @@ -4,8 +4,8 @@ Runs only against a proxy booted from tests/e2e/gateway/redis_timeout_ci_config. points cache_params at a real Redis with socket_timeout 0.001. The test holds that Redis in CLIENT PAUSE WRITE for its duration, so every write the proxy sends, the spend counter increment included, hangs past the timeout, and it proves the degradation was real from the breaker metrics on /metrics: fresh timeouts, a breaker transition, -or an already-open breaker rejecting every call, which is the state a customer's worker sits in. Each request fails its primary deployment, whose api_base is a closed -port, retries, falls back to the backup and succeeds, so it carries retry breadcrumbs; its cost +or an already-open breaker rejecting every call, which is the state a customer's worker sits in. The test registers two deployments through /model/new: a primary whose api_base is a closed port +and a backup that answers with a mock. Each request fails the primary, retries, falls back and succeeds, so it carries retry breadcrumbs; its cost tracking then fails on the spend counter increment and stringifies the request metadata into a failed-tracking alert. On v1.100.0 that string doubled per request until the worker hung (LIT-6780). Deselected unless E2E_REDIS_TIMEOUT is set, since it needs that dedicated proxy. @@ -26,7 +26,7 @@ from complexity_router_client import ComplexityRouterClient from e2e_config import unique_marker from e2e_http import NoBody, Result, Success from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, ChatResponse, KeyGenerateBody +from models import ChatBody, ChatMessage, ChatResponse, KeyGenerateBody, LiteLLMParamsBody from proxy_client import ProxyClient from pydantic import BaseModel @@ -34,6 +34,8 @@ pytestmark = [pytest.mark.e2e, pytest.mark.redis_timeout] PRIMARY_MODEL: Final = "redis-timeout-primary" BACKUP_MODEL: Final = "redis-timeout-backup" +BACKING_MODEL: Final = "openai/gpt-5-mini" +CLOSED_PORT_API_BASE: Final = "http://127.0.0.1:1" REQUESTS: Final = 20 MAX_SECONDS_PER_REQUEST: Final = 10.0 MAX_LATENCY_GROWTH_RATIO: Final = 3.0 @@ -134,6 +136,18 @@ class TestRedisTimeout: self, client: ComplexityRouterClient, resources: ResourceManager, endpoint: Endpoint, paused_redis: None ) -> None: proxy = client.proxy + primary_id = proxy.create_model( + PRIMARY_MODEL, + LiteLLMParamsBody( + model=BACKING_MODEL, api_key="sk-redis-timeout-primary-not-used", api_base=CLOSED_PORT_API_BASE + ), + ) + resources.defer(lambda: proxy.delete_model(primary_id)) + backup_id = proxy.create_model( + BACKUP_MODEL, + LiteLLMParamsBody(model=BACKING_MODEL, api_key="sk-redis-timeout-backup-not-used", mock_response="ok"), + ) + resources.defer(lambda: proxy.delete_model(backup_id)) timeouts_before = _metric(proxy, TIMEOUT_FAILURES_RE) transitions_before = _metric(proxy, BREAKER_TRANSITIONS_RE) key = proxy.generate_key( From 1213d7d39929500883fee4c3823f8dfbc073fdb8 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 9 Sep 2026 17:17:33 -0700 Subject: [PATCH 07/27] test(e2e): cover /embeddings and assert memory and fallbacks in the Redis timeout test Add an embeddings case with its own closed-port primary and mock backup (the fallback map in the gateway config gains the pair; LiteLLMParamsBody.mock_response accepts the list an embedding mock needs). Assert from /metrics that the proxy's resident memory grows by no more than 200 MB across each case where the process collector reports it (Linux), that the router counted a successful fallback for every request, and that every spend row is a success. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014ZDULyJPp17ZFiJenRxs2T --- tests/e2e/coverage_registry/reliability.yaml | 2 +- tests/e2e/gateway/redis_timeout_ci_config.yml | 2 + tests/e2e/models.py | 2 +- tests/e2e/router/test_redis_timeout_e2e.py | 142 +++++++++++++----- 4 files changed, 110 insertions(+), 38 deletions(-) diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 3b378b56870..a42e4b2779a 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -31,7 +31,7 @@ - {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"} - {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix"} - {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"} -- {id: reliability.circuit_breaker.redis_timeout.stays_responsive, module: reliability, tier: P1, behavior: circuit_breaker, variant: redis_timeout, assertions: [stays_responsive], exercised_on: [chat_completions, responses], source: "litellm/proxy/hooks/proxy_track_cost_callback.py:386", fail_before_fix: proven, rationale: "With every Redis command timing out and every request retrying then falling back, per-request latency stays flat, liveliness stays fast, and spend rows still land; on v1.100.0 the failed-tracking alert body doubled per request until the worker OOMed (LIT-6780)"} +- {id: reliability.circuit_breaker.redis_timeout.stays_responsive, module: reliability, tier: P1, behavior: circuit_breaker, variant: redis_timeout, assertions: [stays_responsive], exercised_on: [chat_completions, responses, embeddings], source: "litellm/proxy/hooks/proxy_track_cost_callback.py:386", fail_before_fix: proven, rationale: "With every Redis command timing out and every request retrying then falling back, per-request latency stays flat, liveliness stays fast, and spend rows still land; on v1.100.0 the failed-tracking alert body doubled per request until the worker OOMed (LIT-6780)"} - {id: reliability.timeout.request_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: request_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions, messages], source: "litellm/router.py:545-551", rationale: "Per-request timeout raises Timeout"} - {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"} - {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"} diff --git a/tests/e2e/gateway/redis_timeout_ci_config.yml b/tests/e2e/gateway/redis_timeout_ci_config.yml index 9bcf65162d6..fd283dbc7df 100644 --- a/tests/e2e/gateway/redis_timeout_ci_config.yml +++ b/tests/e2e/gateway/redis_timeout_ci_config.yml @@ -17,3 +17,5 @@ router_settings: fallbacks: - redis-timeout-primary: - redis-timeout-backup + - redis-timeout-embed-primary: + - redis-timeout-embed-backup diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 62810e6cfd9..f3c111bdf06 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -920,7 +920,7 @@ class LiteLLMParamsBody(BaseModel): auto_router_default_model: str | None = None auto_router_embedding_model: str | None = None tags: list[str] | None = None - mock_response: str | None = None + mock_response: str | list[float] | None = None timeout: float | None = None tpm: int | None = None weight: int | None = None diff --git a/tests/e2e/router/test_redis_timeout_e2e.py b/tests/e2e/router/test_redis_timeout_e2e.py index 58d10f1d8ea..3f1984c0696 100644 --- a/tests/e2e/router/test_redis_timeout_e2e.py +++ b/tests/e2e/router/test_redis_timeout_e2e.py @@ -3,12 +3,12 @@ Runs only against a proxy booted from tests/e2e/gateway/redis_timeout_ci_config.yml, which points cache_params at a real Redis with socket_timeout 0.001. The test holds that Redis in CLIENT PAUSE WRITE for its duration, so every write the proxy sends, the spend counter increment -included, hangs past the timeout, and it proves the degradation was real from the breaker metrics on /metrics: fresh timeouts, a breaker transition, -or an already-open breaker rejecting every call, which is the state a customer's worker sits in. The test registers two deployments through /model/new: a primary whose api_base is a closed port -and a backup that answers with a mock. Each request fails the primary, retries, falls back and succeeds, so it carries retry breadcrumbs; its cost -tracking then fails on the spend counter increment and stringifies the request metadata into a -failed-tracking alert. On v1.100.0 that string doubled per request until the worker hung -(LIT-6780). Deselected unless E2E_REDIS_TIMEOUT is set, since it needs that dedicated proxy. +included, hangs past the timeout. For each endpoint it registers two deployments through +/model/new: a primary whose api_base is a closed port and a backup that answers with a mock. Each request fails the primary, +retries, falls back and succeeds, so it carries retry breadcrumbs; its cost tracking then fails on +the spend counter increment and stringifies the request metadata into a failed-tracking alert. On +v1.100.0 that string doubled per request until the worker hung (LIT-6780). Deselected unless +E2E_REDIS_TIMEOUT is set, since it needs that dedicated proxy. """ from __future__ import annotations @@ -26,25 +26,29 @@ from complexity_router_client import ComplexityRouterClient from e2e_config import unique_marker from e2e_http import NoBody, Result, Success from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, ChatResponse, KeyGenerateBody, LiteLLMParamsBody +from models import ChatBody, ChatMessage, ChatResponse, EmbedBody, KeyGenerateBody, LiteLLMParamsBody from proxy_client import ProxyClient from pydantic import BaseModel pytestmark = [pytest.mark.e2e, pytest.mark.redis_timeout] -PRIMARY_MODEL: Final = "redis-timeout-primary" -BACKUP_MODEL: Final = "redis-timeout-backup" -BACKING_MODEL: Final = "openai/gpt-5-mini" -CLOSED_PORT_API_BASE: Final = "http://127.0.0.1:1" REQUESTS: Final = 20 MAX_SECONDS_PER_REQUEST: Final = 10.0 MAX_LATENCY_GROWTH_RATIO: Final = 3.0 MAX_LIVELINESS_SECONDS: Final = 2.0 +MAX_RSS_GROWTH_BYTES: Final = 200 * 1024 * 1024 REDIS_PAUSE_MS: Final = 600_000 BREAKER_FAILURE_THRESHOLD: Final = 5 +CLOSED_PORT_API_BASE: Final = "http://127.0.0.1:1" TIMEOUT_FAILURES_RE: Final = re.compile( r'^litellm_redis_circuit_breaker_failures_total\{failure_class="timeout"\} ([0-9.e+]+)$', re.M ) +BREAKER_OPEN_RE: Final = re.compile(r'^litellm_redis_circuit_breaker_state\{state="open"\} ([0-9.e+]+)$', re.M) +BREAKER_TRANSITIONS_RE: Final = re.compile( + r'^litellm_redis_circuit_breaker_transitions_total\{state="[a-z_]+"\} ([0-9.e+]+)$', re.M +) +FALLBACKS_RE: Final = re.compile(r"^litellm_deployment_successful_fallbacks_total\{[^}]*\} ([0-9.e+]+)$", re.M) +RSS_RE: Final = re.compile(r"^process_resident_memory_bytes ([0-9.e+]+)$", re.M) class ResponsesBody(BaseModel): @@ -59,48 +63,95 @@ class ResponsesObject(BaseModel): output: list[object] = [] +class EmbeddingsObject(BaseModel): + model: str | None = None + data: list[object] = [] + + @dataclass(frozen=True, slots=True) class Endpoint: + """One endpoint's deployments and request shape.""" + name: str - send: Callable[[ProxyClient, str, str], Result[BaseModel]] + primary: str + backup: str + primary_params: LiteLLMParamsBody + backup_params: LiteLLMParamsBody + send: Callable[[ProxyClient, str, str, str], Result[BaseModel]] served: Callable[[BaseModel], bool] -def _send_chat(proxy: ProxyClient, key: str, marker: str) -> Result[BaseModel]: +def _send_chat(proxy: ProxyClient, key: str, model: str, marker: str) -> Result[BaseModel]: return proxy.transport.post( "/chat/completions", headers=proxy.transport.bearer(key), - json=ChatBody(model=PRIMARY_MODEL, messages=[ChatMessage(role="user", content=marker)], max_tokens=5), + json=ChatBody(model=model, messages=[ChatMessage(role="user", content=marker)], max_tokens=5), response_type=ChatResponse, timeout=MAX_SECONDS_PER_REQUEST, ) -def _send_responses(proxy: ProxyClient, key: str, marker: str) -> Result[BaseModel]: +def _send_responses(proxy: ProxyClient, key: str, model: str, marker: str) -> Result[BaseModel]: return proxy.transport.post( "/v1/responses", headers=proxy.transport.bearer(key), - json=ResponsesBody(model=PRIMARY_MODEL, input=marker), + json=ResponsesBody(model=model, input=marker), response_type=ResponsesObject, timeout=MAX_SECONDS_PER_REQUEST, ) +def _send_embeddings(proxy: ProxyClient, key: str, model: str, marker: str) -> Result[BaseModel]: + return proxy.transport.post( + "/embeddings", + headers=proxy.transport.bearer(key), + json=EmbedBody(model=model, input=marker), + response_type=EmbeddingsObject, + timeout=MAX_SECONDS_PER_REQUEST, + ) + + +_CHAT_PRIMARY: Final = LiteLLMParamsBody( + model="openai/gpt-5-mini", api_key="sk-redis-timeout-primary-not-used", api_base=CLOSED_PORT_API_BASE +) +_CHAT_BACKUP: Final = LiteLLMParamsBody( + model="openai/gpt-5-nano", api_key="sk-redis-timeout-backup-not-used", mock_response="ok" +) +_EMBED_PRIMARY: Final = LiteLLMParamsBody( + model="openai/text-embedding-3-small", api_key="sk-redis-timeout-primary-not-used", api_base=CLOSED_PORT_API_BASE +) +_EMBED_BACKUP: Final = LiteLLMParamsBody( + model="openai/text-embedding-3-large", api_key="sk-redis-timeout-backup-not-used", mock_response=[0.1, 0.2, 0.3] +) + ENDPOINTS: Final = ( Endpoint( name="chat_completions", + primary="redis-timeout-primary", + backup="redis-timeout-backup", + primary_params=_CHAT_PRIMARY, + backup_params=_CHAT_BACKUP, send=_send_chat, served=lambda data: isinstance(data, ChatResponse) and bool(data.choices), ), Endpoint( name="responses", + primary="redis-timeout-primary", + backup="redis-timeout-backup", + primary_params=_CHAT_PRIMARY, + backup_params=_CHAT_BACKUP, send=_send_responses, served=lambda data: isinstance(data, ResponsesObject) and bool(data.output), ), -) -BREAKER_OPEN_RE: Final = re.compile(r'^litellm_redis_circuit_breaker_state\{state="open"\} ([0-9.e+]+)$', re.M) -BREAKER_TRANSITIONS_RE: Final = re.compile( - r'^litellm_redis_circuit_breaker_transitions_total\{state="[a-z_]+"\} ([0-9.e+]+)$', re.M + Endpoint( + name="embeddings", + primary="redis-timeout-embed-primary", + backup="redis-timeout-embed-backup", + primary_params=_EMBED_PRIMARY, + backup_params=_EMBED_BACKUP, + send=_send_embeddings, + served=lambda data: isinstance(data, EmbeddingsObject) and bool(data.data), + ), ) @@ -126,48 +177,51 @@ def _metric(proxy: ProxyClient, pattern: re.Pattern[str]) -> float: return sum(float(match.group(1)) for match in pattern.finditer(body)) +def _rss_bytes(proxy: ProxyClient) -> float | None: + """The proxy's resident memory from the Prometheus process collector, which reads /proc and + so reports on Linux only; None where the metric is absent.""" + body = proxy.probe("/metrics", params=NoBody()).body + match = RSS_RE.search(body) + return float(match.group(1)) if match else None + + class TestRedisTimeout: @pytest.mark.parametrize("endpoint", ENDPOINTS, ids=[endpoint.name for endpoint in ENDPOINTS]) @pytest.mark.covers( "reliability.circuit_breaker.redis_timeout.stays_responsive", - exercised_on=["chat_completions", "responses"], + exercised_on=["chat_completions", "responses", "embeddings"], ) def test_retries_under_redis_timeouts_keep_answering( self, client: ComplexityRouterClient, resources: ResourceManager, endpoint: Endpoint, paused_redis: None ) -> None: proxy = client.proxy - primary_id = proxy.create_model( - PRIMARY_MODEL, - LiteLLMParamsBody( - model=BACKING_MODEL, api_key="sk-redis-timeout-primary-not-used", api_base=CLOSED_PORT_API_BASE - ), - ) + primary_id = proxy.create_model(endpoint.primary, endpoint.primary_params) resources.defer(lambda: proxy.delete_model(primary_id)) - backup_id = proxy.create_model( - BACKUP_MODEL, - LiteLLMParamsBody(model=BACKING_MODEL, api_key="sk-redis-timeout-backup-not-used", mock_response="ok"), - ) + backup_id = proxy.create_model(endpoint.backup, endpoint.backup_params) resources.defer(lambda: proxy.delete_model(backup_id)) - timeouts_before = _metric(proxy, TIMEOUT_FAILURES_RE) - transitions_before = _metric(proxy, BREAKER_TRANSITIONS_RE) key = proxy.generate_key( KeyGenerateBody( - models=[PRIMARY_MODEL, BACKUP_MODEL], key_alias=f"e2e-redis-timeout-{endpoint.name}-{unique_marker()}" + models=[endpoint.primary, endpoint.backup], + key_alias=f"e2e-redis-timeout-{endpoint.name}-{unique_marker()}", ) ) resources.defer(lambda: proxy.delete_key(key)) + timeouts_before = _metric(proxy, TIMEOUT_FAILURES_RE) + transitions_before = _metric(proxy, BREAKER_TRANSITIONS_RE) + fallbacks_before = _metric(proxy, FALLBACKS_RE) + rss_before = _rss_bytes(proxy) latencies: list[float] = [] for request_number in range(1, REQUESTS + 1): started = time.monotonic() - result = endpoint.send(proxy, key, f"redis timeout {unique_marker()} {request_number}") + result = endpoint.send(proxy, key, endpoint.primary, f"redis timeout {unique_marker()} {request_number}") elapsed = time.monotonic() - started assert isinstance(result, Success), ( f"{endpoint.name} request {request_number} failed after {elapsed:.1f}s with Redis timing out: {result}; " f"earlier requests took {[round(seconds, 2) for seconds in latencies]}" ) assert endpoint.served(result.data), ( - f"{endpoint.name} request {request_number}: fallback to {BACKUP_MODEL} returned no output" + f"{endpoint.name} request {request_number}: fallback to {endpoint.backup} returned no output" ) assert elapsed < MAX_SECONDS_PER_REQUEST, ( f"{endpoint.name} request {request_number} took {elapsed:.1f}s with Redis timing out; " @@ -191,6 +245,20 @@ class TestRedisTimeout: f"/health/liveliness took {liveliness_seconds:.1f}s after the loop; the worker is stalled" ) + if rss_before is not None: + rss_after = _rss_bytes(proxy) + assert rss_after is not None + assert rss_after - rss_before <= MAX_RSS_GROWTH_BYTES, ( + f"proxy RSS grew {(rss_after - rss_before) / 2**20:.0f} MB across {REQUESTS} {endpoint.name} requests " + "with Redis timing out; on v1.100.0 this path grew by gigabytes" + ) + + fallbacks = _metric(proxy, FALLBACKS_RE) - fallbacks_before + assert fallbacks >= REQUESTS, ( + f"only {fallbacks:.0f} successful fallbacks were counted across {REQUESTS} {endpoint.name} requests; " + "the closed-port primary did not fail every request, so the retry path was not exercised" + ) + timeouts_total = _metric(proxy, TIMEOUT_FAILURES_RE) timeouts = timeouts_total - timeouts_before transitions = _metric(proxy, BREAKER_TRANSITIONS_RE) - transitions_before @@ -210,3 +278,5 @@ class TestRedisTimeout: f"only {len(rows)} of {REQUESTS} {endpoint.name} requests reached the spend log; " "a Redis outage must not lose spend rows" ) + failed_rows = [row.status for row in rows if row.status not in (None, "success")] + assert not failed_rows, f"{len(failed_rows)} {endpoint.name} spend rows are not successes: {failed_rows[:3]}" From be7dce30a3ab832bdd104b2d2f15bace5d622b4d Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 10 Sep 2026 15:16:24 -0700 Subject: [PATCH 08/27] ci(e2e): pin the Redis timeout workflow's Postgres image by digest Co-Authored-By: Claude Code --- .github/workflows/test-e2e-redis-timeout.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-e2e-redis-timeout.yml b/.github/workflows/test-e2e-redis-timeout.yml index 9c318a1346c..501af0f240f 100644 --- a/.github/workflows/test-e2e-redis-timeout.yml +++ b/.github/workflows/test-e2e-redis-timeout.yml @@ -15,7 +15,7 @@ jobs: timeout-minutes: 30 services: postgres: - image: postgres:16.6 + image: postgres:16.6@sha256:557fea37a744d5f4c8faab304b0a90858b53ab119735a88c131fd19dab802f36 env: POSTGRES_USER: llmproxy POSTGRES_PASSWORD: dbpassword9090 From 33ec56ed759a3d2876e821d970198cc30563f8d6 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 10 Sep 2026 21:55:23 -0700 Subject: [PATCH 09/27] test(e2e): rewrite the Redis timeout test as a locust chaos load test The sequential version sent one request at a time, so a Redis outage never reached the concurrency where the failed-tracking alert body actually grows. This drives the proxy with locust against one model group of three mock deployments, two failing at order 1 and one serving at order 2, so every request spends its retries on the failing pair and lands on the serving deployment through the order-based fallback. Two phases, a healthy baseline and a CLIENT PAUSE WRITE window, and every request must succeed in both. Latency, RSS and CPU are reported as p50/p90/p99 per phase rather than asserted on: RSS and CPU come from psutil on the proxy's process tree, since a multi-worker proxy serves /metrics from the prometheus multiprocess collector and that drops the process collector's series. Thresholds stay open until weekly runs give real baselines. Co-Authored-By: Claude Code --- .github/e2e-stack/select_tests.py | 1 - ...s-timeout.yml => test-e2e-redis-chaos.yml} | 21 +- pyproject.toml | 1 + tests/e2e/CLAUDE.md | 4 +- tests/e2e/CONTRIBUTING.md | 2 +- tests/e2e/conftest.py | 4 +- tests/e2e/coverage_registry/reliability.yaml | 2 +- tests/e2e/e2e_config.py | 2 +- ...i_config.yml => redis_chaos_ci_config.yml} | 11 +- tests/e2e/load/conftest.py | 22 +- tests/e2e/load/locust_load.py | 103 ++++++- tests/e2e/load/locustfile.py | 36 +++ tests/e2e/load/proxy_usage.py | 156 ++++++++++ tests/e2e/load/test_locust_load.py | 44 ++- tests/e2e/load/test_proxy_usage.py | 59 ++++ tests/e2e/load/test_redis_chaos_e2e.py | 258 ++++++++++++++++ tests/e2e/models.py | 1 + tests/e2e/pytest.ini | 2 +- tests/e2e/router/conftest.py | 12 - tests/e2e/router/test_redis_timeout_e2e.py | 282 ------------------ uv.lock | 4 +- 21 files changed, 673 insertions(+), 354 deletions(-) rename .github/workflows/{test-e2e-redis-timeout.yml => test-e2e-redis-chaos.yml} (82%) rename tests/e2e/gateway/{redis_timeout_ci_config.yml => redis_chaos_ci_config.yml} (59%) create mode 100644 tests/e2e/load/locustfile.py create mode 100644 tests/e2e/load/proxy_usage.py create mode 100644 tests/e2e/load/test_proxy_usage.py create mode 100644 tests/e2e/load/test_redis_chaos_e2e.py delete mode 100644 tests/e2e/router/test_redis_timeout_e2e.py diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py index 10f26bd3b8a..a62358f81ff 100644 --- a/.github/e2e-stack/select_tests.py +++ b/.github/e2e-stack/select_tests.py @@ -8,7 +8,6 @@ UNSUPPORTED: Final = re.compile( r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$" r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$" r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$" - r"|^tests/e2e/router/test_redis_timeout_e2e\.py$" ) HARNESS: Final = re.compile( r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" diff --git a/.github/workflows/test-e2e-redis-timeout.yml b/.github/workflows/test-e2e-redis-chaos.yml similarity index 82% rename from .github/workflows/test-e2e-redis-timeout.yml rename to .github/workflows/test-e2e-redis-chaos.yml index 501af0f240f..2ce20836db7 100644 --- a/.github/workflows/test-e2e-redis-timeout.yml +++ b/.github/workflows/test-e2e-redis-chaos.yml @@ -1,4 +1,4 @@ -name: "Weekly Redis Timeout E2E" +name: "Weekly Redis Chaos E2E" on: schedule: @@ -9,9 +9,9 @@ permissions: contents: read jobs: - redis-timeout-e2e: + redis-chaos-e2e: if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm' - runs-on: ubuntu-latest + runs-on: ubuntu-latest-16-cores timeout-minutes: 30 services: postgres: @@ -38,7 +38,7 @@ jobs: --health-retries 10 env: DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm - LITELLM_MASTER_KEY: sk-redis-timeout-e2e + LITELLM_MASTER_KEY: sk-redis-chaos-e2e LITELLM_LOG: WARNING steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 @@ -60,7 +60,7 @@ jobs: - name: Install dependencies run: | - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --group e2e-dev --extra proxy - name: Cache Prisma binaries uses: ./.github/actions/cache-prisma-binaries @@ -69,9 +69,10 @@ jobs: run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - - name: Start the proxy with a Redis that times out on every command + - name: Start a multi-worker proxy on the chaos config run: | - nohup uv run --no-sync litellm --config tests/e2e/gateway/redis_timeout_ci_config.yml --port 4000 > proxy.log 2>&1 & + nohup uv run --no-sync litellm --config tests/e2e/gateway/redis_chaos_ci_config.yml --port 4000 --num_workers 4 > proxy.log 2>&1 & + echo "E2E_PROXY_PID=$!" >> "$GITHUB_ENV" for _ in $(seq 1 90); do if curl -fs http://localhost:4000/health/liveliness > /dev/null; then exit 0 @@ -82,14 +83,14 @@ jobs: tail -n 100 proxy.log exit 1 - - name: Run the Redis timeout e2e test + - name: Run the Redis chaos load test env: - E2E_REDIS_TIMEOUT: "1" + E2E_REDIS_CHAOS: "1" LITELLM_PROXY_URL: http://localhost:4000 REDIS_HOST: 127.0.0.1 REDIS_PORT: "6379" run: | - uv run --no-sync pytest tests/e2e/router/test_redis_timeout_e2e.py -v --tb=short -rA + uv run --no-sync pytest tests/e2e/load/test_redis_chaos_e2e.py -v --tb=short -rA -s - name: Show proxy log on failure if: failure() diff --git a/pyproject.toml b/pyproject.toml index 04f2f3fd1dd..ae1698feb5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -220,6 +220,7 @@ e2e-dev = [ "playwright==1.61.0", "websockets>=15.0.1,<16.0", "locust==2.45.0", + "psutil==7.2.2", "mcp>=1.28.1,<2.0", ] proxy-dev = [ diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 76e8550cb9a..dc197996ac2 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -17,8 +17,8 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server (see "MCP suite: real Datadog only" below); plus the gateway-managed OAuth (authorization_code) path exercised through `/chat/completions`, the one behavior Datadog's static-header auth cannot reach, seeding the per-user upstream token via the interactive authorize dance driven with the mcp SDK's own OAuth client (headless-browser consent from a saved session) and asserting the completion lists and executes the server's tools with the stored per-user token - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection -- `router/` - routing and reliability behavior (fallbacks, cooldowns). Also holds `test_redis_timeout_e2e.py`, which needs a proxy booted from `gateway/redis_timeout_ci_config.yml` (real Redis, `socket_timeout: 0.001`, so every command times out); it is marked `redis_timeout`, deselected unless `E2E_REDIS_TIMEOUT` is set, excluded from the per-PR check, and driven weekly by `.github/workflows/test-e2e-redis-timeout.yml` -- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What remains here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`) and markerless harness unit tests for the Locust/session-anomaly aggregation logic +- `router/` - routing and reliability behavior (fallbacks, cooldowns) +- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments with `CLIENT PAUSE WRITE` on the proxy's Redis mid-run, asserting zero failed requests and reporting latency, RSS, and CPU as p50/p90/p99 per phase; needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven weekly by `.github/workflows/test-e2e-redis-chaos.yml`), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 7a859965c80..b1331de190a 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -65,7 +65,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, `guardrails/test_presidio_masking_e2e.py`, and `router/test_redis_timeout_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start, and the Redis timeout test needs a proxy whose Redis times out on every command (`gateway/redis_timeout_ci_config.yml`), which `.github/workflows/test-e2e-redis-timeout.yml` boots on a weekly schedule +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots on a weekly schedule Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index ff3d2dcc6bd..99a3f4967c3 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -60,8 +60,8 @@ def pytest_configure(config: pytest.Config) -> None: ) config.addinivalue_line( "markers", - "redis_timeout: needs a proxy booted from gateway/redis_timeout_ci_config.yml whose Redis times out on every " - "command; deselected unless E2E_REDIS_TIMEOUT is set", + "redis_chaos: load test that pauses the proxy's Redis writes mid-run; needs a proxy booted from " + "gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set", ) diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index a42e4b2779a..23163ab666c 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -31,7 +31,7 @@ - {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"} - {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix"} - {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"} -- {id: reliability.circuit_breaker.redis_timeout.stays_responsive, module: reliability, tier: P1, behavior: circuit_breaker, variant: redis_timeout, assertions: [stays_responsive], exercised_on: [chat_completions, responses, embeddings], source: "litellm/proxy/hooks/proxy_track_cost_callback.py:386", fail_before_fix: proven, rationale: "With every Redis command timing out and every request retrying then falling back, per-request latency stays flat, liveliness stays fast, and spend rows still land; on v1.100.0 the failed-tracking alert body doubled per request until the worker OOMed (LIT-6780)"} +- {id: reliability.circuit_breaker.redis_timeout.stays_responsive, module: reliability, tier: P1, behavior: circuit_breaker, variant: redis_timeout, assertions: [stays_responsive], exercised_on: [chat_completions], source: "litellm/proxy/hooks/proxy_track_cost_callback.py:386", fail_before_fix: proven, rationale: "Under locust load with every request retrying through failing mock deployments, pausing Redis writes mid-run trips the breaker and every request still succeeds, with latency, RSS, and CPU reported as p50/p90/p99 against the pre-pause baseline; on v1.100.0 the failed-tracking alert body doubled per request until the worker OOMed (LIT-6780)"} - {id: reliability.timeout.request_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: request_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions, messages], source: "litellm/router.py:545-551", rationale: "Per-request timeout raises Timeout"} - {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"} - {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 6cb59ea4e90..7344a2b6cec 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -143,7 +143,7 @@ LOAD_MIN_CONCURRENCY_EFFICIENCY = float(os.environ.get("E2E_LOAD_MIN_CONCURRENCY WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY" MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK" -REDIS_TIMEOUT_OPT_IN_ENV = "E2E_REDIS_TIMEOUT" +REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) diff --git a/tests/e2e/gateway/redis_timeout_ci_config.yml b/tests/e2e/gateway/redis_chaos_ci_config.yml similarity index 59% rename from tests/e2e/gateway/redis_timeout_ci_config.yml rename to tests/e2e/gateway/redis_chaos_ci_config.yml index fd283dbc7df..37b024502f7 100644 --- a/tests/e2e/gateway/redis_timeout_ci_config.yml +++ b/tests/e2e/gateway/redis_chaos_ci_config.yml @@ -5,17 +5,14 @@ general_settings: litellm_settings: callbacks: ["prometheus"] require_auth_for_metrics_endpoint: false + enable_redis_auth_cache: true cache: true cache_params: type: redis host: 127.0.0.1 port: 6379 - socket_timeout: 0.001 + socket_timeout: 0.1 router_settings: - num_retries: 1 - fallbacks: - - redis-timeout-primary: - - redis-timeout-backup - - redis-timeout-embed-primary: - - redis-timeout-embed-backup + num_retries: 2 + disable_cooldowns: true diff --git a/tests/e2e/load/conftest.py b/tests/e2e/load/conftest.py index 3a926ef2a61..e6f3c7aa538 100644 --- a/tests/e2e/load/conftest.py +++ b/tests/e2e/load/conftest.py @@ -4,25 +4,23 @@ import os import pytest -from e2e_config import WEEKLY_ANOMALY_OPT_IN_ENV +from e2e_config import REDIS_CHAOS_OPT_IN_ENV, WEEKLY_ANOMALY_OPT_IN_ENV from load_client import LoadClient, build_client from proxy_client import ProxyClient +_OPT_IN_MARKERS = ( + ("weekly", WEEKLY_ANOMALY_OPT_IN_ENV), + ("redis_chaos", REDIS_CHAOS_OPT_IN_ENV), +) -def pytest_collection_modifyitems( - config: pytest.Config, items: list[pytest.Item] -) -> None: - if os.environ.get(WEEKLY_ANOMALY_OPT_IN_ENV): - return - deselected = [ - item for item in items if item.get_closest_marker("weekly") is not None - ] + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + opted_out = {marker for marker, opt_in_env in _OPT_IN_MARKERS if not os.environ.get(opt_in_env)} + deselected = [item for item in items if any(item.get_closest_marker(marker) is not None for marker in opted_out)] if not deselected: return config.hook.pytest_deselected(items=deselected) - items[:] = [ - item for item in items if item.get_closest_marker("weekly") is None - ] + items[:] = [item for item in items if item not in deselected] @pytest.fixture(scope="session") diff --git a/tests/e2e/load/locust_load.py b/tests/e2e/load/locust_load.py index e0da8ba70c2..a1668c5bfc5 100644 --- a/tests/e2e/load/locust_load.py +++ b/tests/e2e/load/locust_load.py @@ -1,12 +1,19 @@ from __future__ import annotations import csv +import os +import subprocess +import sys +import tempfile from dataclasses import dataclass from itertools import accumulate from pathlib import Path +from typing import Final from pydantic import BaseModel, TypeAdapter +_LOCUSTFILE = Path(__file__).with_name("locustfile.py") +_CSV_PREFIX = "locust" _GENERATOR_SATURATION_MARKER = "CPU usage above" _MAX_REPORTED_ERRORS = 5 @@ -34,7 +41,9 @@ class LoadResult: requests: int failures: int requests_per_second: float - median_response_seconds: float + p50_seconds: float + p90_seconds: float + p99_seconds: float errors: tuple[LoadError, ...] generator_warnings: tuple[str, ...] @@ -53,18 +62,24 @@ class LoadResult: lines.append("locust recorded no error breakdown") return "; ".join((*lines, *self.generator_warnings)) + def latency_summary(self) -> str: + return f"p50 {self.p50_seconds:.3f}s, p90 {self.p90_seconds:.3f}s, p99 {self.p99_seconds:.3f}s" -def median_seconds(entries: list[LocustStatEntry]) -> float: - samples = sorted( - (milliseconds, count) for entry in entries for milliseconds, count in entry.response_times.items() - ) + +def percentile_seconds(entries: list[LocustStatEntry], fraction: float) -> float: + """The response time at `fraction` of the merged histograms, in seconds. + + Locust buckets response times by millisecond, so this reads the first bucket whose + running count reaches the rank, the same lower-sample convention locust's own + percentiles use. + """ + samples = sorted((milliseconds, count) for entry in entries for milliseconds, count in entry.response_times.items()) total = sum(count for _, count in samples) if total == 0: return 0.0 running = accumulate(count for _, count in samples) - return next( - milliseconds for (milliseconds, _), seen in zip(samples, running) if seen >= total / 2 - ) / 1000.0 + rank: Final = total * fraction + return next(milliseconds for (milliseconds, _), seen in zip(samples, running) if seen >= rank) / 1000.0 def aggregate_stats( @@ -79,7 +94,9 @@ def aggregate_stats( requests=requests, failures=failures, requests_per_second=0.0, - median_response_seconds=0.0, + p50_seconds=0.0, + p90_seconds=0.0, + p99_seconds=0.0, errors=errors, generator_warnings=generator_warnings, ) @@ -88,7 +105,9 @@ def aggregate_stats( requests=requests, failures=failures, requests_per_second=requests / elapsed if elapsed > 0 else 0.0, - median_response_seconds=median_seconds(entries), + p50_seconds=percentile_seconds(entries, 0.5), + p90_seconds=percentile_seconds(entries, 0.9), + p99_seconds=percentile_seconds(entries, 0.99), errors=errors, generator_warnings=generator_warnings, ) @@ -121,3 +140,67 @@ def read_generator_warnings(stderr: str) -> tuple[str, ...]: if _GENERATOR_SATURATION_MARKER in line ) return tuple(dict.fromkeys(saturated)) + + +def run_chat_load( + *, + base_url: str, + api_keys: tuple[str, ...], + model: str, + users: int, + spawn_rate: float, + duration_seconds: float, +) -> LoadResult: + """Drive /chat/completions from headless locust and aggregate what it reported. + + Each simulated user picks one of `api_keys`, so auth and budget lookups spread over a + pool of virtual keys instead of keeping one key's cache entry permanently warm. + """ + with tempfile.TemporaryDirectory(prefix="e2e-load-") as report_dir: + csv_prefix = Path(report_dir) / _CSV_PREFIX + completed = subprocess.run( + [ + sys.executable, + "-m", + "locust", + "--headless", + "--json", + "--csv", + str(csv_prefix), + "--locustfile", + str(_LOCUSTFILE), + "--host", + base_url, + "--users", + str(users), + "--spawn-rate", + str(spawn_rate), + "--run-time", + f"{int(duration_seconds)}s", + "--exit-code-on-error", + "0", + ], + env={**os.environ, "LOAD_API_KEYS": ",".join(api_keys), "LOAD_MODEL": model}, + capture_output=True, + text=True, + timeout=duration_seconds + 120, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + f"locust exited {completed.returncode} before it could report throughput " + f"(a startup failure, not request failures, which are folded into the JSON summary via " + f"--exit-code-on-error 0):\n{completed.stderr}" + ) + try: + entries = _STATS_ADAPTER.validate_json(completed.stdout) + except ValueError as exc: + raise RuntimeError( + f"locust exited 0 but did not print a parseable --json throughput summary on stdout; " + f"got stdout={completed.stdout!r}, stderr={completed.stderr!r}" + ) from exc + return aggregate_stats( + entries, + read_errors(csv_prefix.with_name(f"{_CSV_PREFIX}_failures.csv")), + read_generator_warnings(completed.stderr), + ) diff --git a/tests/e2e/load/locustfile.py b/tests/e2e/load/locustfile.py new file mode 100644 index 00000000000..3d9ef7c83ea --- /dev/null +++ b/tests/e2e/load/locustfile.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import os +import random +import uuid +from typing import Final + +from locust import FastHttpUser, constant, task + +_MODEL: Final = os.environ["LOAD_MODEL"] +_API_KEYS: Final = tuple(os.environ["LOAD_API_KEYS"].split(",")) + + +def _payload() -> dict[str, object]: + """A prompt no other request sent, so the response cache never answers for the deployment.""" + return { + "model": _MODEL, + "messages": [{"role": "user", "content": f"load test ping {uuid.uuid4().hex}"}], + "max_tokens": 16, + } + + +class ChatUser(FastHttpUser): + wait_time = constant(0) + + def on_start(self) -> None: + self.headers = {"Authorization": f"Bearer {random.choice(_API_KEYS)}"} + + @task + def chat(self) -> None: + self.client.post( # pyright: ignore[reportUnknownMemberType] # locust FastHttpSession.post types json/**kwargs as Any + "/chat/completions", + json=_payload(), + headers=self.headers, + name="/chat/completions", + ) diff --git a/tests/e2e/load/proxy_usage.py b/tests/e2e/load/proxy_usage.py new file mode 100644 index 00000000000..b48129f0099 --- /dev/null +++ b/tests/e2e/load/proxy_usage.py @@ -0,0 +1,156 @@ +"""Resident memory and CPU of the proxy process tree, sampled on a background thread. + +The proxy under load runs several worker processes, and `/metrics` cannot report their +memory: litellm sets PROMETHEUS_MULTIPROC_DIR when num_workers > 1, and the multiprocess +collector drops the process collector's `process_resident_memory_bytes` / +`process_cpu_seconds_total` entirely. So the test measures the tree itself through psutil, +which needs the proxy to run on the same host as the test. +""" + +from __future__ import annotations + +import math +import threading +import time +from dataclasses import dataclass +from typing import Final + +import psutil +from pydantic import BaseModel, ConfigDict + + +class _MemoryInfo(BaseModel): + model_config = ConfigDict(from_attributes=True) + + rss: int + + +@dataclass(frozen=True, slots=True) +class UsageSample: + elapsed_seconds: float + rss_bytes: int + cpu_seconds: float + + +@dataclass(frozen=True, slots=True) +class UsageWindow: + """The samples taken across one phase, plus what they say about that phase.""" + + samples: tuple[UsageSample, ...] + + def rss_percentile(self, fraction: float) -> int: + if not self.samples: + return 0 + ordered: Final = sorted(sample.rss_bytes for sample in self.samples) + return ordered[_rank(len(ordered), fraction)] + + def cpu_seconds_consumed(self) -> float: + """CPU seconds the tree burned across the window, from its monotonic counter.""" + if len(self.samples) < 2: + return 0.0 + return self.samples[-1].cpu_seconds - self.samples[0].cpu_seconds + + def cpu_utilization_percentiles(self) -> tuple[float, float, float]: + """Per-interval CPU utilization (cores busy) at p50, p90 and p99. + + Derived from consecutive samples of the cumulative counter rather than + psutil's own cpu_percent, so it covers every process in the tree including + workers that came and went between samples. + """ + rates: Final = sorted( + (later.cpu_seconds - earlier.cpu_seconds) / (later.elapsed_seconds - earlier.elapsed_seconds) + for earlier, later in zip(self.samples, self.samples[1:]) + if later.elapsed_seconds > earlier.elapsed_seconds + ) + if not rates: + return 0.0, 0.0, 0.0 + return ( + rates[_rank(len(rates), 0.5)], + rates[_rank(len(rates), 0.9)], + rates[_rank(len(rates), 0.99)], + ) + + def summary(self) -> str: + p50_cpu, p90_cpu, p99_cpu = self.cpu_utilization_percentiles() + return ( + f"RSS p50 {self.rss_percentile(0.5) / 2**20:.0f} MB, " + f"p90 {self.rss_percentile(0.9) / 2**20:.0f} MB, " + f"p99 {self.rss_percentile(0.99) / 2**20:.0f} MB; " + f"CPU cores busy p50 {p50_cpu:.2f}, p90 {p90_cpu:.2f}, p99 {p99_cpu:.2f}; " + f"{self.cpu_seconds_consumed():.1f} CPU seconds consumed" + ) + + +def _rank(count: int, fraction: float) -> int: + """Index of the sample at `fraction`, the same lower-sample convention as locust's percentiles.""" + return min(count - 1, max(0, math.ceil(count * fraction) - 1)) + + +def _read_process(process: psutil.Process) -> tuple[int, float] | None: + try: + with process.oneshot(): + memory: Final = _MemoryInfo.model_validate(process.memory_info()) + times: Final = process.cpu_times() + return memory.rss, times.user + times.system + except (psutil.NoSuchProcess, psutil.AccessDenied): + return None + + +class ProxyUsageSampler: + """Samples the proxy process tree every `interval_seconds` until stopped. + + `split()` returns the samples taken so far and starts a new window, so one sampler + covers a baseline phase and a chaos phase without a gap between them. + """ + + def __init__(self, pid: int, interval_seconds: float = 1.0) -> None: + self._process: Final = psutil.Process(pid) + self._interval: Final = interval_seconds + self._stop: Final = threading.Event() + self._lock: Final = threading.Lock() + self._samples: list[UsageSample] = [] # mutable-ok: a sampling buffer the reader drains under a lock + self._started: Final = time.monotonic() + self._thread: Final = threading.Thread(target=self._run, name="proxy-usage-sampler", daemon=True) + + def __enter__(self) -> ProxyUsageSampler: + self._thread.start() + return self + + def __exit__(self, *_: object) -> None: + self._stop.set() + self._thread.join(timeout=self._interval * 5) + + def _tree(self) -> tuple[psutil.Process, ...]: + try: + return (self._process, *self._process.children(recursive=True)) + except psutil.NoSuchProcess: + return () + + def _sample(self) -> UsageSample | None: + readings: Final = tuple(reading for process in self._tree() if (reading := _read_process(process)) is not None) + if not readings: + return None + return UsageSample( + elapsed_seconds=time.monotonic() - self._started, + rss_bytes=sum(rss for rss, _ in readings), + cpu_seconds=sum(cpu for _, cpu in readings), + ) + + def _run(self) -> None: + while not self._stop.is_set(): + sample = self._sample() + if sample is not None: + with self._lock: + self._samples.append(sample) + self._stop.wait(self._interval) + + def split(self) -> UsageWindow: + """The window that ends now; the next one starts from this window's last sample. + + The boundary sample is carried into the next window so its CPU counter has a + starting point, which is what makes the two windows' utilization comparable. + """ + with self._lock: + taken = tuple(self._samples) + self._samples = [taken[-1]] if taken else [] # rebind-ok: drains the buffer under the lock + return UsageWindow(samples=taken) diff --git a/tests/e2e/load/test_locust_load.py b/tests/e2e/load/test_locust_load.py index e3cc6f3efd5..40a238ef624 100644 --- a/tests/e2e/load/test_locust_load.py +++ b/tests/e2e/load/test_locust_load.py @@ -7,7 +7,7 @@ from locust_load import ( LoadResult, LocustStatEntry, aggregate_stats, - median_seconds, + percentile_seconds, read_errors, read_generator_warnings, ) @@ -41,35 +41,47 @@ def _result( requests=10, failures=10, requests_per_second=1.0, - median_response_seconds=0.05, + p50_seconds=0.05, + p90_seconds=0.08, + p99_seconds=0.1, errors=errors, generator_warnings=generator_warnings, ) -class TestSerialLatency: +class TestPercentiles: def test_median_is_the_middle_sample_not_the_mean_a_slow_tail_would_drag(self) -> None: # Nine fast requests and one very slow one: the mean is 1.99s, the median is 20ms. entry = _entry(num_requests=10, response_times={20: 9, 20000: 1}) - assert median_seconds([entry]) == 0.02 + assert percentile_seconds([entry], 0.5) == 0.02 - def test_median_merges_the_histograms_of_every_stats_entry(self) -> None: + def test_the_tail_percentiles_reach_the_slow_samples_the_median_hides(self) -> None: + # 100 samples: 89 fast, 10 slow, 1 very slow. p50 sits in the fast bucket, p90 in the + # slow one, and p99 lands on the single very slow sample. + entry = _entry(num_requests=100, response_times={20: 89, 500: 10, 20000: 1}) + + assert percentile_seconds([entry], 0.5) == 0.02 + assert percentile_seconds([entry], 0.9) == 0.5 + assert percentile_seconds([entry], 0.99) == 0.5 + assert percentile_seconds([entry], 1.0) == 20.0 + + def test_percentiles_merge_the_histograms_of_every_stats_entry(self) -> None: # Per entry the median would be 10ms and 90ms; merged, the middle of the five samples is 90ms. entries = [ _entry(num_requests=2, response_times={10: 2}), _entry(num_requests=3, response_times={90: 3}), ] - assert median_seconds(entries) == 0.09 + assert percentile_seconds(entries, 0.5) == 0.09 def test_an_even_split_takes_the_lower_middle_sample_as_locust_itself_does(self) -> None: entry = _entry(num_requests=4, response_times={10: 2, 90: 2}) - assert median_seconds([entry]) == 0.01 + assert percentile_seconds([entry], 0.5) == 0.01 def test_no_samples_reports_zero_rather_than_dividing_by_an_empty_histogram(self) -> None: - assert median_seconds([]) == 0.0 + assert percentile_seconds([], 0.5) == 0.0 class TestAggregate: @@ -84,9 +96,20 @@ class TestAggregate: result = aggregate_stats([entry], (), ()) assert result.requests_per_second == 3.0 - assert result.median_response_seconds == 0.057 + assert result.p50_seconds == 0.057 + assert result.p99_seconds == 0.057 assert result.failure_ratio == 0.0 + def test_tail_percentiles_come_from_the_slow_end_of_the_histogram(self) -> None: + entry = _entry(num_requests=100, response_times={20: 89, 500: 10, 3000: 1}) + + result = aggregate_stats([entry], (), ()) + + assert result.p50_seconds == 0.02 + assert result.p90_seconds == 0.5 + assert result.p99_seconds == 0.5 + assert result.latency_summary() == "p50 0.020s, p90 0.500s, p99 0.500s" + def test_throughput_spans_from_the_earliest_start_when_locust_reports_several_entries(self) -> None: entries = [ _entry(num_requests=60, start_time=1000.0, last_request_timestamp=1030.0), @@ -133,8 +156,7 @@ class TestErrorBreakdown: def test_diagnosis_caps_the_list_and_says_how_many_it_left_out(self) -> None: result = _result( errors=tuple( - LoadError(name="/chat/completions", error=f"error-{index}", occurrences=index) - for index in range(1, 9) + LoadError(name="/chat/completions", error=f"error-{index}", occurrences=index) for index in range(1, 9) ) ) diff --git a/tests/e2e/load/test_proxy_usage.py b/tests/e2e/load/test_proxy_usage.py new file mode 100644 index 00000000000..8d9f848825f --- /dev/null +++ b/tests/e2e/load/test_proxy_usage.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from typing import Final + +from proxy_usage import UsageSample, UsageWindow + +_MB: Final = 2**20 + + +def _window(*points: tuple[float, int, float]) -> UsageWindow: + return UsageWindow( + samples=tuple( + UsageSample(elapsed_seconds=elapsed, rss_bytes=rss, cpu_seconds=cpu) for elapsed, rss, cpu in points + ) + ) + + +class TestRssPercentiles: + def test_the_tail_percentiles_reach_the_peak_the_median_hides(self) -> None: + # 100 one-second samples: 89 flat, 10 elevated, 1 spike. The median stays flat, p90 sees the + # elevated plateau, and only the max reaches the spike. + window: Final = _window( + *((float(i), 100 * _MB, float(i)) for i in range(89)), + *((float(89 + i), 300 * _MB, float(89 + i)) for i in range(10)), + (99.0, 900 * _MB, 99.0), + ) + + assert window.rss_percentile(0.5) == 100 * _MB + assert window.rss_percentile(0.9) == 300 * _MB + assert window.rss_percentile(0.99) == 300 * _MB + assert window.rss_percentile(1.0) == 900 * _MB + + def test_an_empty_window_reports_zero_rather_than_indexing_nothing(self) -> None: + assert _window().rss_percentile(0.5) == 0 + + +class TestCpuUtilization: + def test_utilization_is_the_counter_delta_over_the_interval_not_the_counter_itself(self) -> None: + # The counter climbs 0.5 CPU seconds per second, then 4.0 per second: half a core, then four. + window: Final = _window((0.0, _MB, 0.0), (1.0, _MB, 0.5), (2.0, _MB, 1.0), (3.0, _MB, 5.0)) + + p50, p90, p99 = window.cpu_utilization_percentiles() + + assert (p50, p90, p99) == (0.5, 4.0, 4.0) + assert window.cpu_seconds_consumed() == 5.0 + + def test_a_single_sample_has_no_interval_and_reports_zero(self) -> None: + window: Final = _window((0.0, _MB, 3.0)) + + assert window.cpu_utilization_percentiles() == (0.0, 0.0, 0.0) + assert window.cpu_seconds_consumed() == 0.0 + + def test_summary_reports_every_percentile_in_human_units(self) -> None: + window: Final = _window((0.0, 200 * _MB, 0.0), (1.0, 200 * _MB, 1.5), (2.0, 200 * _MB, 3.0)) + + assert window.summary() == ( + "RSS p50 200 MB, p90 200 MB, p99 200 MB; " + "CPU cores busy p50 1.50, p90 1.50, p99 1.50; 3.0 CPU seconds consumed" + ) diff --git a/tests/e2e/load/test_redis_chaos_e2e.py b/tests/e2e/load/test_redis_chaos_e2e.py new file mode 100644 index 00000000000..f91b9c2fa09 --- /dev/null +++ b/tests/e2e/load/test_redis_chaos_e2e.py @@ -0,0 +1,258 @@ +"""Live e2e: the proxy under load keeps serving every request while Redis writes time out. + +Runs against a proxy booted from tests/e2e/gateway/redis_chaos_ci_config.yml, which points +cache_params at a real Redis with litellm's default socket_timeout. That one client backs all +three Redis touchpoints on the request path: the virtual-key auth cache, the response cache, +and the cross-pod spend counter the cost-tracking callback awaits. + +The load runs in two phases against one model group of three mock deployments. The two at +order 1 raise InternalServerError and the one at order 2 serves, so every request burns its +retries on the failing pair (a 500 is retryable, so retries keep re-picking inside the lowest +order) and the router's order-based fallback then re-targets order 2. Every request is expected +to succeed, and each one carries retry breadcrumbs into cost tracking. + +Phase A is a baseline with Redis healthy; phase B holds Redis in CLIENT PAUSE WRITE, so the +spend counter increment times out and the callback stringifies the request metadata, +breadcrumbs included, into a failed-tracking alert. On v1.100.0 that string doubled per request +until the worker hung (LIT-6780), which is what the per-phase RSS and CPU percentiles are here +to catch. + +Needs the proxy on the same host, since RSS and CPU come from psutil on its process tree: +a multi-worker proxy serves /metrics from the prometheus multiprocess collector, which drops +the process collector's memory and CPU series. Deselected unless E2E_REDIS_CHAOS is set. +""" + +from __future__ import annotations + +import os +import re +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Final + +import pytest +import redis +from e2e_config import PROXY_BASE_URL, unique_marker +from e2e_http import NoBody +from lifecycle import ResourceManager +from load_client import LoadClient +from locust_load import LoadResult, run_chat_load +from models import KeyGenerateBody, LiteLLMParamsBody +from proxy_client import ProxyClient +from proxy_usage import ProxyUsageSampler, UsageWindow + +pytestmark = [pytest.mark.e2e, pytest.mark.redis_chaos] + +MODEL_GROUP: Final = "redis-chaos-fable" +MOCK_MODEL: Final = "anthropic/claude-fable-5-1" +FAILING_DEPLOYMENTS: Final = 2 +SERVING_DEPLOYMENTS: Final = 1 +FAILING_ORDER: Final = 1 +SERVING_ORDER: Final = 2 +KEY_POOL_SIZE: Final = 8 +LOCUST_USERS: Final = 50 +LOCUST_SPAWN_RATE: Final = 50.0 +BASELINE_SECONDS: Final = 60.0 +CHAOS_SECONDS: Final = 90.0 +REDIS_PAUSE_MS: Final = 600_000 +BASELINE_TIMEOUT_RATE_CEILING: Final = 0.05 +CHAOS_TIMEOUT_RATE_FLOOR: Final = 0.20 + +TIMEOUT_FAILURES_RE: Final = re.compile( + r'^litellm_redis_circuit_breaker_failures_total\{failure_class="timeout"\} ([0-9.e+]+)$', re.M +) +BREAKER_OPEN_RE: Final = re.compile(r'^litellm_redis_circuit_breaker_state\{state="open"\} ([0-9.e+]+)$', re.M) +BREAKER_TRANSITIONS_RE: Final = re.compile( + r'^litellm_redis_circuit_breaker_transitions_total\{state="[a-z_]+"\} ([0-9.e+]+)$', re.M +) +RETRIES_RE: Final = re.compile(r"^litellm_deployment_failure_responses_total\{[^}]*\} ([0-9.e+]+)$", re.M) +COOLDOWN_RE: Final = re.compile(r"^litellm_deployment_cooled_down_total\{[^}]*\} ([0-9.e+]+)$", re.M) + + +@dataclass(frozen=True, slots=True) +class Phase: + """One load phase's traffic and what the proxy's process tree did during it.""" + + name: str + load: LoadResult + usage: UsageWindow + + def report(self) -> str: + return ( + f"{self.name}: {self.load.requests} requests, {self.load.failures} failures, " + f"{self.load.requests_per_second:.0f} rps, {self.load.latency_summary()}; {self.usage.summary()}" + ) + + +def _failing_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=MOCK_MODEL, + api_key="sk-redis-chaos-not-used", + mock_response="litellm.InternalServerError", + order=FAILING_ORDER, + ) + + +def _serving_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=MOCK_MODEL, + api_key="sk-redis-chaos-not-used", + mock_response="redis chaos ok", + order=SERVING_ORDER, + ) + + +@pytest.fixture +def proxy_pid() -> int: + """The proxy's PID, which the workflow exports after starting it. + + Required rather than discovered: picking a process out of the table by name would be + ambiguous on a developer machine running more than one proxy. + """ + pid: Final = os.environ.get("E2E_PROXY_PID") + assert pid and pid.isdigit(), ( + "E2E_PROXY_PID must hold the PID of the proxy under test; RSS and CPU are read from " + "its process tree because a multi-worker proxy does not report them on /metrics" + ) + return int(pid) + + +@pytest.fixture +def redis_control() -> Iterator[redis.Redis[bytes]]: + """A control connection to the proxy's Redis, which unpauses writes in teardown. + + Only writes are paused: CLIENT PAUSE ALL would freeze this connection too, leaving + nothing able to lift the pause. + """ + host: Final = os.environ.get("REDIS_HOST") + port: Final = os.environ.get("REDIS_PORT") + assert host and port, "REDIS_HOST and REDIS_PORT must name the Redis the proxy under test uses" + control: Final = redis.Redis(host=host, port=int(port), socket_timeout=5) + try: + yield control + finally: + control.client_unpause() # pyright: ignore[reportUnknownMemberType] # redis-py stubs return Any + control.close() + + +def _metric(proxy: ProxyClient, pattern: re.Pattern[str]) -> float: + body: Final = proxy.probe("/metrics", params=NoBody()).body + return sum(float(match.group(1)) for match in pattern.finditer(body)) + + +def _register_deployments(proxy: ProxyClient, resources: ResourceManager) -> None: + for _ in range(FAILING_DEPLOYMENTS): + failing_id = proxy.create_model(MODEL_GROUP, _failing_params()) + resources.defer(lambda model_id=failing_id: proxy.delete_model(model_id)) + for _ in range(SERVING_DEPLOYMENTS): + serving_id = proxy.create_model(MODEL_GROUP, _serving_params()) + resources.defer(lambda model_id=serving_id: proxy.delete_model(model_id)) + + +def _generate_key_pool(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, ...]: + """A pool of virtual keys so auth and budget lookups are not one permanently warm + cache entry; each locust user picks one, so Redis auth reads actually happen.""" + keys: Final = tuple( + proxy.generate_key( + KeyGenerateBody(models=[MODEL_GROUP], key_alias=f"e2e-redis-chaos-{unique_marker()}-{index}") + ) + for index in range(KEY_POOL_SIZE) + ) + for key in keys: + resources.defer(lambda doomed=key: proxy.delete_key(doomed)) + return keys + + +def _drive(keys: tuple[str, ...], seconds: float) -> LoadResult: + return run_chat_load( + base_url=PROXY_BASE_URL, + api_keys=keys, + model=MODEL_GROUP, + users=LOCUST_USERS, + spawn_rate=LOCUST_SPAWN_RATE, + duration_seconds=seconds, + ) + + +class TestRedisChaos: + @pytest.mark.covers( + "reliability.circuit_breaker.redis_timeout.stays_responsive", + exercised_on=["chat_completions"], + ) + def test_load_survives_redis_write_timeouts( + self, + client: LoadClient, + resources: ResourceManager, + proxy_pid: int, + redis_control: redis.Redis[bytes], + ) -> None: + proxy: Final = client.proxy + _register_deployments(proxy, resources) + keys: Final = _generate_key_pool(proxy, resources) + + timeouts_at_start: Final = _metric(proxy, TIMEOUT_FAILURES_RE) + retries_before: Final = _metric(proxy, RETRIES_RE) + cooldowns_before: Final = _metric(proxy, COOLDOWN_RE) + + with ProxyUsageSampler(proxy_pid) as sampler: + baseline: Final = Phase(name="baseline", load=_drive(keys, BASELINE_SECONDS), usage=sampler.split()) + timeouts_after_baseline: Final = _metric(proxy, TIMEOUT_FAILURES_RE) + + redis_control.client_pause(REDIS_PAUSE_MS, all=False) # pyright: ignore[reportUnknownMemberType] # redis-py stubs return Any + chaos: Final = Phase(name="chaos", load=_drive(keys, CHAOS_SECONDS), usage=sampler.split()) + + report: Final = f"{baseline.report()} | {chaos.report()}" + + for phase in (baseline, chaos): + assert phase.load.requests > 0, ( + f"{phase.name} drove no traffic at all, so it proved nothing: {phase.load.diagnosis()}. {report}" + ) + assert phase.load.failures == 0, ( + f"{phase.name} had {phase.load.failures} of {phase.load.requests} requests fail. Every request " + f"must succeed: the failing deployments sit at order {FAILING_ORDER} and the serving one at order " + f"{SERVING_ORDER}, so once the retries on order {FAILING_ORDER} are spent the order-based fallback " + f"lands on the serving deployment. Failures mean it was cooled down, the fallback did not run, or " + f"a Redis failure reached the response path. {phase.load.diagnosis()}. {report}" + ) + + cooldowns: Final = _metric(proxy, COOLDOWN_RE) - cooldowns_before + assert cooldowns == 0, ( + f"{cooldowns:.0f} deployments were cooled down during the run; the failing deployments are supposed " + f"to stay in rotation so every request keeps exercising the retry path. {report}" + ) + + retries: Final = _metric(proxy, RETRIES_RE) - retries_before + assert retries >= baseline.load.requests + chaos.load.requests, ( + f"only {retries:.0f} deployment failures were counted across " + f"{baseline.load.requests + chaos.load.requests} requests; the mock deployments did not fail, so no " + f"request carried retry breadcrumbs into cost tracking and the regression path was never entered. " + f"{report}" + ) + + baseline_timeout_rate: Final = (timeouts_after_baseline - timeouts_at_start) / baseline.load.requests + assert baseline_timeout_rate <= BASELINE_TIMEOUT_RATE_CEILING, ( + f"a healthy Redis timed out on {baseline_timeout_rate:.1%} of baseline requests, over the " + f"{BASELINE_TIMEOUT_RATE_CEILING:.0%} this test tolerates; at litellm's default socket_timeout a " + f"loaded Redis does time out occasionally, but this much means the baseline is already degraded and " + f"the two phases are not comparable. {report}" + ) + + chaos_timeouts: Final = _metric(proxy, TIMEOUT_FAILURES_RE) - timeouts_after_baseline + chaos_timeout_rate: Final = chaos_timeouts / chaos.load.requests + transitions: Final = _metric(proxy, BREAKER_TRANSITIONS_RE) + breaker_open: Final = _metric(proxy, BREAKER_OPEN_RE) >= 1 + assert chaos_timeout_rate >= CHAOS_TIMEOUT_RATE_FLOOR or transitions >= 1 or breaker_open, ( + f"with writes paused the breaker saw Redis time out on only {chaos_timeout_rate:.1%} of requests " + f"against {baseline_timeout_rate:.1%} at baseline, under the {CHAOS_TIMEOUT_RATE_FLOOR:.0%} a real " + f"outage produces, and it counted {transitions:.0f} state transitions and ended " + f"{'open' if breaker_open else 'closed'}. The spend counter increment never failed, so this run " + f"proved nothing. {report}" + ) + + rows: Final = proxy.poll_logs_for_key(keys[0], min_rows=1) + assert rows, ( + f"no spend rows landed for the first key in the pool; a Redis outage must not cost the proxy its " + f"spend logs, which are written to Postgres through a queue rather than through Redis. {report}" + ) + + print(f"\nredis chaos load: {report}") # noqa: T201 # the numbers this test exists to report, read off the CI log diff --git a/tests/e2e/models.py b/tests/e2e/models.py index f3c111bdf06..31eb6e31401 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -924,6 +924,7 @@ class LiteLLMParamsBody(BaseModel): timeout: float | None = None tpm: int | None = None weight: int | None = None + order: int | None = None ModelMode = Literal["batch", "realtime", "image_generation"] diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index f69d5d058f3..1dcb50a6fab 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -9,4 +9,4 @@ markers = load: heavy throughput/load test; collected last so it never perturbs latency-sensitive suites weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set - redis_timeout: needs a proxy booted from gateway/redis_timeout_ci_config.yml whose Redis times out on every command; deselected unless E2E_REDIS_TIMEOUT is set + redis_chaos: load test that pauses the proxy's Redis writes mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set diff --git a/tests/e2e/router/conftest.py b/tests/e2e/router/conftest.py index 57fde729f41..2d5e546dc6c 100644 --- a/tests/e2e/router/conftest.py +++ b/tests/e2e/router/conftest.py @@ -10,12 +10,10 @@ proxy does not already list it (compose has it in static config; stage does not) from __future__ import annotations -import os from collections.abc import Iterator import pytest from complexity_router_client import ComplexityRouterClient, build_client -from e2e_config import REDIS_TIMEOUT_OPT_IN_ENV from e2e_http import NoBody, Success from lifecycle import ResourceManager from models import ( @@ -46,16 +44,6 @@ ROUTER_PARAMS = LiteLLMParamsBody( ROUTER_KEY_MODELS = [ROUTER_MODEL, "gpt-5.5", "claude-haiku-4-5"] -def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: - if os.environ.get(REDIS_TIMEOUT_OPT_IN_ENV): - return - deselected = [item for item in items if item.get_closest_marker("redis_timeout") is not None] - if not deselected: - return - config.hook.pytest_deselected(items=deselected) - items[:] = [item for item in items if item.get_closest_marker("redis_timeout") is None] - - @pytest.fixture(scope="session") def client(proxy: ProxyClient) -> ComplexityRouterClient: return build_client(proxy) diff --git a/tests/e2e/router/test_redis_timeout_e2e.py b/tests/e2e/router/test_redis_timeout_e2e.py deleted file mode 100644 index 3f1984c0696..00000000000 --- a/tests/e2e/router/test_redis_timeout_e2e.py +++ /dev/null @@ -1,282 +0,0 @@ -"""Live e2e: the proxy keeps answering while every Redis command times out. - -Runs only against a proxy booted from tests/e2e/gateway/redis_timeout_ci_config.yml, which -points cache_params at a real Redis with socket_timeout 0.001. The test holds that Redis in -CLIENT PAUSE WRITE for its duration, so every write the proxy sends, the spend counter increment -included, hangs past the timeout. For each endpoint it registers two deployments through -/model/new: a primary whose api_base is a closed port and a backup that answers with a mock. Each request fails the primary, -retries, falls back and succeeds, so it carries retry breadcrumbs; its cost tracking then fails on -the spend counter increment and stringifies the request metadata into a failed-tracking alert. On -v1.100.0 that string doubled per request until the worker hung (LIT-6780). Deselected unless -E2E_REDIS_TIMEOUT is set, since it needs that dedicated proxy. -""" - -from __future__ import annotations - -import os -import re -import time -from collections.abc import Callable, Iterator -from dataclasses import dataclass -from typing import Final - -import pytest -import redis -from complexity_router_client import ComplexityRouterClient -from e2e_config import unique_marker -from e2e_http import NoBody, Result, Success -from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, ChatResponse, EmbedBody, KeyGenerateBody, LiteLLMParamsBody -from proxy_client import ProxyClient -from pydantic import BaseModel - -pytestmark = [pytest.mark.e2e, pytest.mark.redis_timeout] - -REQUESTS: Final = 20 -MAX_SECONDS_PER_REQUEST: Final = 10.0 -MAX_LATENCY_GROWTH_RATIO: Final = 3.0 -MAX_LIVELINESS_SECONDS: Final = 2.0 -MAX_RSS_GROWTH_BYTES: Final = 200 * 1024 * 1024 -REDIS_PAUSE_MS: Final = 600_000 -BREAKER_FAILURE_THRESHOLD: Final = 5 -CLOSED_PORT_API_BASE: Final = "http://127.0.0.1:1" -TIMEOUT_FAILURES_RE: Final = re.compile( - r'^litellm_redis_circuit_breaker_failures_total\{failure_class="timeout"\} ([0-9.e+]+)$', re.M -) -BREAKER_OPEN_RE: Final = re.compile(r'^litellm_redis_circuit_breaker_state\{state="open"\} ([0-9.e+]+)$', re.M) -BREAKER_TRANSITIONS_RE: Final = re.compile( - r'^litellm_redis_circuit_breaker_transitions_total\{state="[a-z_]+"\} ([0-9.e+]+)$', re.M -) -FALLBACKS_RE: Final = re.compile(r"^litellm_deployment_successful_fallbacks_total\{[^}]*\} ([0-9.e+]+)$", re.M) -RSS_RE: Final = re.compile(r"^process_resident_memory_bytes ([0-9.e+]+)$", re.M) - - -class ResponsesBody(BaseModel): - model: str - input: str - max_output_tokens: int = 5 - - -class ResponsesObject(BaseModel): - id: str | None = None - status: str | None = None - output: list[object] = [] - - -class EmbeddingsObject(BaseModel): - model: str | None = None - data: list[object] = [] - - -@dataclass(frozen=True, slots=True) -class Endpoint: - """One endpoint's deployments and request shape.""" - - name: str - primary: str - backup: str - primary_params: LiteLLMParamsBody - backup_params: LiteLLMParamsBody - send: Callable[[ProxyClient, str, str, str], Result[BaseModel]] - served: Callable[[BaseModel], bool] - - -def _send_chat(proxy: ProxyClient, key: str, model: str, marker: str) -> Result[BaseModel]: - return proxy.transport.post( - "/chat/completions", - headers=proxy.transport.bearer(key), - json=ChatBody(model=model, messages=[ChatMessage(role="user", content=marker)], max_tokens=5), - response_type=ChatResponse, - timeout=MAX_SECONDS_PER_REQUEST, - ) - - -def _send_responses(proxy: ProxyClient, key: str, model: str, marker: str) -> Result[BaseModel]: - return proxy.transport.post( - "/v1/responses", - headers=proxy.transport.bearer(key), - json=ResponsesBody(model=model, input=marker), - response_type=ResponsesObject, - timeout=MAX_SECONDS_PER_REQUEST, - ) - - -def _send_embeddings(proxy: ProxyClient, key: str, model: str, marker: str) -> Result[BaseModel]: - return proxy.transport.post( - "/embeddings", - headers=proxy.transport.bearer(key), - json=EmbedBody(model=model, input=marker), - response_type=EmbeddingsObject, - timeout=MAX_SECONDS_PER_REQUEST, - ) - - -_CHAT_PRIMARY: Final = LiteLLMParamsBody( - model="openai/gpt-5-mini", api_key="sk-redis-timeout-primary-not-used", api_base=CLOSED_PORT_API_BASE -) -_CHAT_BACKUP: Final = LiteLLMParamsBody( - model="openai/gpt-5-nano", api_key="sk-redis-timeout-backup-not-used", mock_response="ok" -) -_EMBED_PRIMARY: Final = LiteLLMParamsBody( - model="openai/text-embedding-3-small", api_key="sk-redis-timeout-primary-not-used", api_base=CLOSED_PORT_API_BASE -) -_EMBED_BACKUP: Final = LiteLLMParamsBody( - model="openai/text-embedding-3-large", api_key="sk-redis-timeout-backup-not-used", mock_response=[0.1, 0.2, 0.3] -) - -ENDPOINTS: Final = ( - Endpoint( - name="chat_completions", - primary="redis-timeout-primary", - backup="redis-timeout-backup", - primary_params=_CHAT_PRIMARY, - backup_params=_CHAT_BACKUP, - send=_send_chat, - served=lambda data: isinstance(data, ChatResponse) and bool(data.choices), - ), - Endpoint( - name="responses", - primary="redis-timeout-primary", - backup="redis-timeout-backup", - primary_params=_CHAT_PRIMARY, - backup_params=_CHAT_BACKUP, - send=_send_responses, - served=lambda data: isinstance(data, ResponsesObject) and bool(data.output), - ), - Endpoint( - name="embeddings", - primary="redis-timeout-embed-primary", - backup="redis-timeout-embed-backup", - primary_params=_EMBED_PRIMARY, - backup_params=_EMBED_BACKUP, - send=_send_embeddings, - served=lambda data: isinstance(data, EmbeddingsObject) and bool(data.data), - ), -) - - -@pytest.fixture -def paused_redis() -> Iterator[None]: - """Hold the proxy's Redis in CLIENT PAUSE WRITE so every write it sends outlives the 1 ms socket - timeout. A loopback Redis otherwise answers many commands inside that budget. Reads stay live so - this control connection can lift the pause in teardown.""" - host = os.environ.get("REDIS_HOST") - port = os.environ.get("REDIS_PORT") - assert host and port, "REDIS_HOST and REDIS_PORT must name the Redis the proxy under test uses" - control = redis.Redis(host=host, port=int(port), socket_timeout=5) - control.client_pause(REDIS_PAUSE_MS, all=False) # pyright: ignore[reportUnknownMemberType] # redis-py stubs return Any - try: - yield - finally: - control.client_unpause() # pyright: ignore[reportUnknownMemberType] # redis-py stubs return Any - control.close() - - -def _metric(proxy: ProxyClient, pattern: re.Pattern[str]) -> float: - body = proxy.probe("/metrics", params=NoBody()).body - return sum(float(match.group(1)) for match in pattern.finditer(body)) - - -def _rss_bytes(proxy: ProxyClient) -> float | None: - """The proxy's resident memory from the Prometheus process collector, which reads /proc and - so reports on Linux only; None where the metric is absent.""" - body = proxy.probe("/metrics", params=NoBody()).body - match = RSS_RE.search(body) - return float(match.group(1)) if match else None - - -class TestRedisTimeout: - @pytest.mark.parametrize("endpoint", ENDPOINTS, ids=[endpoint.name for endpoint in ENDPOINTS]) - @pytest.mark.covers( - "reliability.circuit_breaker.redis_timeout.stays_responsive", - exercised_on=["chat_completions", "responses", "embeddings"], - ) - def test_retries_under_redis_timeouts_keep_answering( - self, client: ComplexityRouterClient, resources: ResourceManager, endpoint: Endpoint, paused_redis: None - ) -> None: - proxy = client.proxy - primary_id = proxy.create_model(endpoint.primary, endpoint.primary_params) - resources.defer(lambda: proxy.delete_model(primary_id)) - backup_id = proxy.create_model(endpoint.backup, endpoint.backup_params) - resources.defer(lambda: proxy.delete_model(backup_id)) - key = proxy.generate_key( - KeyGenerateBody( - models=[endpoint.primary, endpoint.backup], - key_alias=f"e2e-redis-timeout-{endpoint.name}-{unique_marker()}", - ) - ) - resources.defer(lambda: proxy.delete_key(key)) - timeouts_before = _metric(proxy, TIMEOUT_FAILURES_RE) - transitions_before = _metric(proxy, BREAKER_TRANSITIONS_RE) - fallbacks_before = _metric(proxy, FALLBACKS_RE) - rss_before = _rss_bytes(proxy) - - latencies: list[float] = [] - for request_number in range(1, REQUESTS + 1): - started = time.monotonic() - result = endpoint.send(proxy, key, endpoint.primary, f"redis timeout {unique_marker()} {request_number}") - elapsed = time.monotonic() - started - assert isinstance(result, Success), ( - f"{endpoint.name} request {request_number} failed after {elapsed:.1f}s with Redis timing out: {result}; " - f"earlier requests took {[round(seconds, 2) for seconds in latencies]}" - ) - assert endpoint.served(result.data), ( - f"{endpoint.name} request {request_number}: fallback to {endpoint.backup} returned no output" - ) - assert elapsed < MAX_SECONDS_PER_REQUEST, ( - f"{endpoint.name} request {request_number} took {elapsed:.1f}s with Redis timing out; " - f"earlier requests took {[round(seconds, 2) for seconds in latencies]}" - ) - latencies.append(elapsed) - - third = REQUESTS // 3 - early = sum(latencies[:third]) / third - late = sum(latencies[-third:]) / third - assert late <= max(early * MAX_LATENCY_GROWTH_RATIO, 0.5), ( - f"{endpoint.name} per-request latency grew from {early:.2f}s to {late:.2f}s across {REQUESTS} requests " - "while Redis timed out; the proxy is paying more for each failed request" - ) - - started = time.monotonic() - probe = proxy.transport.probe("/health/liveliness", params=NoBody()) - liveliness_seconds = time.monotonic() - started - assert probe.healthy, f"/health/liveliness returned {probe.status_code} after the Redis timeout loop" - assert liveliness_seconds < MAX_LIVELINESS_SECONDS, ( - f"/health/liveliness took {liveliness_seconds:.1f}s after the loop; the worker is stalled" - ) - - if rss_before is not None: - rss_after = _rss_bytes(proxy) - assert rss_after is not None - assert rss_after - rss_before <= MAX_RSS_GROWTH_BYTES, ( - f"proxy RSS grew {(rss_after - rss_before) / 2**20:.0f} MB across {REQUESTS} {endpoint.name} requests " - "with Redis timing out; on v1.100.0 this path grew by gigabytes" - ) - - fallbacks = _metric(proxy, FALLBACKS_RE) - fallbacks_before - assert fallbacks >= REQUESTS, ( - f"only {fallbacks:.0f} successful fallbacks were counted across {REQUESTS} {endpoint.name} requests; " - "the closed-port primary did not fail every request, so the retry path was not exercised" - ) - - timeouts_total = _metric(proxy, TIMEOUT_FAILURES_RE) - timeouts = timeouts_total - timeouts_before - transitions = _metric(proxy, BREAKER_TRANSITIONS_RE) - transitions_before - breaker_open = _metric(proxy, BREAKER_OPEN_RE) >= 1 - assert timeouts_total >= BREAKER_FAILURE_THRESHOLD, ( - f"the proxy counted only {timeouts_total:.0f} Redis timeouts in its lifetime; the write-paused Redis " - "never made its spend counter writes time out, so this run proved nothing" - ) - assert timeouts >= REQUESTS or transitions >= 1 or breaker_open, ( - f"during {REQUESTS} {endpoint.name} requests the breaker counted {timeouts:.0f} new timeouts, " - f"{transitions:.0f} state transitions, and ended {'open' if breaker_open else 'closed'}; " - "Redis was healthy for this case, so it proved nothing" - ) - - rows = proxy.poll_logs_for_key(key, min_rows=REQUESTS) - assert len(rows) >= REQUESTS, ( - f"only {len(rows)} of {REQUESTS} {endpoint.name} requests reached the spend log; " - "a Redis outage must not lose spend rows" - ) - failed_rows = [row.status for row in rows if row.status not in (None, "success")] - assert not failed_rows, f"{len(failed_rows)} {endpoint.name} spend rows are not successes: {failed_rows[:3]}" diff --git a/uv.lock b/uv.lock index 0fe787645a2..be124e2ead7 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-09-06T00:40:30.433549Z" +exclude-newer = "2026-09-08T03:56:24.358378Z" exclude-newer-span = "P3D" [manifest] @@ -4559,6 +4559,7 @@ e2e-dev = [ { name = "locust" }, { name = "mcp" }, { name = "playwright" }, + { name = "psutil" }, { name = "websockets" }, ] healthcheck = [ @@ -4747,6 +4748,7 @@ e2e-dev = [ { name = "locust", specifier = "==2.45.0" }, { name = "mcp", specifier = ">=1.28.1,<2.0" }, { name = "playwright", specifier = "==1.61.0" }, + { name = "psutil", specifier = "==7.2.2" }, { name = "websockets", specifier = ">=15.0.1,<16.0" }, ] healthcheck = [ From 5f17261534caa773206edc9fa1d85a91fc4435fd Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 11:06:06 -0700 Subject: [PATCH 10/27] test(load): pause Redis outright and budget the chaos phase against the baseline CLIENT PAUSE ALL for the length of the chaos phase instead of CLIENT PAUSE WRITE, so every Redis touchpoint on the request path times out rather than just the writes. The pause is sized to the phase because it freezes the control connection too; teardown's CLIENT UNPAUSE is a safety net for a phase that overran Latency, RSS and CPU are now budgeted as chaos-over-baseline ratios (p50/p90/p99 for latency and RSS, CPU seconds per request once) through a small phase_budget module, replacing the machine-shaped absolutes. The Redis timeout rate is reported but no longer asserted The final /metrics scrape waits for litellm_deployment_failure_responses_total to stop moving, since that counter is bumped from the async logging queue and lagged the load generator by thousands of increments. The model group carries a unique marker so a deployment left behind by an aborted run cannot absorb this run's retries Co-Authored-By: Claude Code --- tests/e2e/CLAUDE.md | 2 +- tests/e2e/conftest.py | 2 +- tests/e2e/load/phase_budget.py | 52 +++++ tests/e2e/load/proxy_usage.py | 8 + tests/e2e/load/test_phase_budget.py | 64 +++++++ tests/e2e/load/test_proxy_usage.py | 12 ++ tests/e2e/load/test_redis_chaos_e2e.py | 254 +++++++++++++++++++------ tests/e2e/pytest.ini | 2 +- 8 files changed, 337 insertions(+), 59 deletions(-) create mode 100644 tests/e2e/load/phase_budget.py create mode 100644 tests/e2e/load/test_phase_budget.py diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index dc197996ac2..bfab4553709 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -18,7 +18,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) -- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments with `CLIENT PAUSE WRITE` on the proxy's Redis mid-run, asserting zero failed requests and reporting latency, RSS, and CPU as p50/p90/p99 per phase; needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven weekly by `.github/workflows/test-e2e-redis-chaos.yml`), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic +- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests and budgeting p50/p90/p99 latency, RSS, and CPU-per-request as ratios against the same run's healthy phase; needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven weekly by `.github/workflows/test-e2e-redis-chaos.yml`), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 99a3f4967c3..700dc822d59 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -60,7 +60,7 @@ def pytest_configure(config: pytest.Config) -> None: ) config.addinivalue_line( "markers", - "redis_chaos: load test that pauses the proxy's Redis writes mid-run; needs a proxy booted from " + "redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from " "gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set", ) diff --git a/tests/e2e/load/phase_budget.py b/tests/e2e/load/phase_budget.py new file mode 100644 index 00000000000..7d354a678bc --- /dev/null +++ b/tests/e2e/load/phase_budget.py @@ -0,0 +1,52 @@ +"""Comparing one load phase against another, for tests that degrade a dependency mid-run. + +A chaos phase's absolute numbers say very little on their own: RSS scales with worker count, +latency with core count, so a ceiling calibrated on one machine is meaningless on the next. +What travels is the ratio against a healthy phase measured on the same machine in the same run. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final + + +@dataclass(frozen=True, slots=True) +class Budget: + """One metric's healthy value, its degraded value, and how much growth is allowed.""" + + name: str + baseline: float + degraded: float + ratio_ceiling: float + unit: str + decimals: int = 1 + + @property + def ratio(self) -> float | None: + """How many times the baseline the degraded value is, or None if there is no baseline.""" + return self.degraded / self.baseline if self.baseline > 0 else None + + def _rendered(self, value: float) -> str: + return f"{value:.{self.decimals}f}{self.unit}" + + def violation(self) -> str | None: + """Why this metric fails its budget, or None if it passes.""" + ratio: Final = self.ratio + if ratio is None: + return ( + f"{self.name} measured {self._rendered(self.baseline)} in the healthy phase, so there is nothing " + f"to compare the degraded phase against; the measurement did not happen" + ) + if ratio > self.ratio_ceiling: + return ( + f"{self.name} went from {self._rendered(self.baseline)} healthy to " + f"{self._rendered(self.degraded)} degraded, {ratio:.1f}x the baseline and past the " + f"{self.ratio_ceiling:.1f}x allowed" + ) + return None + + +def violations(budgets: tuple[Budget, ...]) -> tuple[str, ...]: + """Every budget the run blew, so one failure reports all of them instead of the first.""" + return tuple(violation for budget in budgets if (violation := budget.violation()) is not None) diff --git a/tests/e2e/load/proxy_usage.py b/tests/e2e/load/proxy_usage.py index b48129f0099..83463c078b8 100644 --- a/tests/e2e/load/proxy_usage.py +++ b/tests/e2e/load/proxy_usage.py @@ -50,6 +50,14 @@ class UsageWindow: return 0.0 return self.samples[-1].cpu_seconds - self.samples[0].cpu_seconds + def cpu_seconds_per_request(self, requests: int) -> float: + """CPU seconds the tree spent per request served. + + The portable cost figure: cores-busy saturates at the worker count under enough load, + so it reads the same whether a request costs 10 ms of CPU or 40 ms. This does not. + """ + return self.cpu_seconds_consumed() / requests if requests else 0.0 + def cpu_utilization_percentiles(self) -> tuple[float, float, float]: """Per-interval CPU utilization (cores busy) at p50, p90 and p99. diff --git a/tests/e2e/load/test_phase_budget.py b/tests/e2e/load/test_phase_budget.py new file mode 100644 index 00000000000..2355fb7cf4c --- /dev/null +++ b/tests/e2e/load/test_phase_budget.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from typing import Final + +from phase_budget import Budget, violations + + +def _budget(*, baseline: float, degraded: float, ceiling: float = 2.0) -> Budget: + return Budget(name="p99 RSS", baseline=baseline, degraded=degraded, ratio_ceiling=ceiling, unit=" MB", decimals=0) + + +class TestBudget: + def test_growth_within_the_ceiling_is_not_a_violation(self) -> None: + assert _budget(baseline=100, degraded=199).violation() is None + + def test_growth_exactly_at_the_ceiling_is_allowed(self) -> None: + assert _budget(baseline=100, degraded=200).violation() is None + + def test_growth_past_the_ceiling_reports_both_values_and_the_ratio(self) -> None: + violation: Final = _budget(baseline=100, degraded=250).violation() + + assert violation is not None + assert "100 MB" in violation + assert "250 MB" in violation + assert "2.5x" in violation + assert "2.0x allowed" in violation + + def test_shrinking_is_never_a_violation(self) -> None: + assert _budget(baseline=100, degraded=10).violation() is None + + def test_a_missing_baseline_is_a_violation_rather_than_a_silent_pass(self) -> None: + # The trap this guards: 0 as a baseline would make every ratio a division by zero, and + # treating it as "no growth" would pass a run that measured nothing at all. + violation: Final = _budget(baseline=0, degraded=4000).violation() + + assert violation is not None + assert "nothing to compare" in violation + + def test_the_unit_and_decimals_carry_into_the_message(self) -> None: + violation: Final = Budget( + name="p99 latency", baseline=0.16, degraded=9.5, ratio_ceiling=8.0, unit="s", decimals=3 + ).violation() + + assert violation is not None + assert "0.160s" in violation + assert "9.500s" in violation + + +class TestViolations: + def test_every_blown_budget_is_reported_not_just_the_first(self) -> None: + blown: Final = violations( + ( + _budget(baseline=100, degraded=500), + _budget(baseline=100, degraded=120), + Budget(name="CPU per request", baseline=10, degraded=90, ratio_ceiling=6.0, unit=" ms"), + ) + ) + + assert len(blown) == 2 + assert blown[0].startswith("p99 RSS") + assert blown[1].startswith("CPU per request") + + def test_a_run_inside_every_budget_reports_nothing(self) -> None: + assert violations((_budget(baseline=100, degraded=150),)) == () diff --git a/tests/e2e/load/test_proxy_usage.py b/tests/e2e/load/test_proxy_usage.py index 8d9f848825f..915c564de50 100644 --- a/tests/e2e/load/test_proxy_usage.py +++ b/tests/e2e/load/test_proxy_usage.py @@ -50,6 +50,18 @@ class TestCpuUtilization: assert window.cpu_utilization_percentiles() == (0.0, 0.0, 0.0) assert window.cpu_seconds_consumed() == 0.0 + def test_cost_per_request_separates_runs_that_cores_busy_reports_identically(self) -> None: + # Both windows pin 4 cores for 10 seconds, so utilization cannot tell them apart. The + # second one served a tenth of the traffic for the same CPU, which is the regression shape. + window: Final = _window(*((float(i), _MB, 4.0 * i) for i in range(11))) + + assert window.cpu_utilization_percentiles()[0] == 4.0 + assert window.cpu_seconds_per_request(4000) == 0.01 + assert window.cpu_seconds_per_request(400) == 0.1 + + def test_no_requests_reports_zero_cost_rather_than_dividing_by_zero(self) -> None: + assert _window((0.0, _MB, 0.0), (1.0, _MB, 1.0)).cpu_seconds_per_request(0) == 0.0 + def test_summary_reports_every_percentile_in_human_units(self) -> None: window: Final = _window((0.0, 200 * _MB, 0.0), (1.0, 200 * _MB, 1.5), (2.0, 200 * _MB, 3.0)) diff --git a/tests/e2e/load/test_redis_chaos_e2e.py b/tests/e2e/load/test_redis_chaos_e2e.py index f91b9c2fa09..950712309dc 100644 --- a/tests/e2e/load/test_redis_chaos_e2e.py +++ b/tests/e2e/load/test_redis_chaos_e2e.py @@ -1,4 +1,4 @@ -"""Live e2e: the proxy under load keeps serving every request while Redis writes time out. +"""Live e2e: the proxy under load keeps serving every request while Redis is down entirely. Runs against a proxy booted from tests/e2e/gateway/redis_chaos_ci_config.yml, which points cache_params at a real Redis with litellm's default socket_timeout. That one client backs all @@ -11,11 +11,13 @@ retries on the failing pair (a 500 is retryable, so retries keep re-picking insi order) and the router's order-based fallback then re-targets order 2. Every request is expected to succeed, and each one carries retry breadcrumbs into cost tracking. -Phase A is a baseline with Redis healthy; phase B holds Redis in CLIENT PAUSE WRITE, so the -spend counter increment times out and the callback stringifies the request metadata, -breadcrumbs included, into a failed-tracking alert. On v1.100.0 that string doubled per request -until the worker hung (LIT-6780), which is what the per-phase RSS and CPU percentiles are here -to catch. +Phase A is a baseline with Redis healthy; phase B holds Redis in CLIENT PAUSE ALL for the +length of the phase, simulating Redis being down outright rather than merely slow to write. +Every touchpoint times out: the auth cache read falls back to Postgres, the response cache +read and write both fail, and the spend counter increment times out and the callback +stringifies the request metadata, breadcrumbs included, into a failed-tracking alert. On +v1.100.0 that string doubled per request until the worker hung (LIT-6780), which is what the +per-phase RSS and CPU percentiles are here to catch. Needs the proxy on the same host, since RSS and CPU come from psutil on its process tree: a multi-worker proxy serves /metrics from the prometheus multiprocess collector, which drops @@ -26,8 +28,10 @@ from __future__ import annotations import os import re +import time from collections.abc import Iterator from dataclasses import dataclass +from itertools import pairwise from typing import Final import pytest @@ -38,12 +42,13 @@ from lifecycle import ResourceManager from load_client import LoadClient from locust_load import LoadResult, run_chat_load from models import KeyGenerateBody, LiteLLMParamsBody +from phase_budget import Budget, violations from proxy_client import ProxyClient from proxy_usage import ProxyUsageSampler, UsageWindow -pytestmark = [pytest.mark.e2e, pytest.mark.redis_chaos] +pytestmark: Final = pytest.mark.e2e -MODEL_GROUP: Final = "redis-chaos-fable" +MODEL_GROUP: Final = f"redis-chaos-fable-{unique_marker()}" MOCK_MODEL: Final = "anthropic/claude-fable-5-1" FAILING_DEPLOYMENTS: Final = 2 SERVING_DEPLOYMENTS: Final = 1 @@ -54,19 +59,42 @@ LOCUST_USERS: Final = 50 LOCUST_SPAWN_RATE: Final = 50.0 BASELINE_SECONDS: Final = 60.0 CHAOS_SECONDS: Final = 90.0 -REDIS_PAUSE_MS: Final = 600_000 -BASELINE_TIMEOUT_RATE_CEILING: Final = 0.05 -CHAOS_TIMEOUT_RATE_FLOOR: Final = 0.20 +REDIS_PAUSE_MS: Final = int(CHAOS_SECONDS * 1000) + +# Chaos-phase ceilings, as a multiple of the same metric in the baseline phase. Ratios rather +# than absolutes because every absolute here is machine-shaped: RSS scales with worker count +# and latency with core count, so a number calibrated on one runner means nothing on another. +# Calibrated from local runs under CLIENT PAUSE ALL that came in around 4x latency at every +# percentile, 1.03x RSS and 4.4x CPU per request, and deliberately loose: the regression these +# guard against grew memory by an order of magnitude, so catching it does not need a tight +# bound, and a tight one would flake on a shared CI runner. Latency gets the most slack because +# it is the metric a Redis outage is legitimately allowed to move, by its socket timeout on +# every call a request attempts. +CHAOS_LATENCY_RATIO_CEILING: Final = 12.0 +CHAOS_RSS_RATIO_CEILING: Final = 1.5 +CHAOS_CPU_PER_REQUEST_RATIO_CEILING: Final = 6.0 + +DRAIN_TIMEOUT_SECONDS: Final = 30.0 +DRAIN_POLL_SECONDS: Final = 1.0 TIMEOUT_FAILURES_RE: Final = re.compile( r'^litellm_redis_circuit_breaker_failures_total\{failure_class="timeout"\} ([0-9.e+]+)$', re.M ) -BREAKER_OPEN_RE: Final = re.compile(r'^litellm_redis_circuit_breaker_state\{state="open"\} ([0-9.e+]+)$', re.M) +# The state gauge carries a pid label under the multiprocess collector, one series per worker, +# so this matches any label order rather than a bare {state="open"} that never appears. +BREAKER_OPEN_RE: Final = re.compile( + r'^litellm_redis_circuit_breaker_state\{[^}]*state="open"[^}]*\} ([0-9.e+]+)$', re.M +) BREAKER_TRANSITIONS_RE: Final = re.compile( r'^litellm_redis_circuit_breaker_transitions_total\{state="[a-z_]+"\} ([0-9.e+]+)$', re.M ) -RETRIES_RE: Final = re.compile(r"^litellm_deployment_failure_responses_total\{[^}]*\} ([0-9.e+]+)$", re.M) -COOLDOWN_RE: Final = re.compile(r"^litellm_deployment_cooled_down_total\{[^}]*\} ([0-9.e+]+)$", re.M) + + +def _deployment_metric_re(name: str, model_ids: tuple[str, ...]) -> re.Pattern[str]: + """A per-deployment counter, narrowed to the deployments one run registered, so traffic + anything else sends the same proxy during the run cannot pad the retry count.""" + ids: Final = "|".join(re.escape(model_id) for model_id in model_ids) + return re.compile(rf'^litellm_{name}\{{[^}}]*model_id="(?:{ids})"[^}}]*\}} ([0-9.e+]+)$', re.M) @dataclass(frozen=True, slots=True) @@ -76,11 +104,22 @@ class Phase: name: str load: LoadResult usage: UsageWindow + redis_timeouts: float + + @property + def timeouts_per_request(self) -> float: + return self.redis_timeouts / self.load.requests if self.load.requests else 0.0 + + @property + def cpu_seconds_per_request(self) -> float: + return self.usage.cpu_seconds_per_request(self.load.requests) def report(self) -> str: return ( f"{self.name}: {self.load.requests} requests, {self.load.failures} failures, " - f"{self.load.requests_per_second:.0f} rps, {self.load.latency_summary()}; {self.usage.summary()}" + f"{self.load.requests_per_second:.0f} rps, {self.load.latency_summary()}; {self.usage.summary()}; " + f"{self.cpu_seconds_per_request * 1000:.1f} ms CPU per request; " + f"{self.timeouts_per_request:.2f} Redis timeouts per request" ) @@ -119,10 +158,12 @@ def proxy_pid() -> int: @pytest.fixture def redis_control() -> Iterator[redis.Redis[bytes]]: - """A control connection to the proxy's Redis, which unpauses writes in teardown. + """A control connection to the proxy's Redis, which unpauses it in teardown as a safety net. - Only writes are paused: CLIENT PAUSE ALL would freeze this connection too, leaving - nothing able to lift the pause. + CLIENT PAUSE ALL freezes every connection including this one, so REDIS_PAUSE_MS is sized + to the chaos phase: by the time teardown runs, the pause has + already lapsed on its own and CLIENT UNPAUSE here returns immediately. It only actually + waits out a lapsed pause if the chaos phase itself overran that duration. """ host: Final = os.environ.get("REDIS_HOST") port: Final = os.environ.get("REDIS_PORT") @@ -135,18 +176,54 @@ def redis_control() -> Iterator[redis.Redis[bytes]]: control.close() -def _metric(proxy: ProxyClient, pattern: re.Pattern[str]) -> float: - body: Final = proxy.probe("/metrics", params=NoBody()).body - return sum(float(match.group(1)) for match in pattern.finditer(body)) +def _scrape(proxy: ProxyClient) -> str: + """One /metrics body, read once per checkpoint so every counter comes from the same instant.""" + scrape: Final = proxy.probe("/metrics", params=NoBody()) + assert scrape.status_code == 200, ( + f"/metrics did not answer ({scrape.status_code}: {scrape.body[:200]}), so no counter can be read; " + f"a silent 0 here would turn every before-and-after difference negative" + ) + return scrape.body -def _register_deployments(proxy: ProxyClient, resources: ResourceManager) -> None: - for _ in range(FAILING_DEPLOYMENTS): - failing_id = proxy.create_model(MODEL_GROUP, _failing_params()) - resources.defer(lambda model_id=failing_id: proxy.delete_model(model_id)) - for _ in range(SERVING_DEPLOYMENTS): - serving_id = proxy.create_model(MODEL_GROUP, _serving_params()) - resources.defer(lambda model_id=serving_id: proxy.delete_model(model_id)) +def _metric(scrape: str, pattern: re.Pattern[str]) -> float: + return sum(float(match.group(1)) for match in pattern.finditer(scrape)) + + +def _scrape_after_drain(proxy: ProxyClient, pattern: re.Pattern[str]) -> str: + """A /metrics body taken once `pattern`'s count has stopped moving. + + `set_llm_deployment_failure_metrics` runs from the async logging callback queue, so a load + generator that just stopped sending traffic can still have thousands of failure increments + in flight, and a scrape taken the instant load stops undercounts them. Settling on the + counter rather than sleeping a fixed duration keeps the wait proportional to how backed up + the queue actually is. + """ + deadline: Final = time.monotonic() + DRAIN_TIMEOUT_SECONDS + + def scrapes() -> Iterator[str]: + yield _scrape(proxy) + while time.monotonic() < deadline: + time.sleep(DRAIN_POLL_SECONDS) + yield _scrape(proxy) + + settled: Final = next( + (later for earlier, later in pairwise(scrapes()) if _metric(earlier, pattern) == _metric(later, pattern)), + None, + ) + return settled if settled is not None else _scrape(proxy) + + +def _register_deployments(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, ...]: + """The model ids this run registered, which scope its per-deployment metric reads.""" + params: Final = ( + *(_failing_params() for _ in range(FAILING_DEPLOYMENTS)), + *(_serving_params() for _ in range(SERVING_DEPLOYMENTS)), + ) + model_ids: Final = tuple(proxy.create_model(MODEL_GROUP, one) for one in params) + for model_id in model_ids: + resources.defer(lambda doomed=model_id: proxy.delete_model(doomed)) + return model_ids def _generate_key_pool(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, ...]: @@ -174,12 +251,68 @@ def _drive(keys: tuple[str, ...], seconds: float) -> LoadResult: ) +def _latency_budget(percentile: str, baseline: float, degraded: float) -> Budget: + return Budget( + name=f"{percentile} latency", + baseline=baseline, + degraded=degraded, + ratio_ceiling=CHAOS_LATENCY_RATIO_CEILING, + unit="s", + decimals=3, + ) + + +def _rss_budget(percentile: str, baseline: UsageWindow, degraded: UsageWindow, fraction: float) -> Budget: + return Budget( + name=f"{percentile} RSS", + baseline=baseline.rss_percentile(fraction) / 2**20, + degraded=degraded.rss_percentile(fraction) / 2**20, + ratio_ceiling=CHAOS_RSS_RATIO_CEILING, + unit=" MB", + decimals=0, + ) + + +def _chaos_budgets(baseline: Phase, chaos: Phase) -> tuple[Budget, ...]: + """What a Redis outage is allowed to cost, measured against the same run's healthy phase. + + Every request still succeeding is the headline assertion, but a proxy can answer every + request while leaking: the v1.100.0 regression (LIT-6780) served traffic the whole way up + to a 61 GB worker. These bound the cost of serving it. + + Latency and RSS are budgeted at p50, p90 and p99 so a regression that only shows up in the + tail (or only in the median) cannot hide behind the other. Latency gets the loosest bound + because a timing-out Redis legitimately adds its socket_timeout to every request that + touches it, several times over on a retried request. RSS gets the tightest: the failure + path has no business allocating more per request. CPU is budgeted once, as CPU seconds per + request rather than per percentile: cores-busy saturates at the worker count under load, so + its percentiles read the same whether a request costs 10 ms of CPU or 40, and cannot budget + anything; seconds per request is the CPU figure that actually moves. + """ + return ( + _latency_budget("p50", baseline.load.p50_seconds, chaos.load.p50_seconds), + _latency_budget("p90", baseline.load.p90_seconds, chaos.load.p90_seconds), + _latency_budget("p99", baseline.load.p99_seconds, chaos.load.p99_seconds), + _rss_budget("p50", baseline.usage, chaos.usage, 0.5), + _rss_budget("p90", baseline.usage, chaos.usage, 0.9), + _rss_budget("p99", baseline.usage, chaos.usage, 0.99), + Budget( + name="CPU per request", + baseline=baseline.cpu_seconds_per_request * 1000, + degraded=chaos.cpu_seconds_per_request * 1000, + ratio_ceiling=CHAOS_CPU_PER_REQUEST_RATIO_CEILING, + unit=" ms", + ), + ) + + +@pytest.mark.redis_chaos class TestRedisChaos: @pytest.mark.covers( "reliability.circuit_breaker.redis_timeout.stays_responsive", - exercised_on=["chat_completions"], + exercised_on=("chat_completions",), ) - def test_load_survives_redis_write_timeouts( + def test_load_survives_redis_being_down( self, client: LoadClient, resources: ResourceManager, @@ -187,20 +320,36 @@ class TestRedisChaos: redis_control: redis.Redis[bytes], ) -> None: proxy: Final = client.proxy - _register_deployments(proxy, resources) + model_ids: Final = _register_deployments(proxy, resources) keys: Final = _generate_key_pool(proxy, resources) - timeouts_at_start: Final = _metric(proxy, TIMEOUT_FAILURES_RE) - retries_before: Final = _metric(proxy, RETRIES_RE) - cooldowns_before: Final = _metric(proxy, COOLDOWN_RE) + retries_re: Final = _deployment_metric_re("deployment_failure_responses_total", model_ids) + cooldown_re: Final = _deployment_metric_re("deployment_cooled_down_total", model_ids) + + at_start: Final = _scrape(proxy) with ProxyUsageSampler(proxy_pid) as sampler: - baseline: Final = Phase(name="baseline", load=_drive(keys, BASELINE_SECONDS), usage=sampler.split()) - timeouts_after_baseline: Final = _metric(proxy, TIMEOUT_FAILURES_RE) + baseline_load: Final = _drive(keys, BASELINE_SECONDS) + baseline_usage: Final = sampler.split() + after_baseline: Final = _scrape(proxy) - redis_control.client_pause(REDIS_PAUSE_MS, all=False) # pyright: ignore[reportUnknownMemberType] # redis-py stubs return Any - chaos: Final = Phase(name="chaos", load=_drive(keys, CHAOS_SECONDS), usage=sampler.split()) + redis_control.client_pause(REDIS_PAUSE_MS, all=True) # pyright: ignore[reportUnknownMemberType] # redis-py stubs return Any + chaos_load: Final = _drive(keys, CHAOS_SECONDS) + chaos_usage: Final = sampler.split() + at_end: Final = _scrape_after_drain(proxy, retries_re) + baseline: Final = Phase( + name="baseline", + load=baseline_load, + usage=baseline_usage, + redis_timeouts=_metric(after_baseline, TIMEOUT_FAILURES_RE) - _metric(at_start, TIMEOUT_FAILURES_RE), + ) + chaos: Final = Phase( + name="chaos", + load=chaos_load, + usage=chaos_usage, + redis_timeouts=_metric(at_end, TIMEOUT_FAILURES_RE) - _metric(after_baseline, TIMEOUT_FAILURES_RE), + ) report: Final = f"{baseline.report()} | {chaos.report()}" for phase in (baseline, chaos): @@ -215,13 +364,13 @@ class TestRedisChaos: f"a Redis failure reached the response path. {phase.load.diagnosis()}. {report}" ) - cooldowns: Final = _metric(proxy, COOLDOWN_RE) - cooldowns_before + cooldowns: Final = _metric(at_end, cooldown_re) - _metric(at_start, cooldown_re) assert cooldowns == 0, ( f"{cooldowns:.0f} deployments were cooled down during the run; the failing deployments are supposed " f"to stay in rotation so every request keeps exercising the retry path. {report}" ) - retries: Final = _metric(proxy, RETRIES_RE) - retries_before + retries: Final = _metric(at_end, retries_re) - _metric(at_start, retries_re) assert retries >= baseline.load.requests + chaos.load.requests, ( f"only {retries:.0f} deployment failures were counted across " f"{baseline.load.requests + chaos.load.requests} requests; the mock deployments did not fail, so no " @@ -229,24 +378,17 @@ class TestRedisChaos: f"{report}" ) - baseline_timeout_rate: Final = (timeouts_after_baseline - timeouts_at_start) / baseline.load.requests - assert baseline_timeout_rate <= BASELINE_TIMEOUT_RATE_CEILING, ( - f"a healthy Redis timed out on {baseline_timeout_rate:.1%} of baseline requests, over the " - f"{BASELINE_TIMEOUT_RATE_CEILING:.0%} this test tolerates; at litellm's default socket_timeout a " - f"loaded Redis does time out occasionally, but this much means the baseline is already degraded and " - f"the two phases are not comparable. {report}" + transitions: Final = _metric(at_end, BREAKER_TRANSITIONS_RE) - _metric(after_baseline, BREAKER_TRANSITIONS_RE) + breaker_open: Final = _metric(at_end, BREAKER_OPEN_RE) >= 1 + assert transitions >= 1 or breaker_open, ( + f"pausing Redis produced no circuit breaker state transitions and it ended closed; nothing on the " + f"request path ever saw Redis fail, so this run proved nothing. {report}" ) - chaos_timeouts: Final = _metric(proxy, TIMEOUT_FAILURES_RE) - timeouts_after_baseline - chaos_timeout_rate: Final = chaos_timeouts / chaos.load.requests - transitions: Final = _metric(proxy, BREAKER_TRANSITIONS_RE) - breaker_open: Final = _metric(proxy, BREAKER_OPEN_RE) >= 1 - assert chaos_timeout_rate >= CHAOS_TIMEOUT_RATE_FLOOR or transitions >= 1 or breaker_open, ( - f"with writes paused the breaker saw Redis time out on only {chaos_timeout_rate:.1%} of requests " - f"against {baseline_timeout_rate:.1%} at baseline, under the {CHAOS_TIMEOUT_RATE_FLOOR:.0%} a real " - f"outage produces, and it counted {transitions:.0f} state transitions and ended " - f"{'open' if breaker_open else 'closed'}. The spend counter increment never failed, so this run " - f"proved nothing. {report}" + blown: Final = violations(_chaos_budgets(baseline, chaos)) + assert not blown, ( + f"pausing Redis cost the proxy more than the socket timeout on the calls it attempts: " + f"{'; '.join(blown)}. {report}" ) rows: Final = proxy.poll_logs_for_key(keys[0], min_rows=1) diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index 1dcb50a6fab..774d9644497 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -9,4 +9,4 @@ markers = load: heavy throughput/load test; collected last so it never perturbs latency-sensitive suites weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set - redis_chaos: load test that pauses the proxy's Redis writes mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set + redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set From 1270ecb781ad0b3d5de25250807e26046559a8b9 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 12:28:02 -0700 Subject: [PATCH 11/27] test(load): drive /v1/messages alongside /chat/completions in the Redis chaos test The Anthropic Messages route reaches the same Redis touchpoints and cost-tracking callback through its own request path, so a failure-path regression there would not surface from chat completions alone. Each simulated user now picks one endpoint round robin and stays on it, and the per-endpoint split is asserted and reported so a run that silently drove only one route fails instead of passing. Co-Authored-By: Claude Code --- tests/e2e/CLAUDE.md | 2 +- tests/e2e/coverage_registry/reliability.yaml | 2 +- tests/e2e/load/locust_load.py | 57 +++++++++++++++++--- tests/e2e/load/locustfile.py | 23 ++++++-- tests/e2e/load/test_locust_load.py | 47 ++++++++++++++++ tests/e2e/load/test_redis_chaos_e2e.py | 21 ++++++-- 6 files changed, 135 insertions(+), 17 deletions(-) diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 24bad9f0d0d..80e78dd4cdd 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -18,7 +18,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) -- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests and budgeting p50/p90/p99 latency, RSS, and CPU-per-request as ratios against the same run's healthy phase; needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven weekly by `.github/workflows/test-e2e-redis-chaos.yml`), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic +- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint and budgeting p50/p90/p99 latency, RSS, and CPU-per-request as ratios against the same run's healthy phase; needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven weekly by `.github/workflows/test-e2e-redis-chaos.yml`), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 23163ab666c..cfbedf3d236 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -31,7 +31,7 @@ - {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"} - {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix"} - {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"} -- {id: reliability.circuit_breaker.redis_timeout.stays_responsive, module: reliability, tier: P1, behavior: circuit_breaker, variant: redis_timeout, assertions: [stays_responsive], exercised_on: [chat_completions], source: "litellm/proxy/hooks/proxy_track_cost_callback.py:386", fail_before_fix: proven, rationale: "Under locust load with every request retrying through failing mock deployments, pausing Redis writes mid-run trips the breaker and every request still succeeds, with latency, RSS, and CPU reported as p50/p90/p99 against the pre-pause baseline; on v1.100.0 the failed-tracking alert body doubled per request until the worker OOMed (LIT-6780)"} +- {id: reliability.circuit_breaker.redis_timeout.stays_responsive, module: reliability, tier: P1, behavior: circuit_breaker, variant: redis_timeout, assertions: [stays_responsive], exercised_on: [chat_completions, messages], source: "litellm/proxy/hooks/proxy_track_cost_callback.py:386", fail_before_fix: proven, rationale: "Under locust load split round robin over /chat/completions and /v1/messages with every request retrying through failing mock deployments, holding Redis in CLIENT PAUSE ALL for the phase trips the breaker and every request still succeeds, with latency, RSS, and CPU reported as p50/p90/p99 against the pre-pause baseline; on v1.100.0 the failed-tracking alert body doubled per request until the worker OOMed (LIT-6780)"} - {id: reliability.timeout.request_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: request_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions, messages], source: "litellm/router.py:545-551", rationale: "Per-request timeout raises Timeout"} - {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"} - {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"} diff --git a/tests/e2e/load/locust_load.py b/tests/e2e/load/locust_load.py index a1668c5bfc5..40f9f333db5 100644 --- a/tests/e2e/load/locust_load.py +++ b/tests/e2e/load/locust_load.py @@ -5,6 +5,7 @@ import os import subprocess import sys import tempfile +from collections.abc import Sequence from dataclasses import dataclass from itertools import accumulate from pathlib import Path @@ -19,6 +20,7 @@ _MAX_REPORTED_ERRORS = 5 class LocustStatEntry(BaseModel): + name: str num_requests: int num_failures: int start_time: float @@ -36,6 +38,16 @@ class LoadError: occurrences: int +@dataclass(frozen=True, slots=True) +class EndpointLoad: + """One route's share of a phase, so a run that silently drove only one of them is visible.""" + + name: str + requests: int + failures: int + p50_seconds: float + + @dataclass(frozen=True, slots=True) class LoadResult: requests: int @@ -44,6 +56,7 @@ class LoadResult: p50_seconds: float p90_seconds: float p99_seconds: float + endpoints: tuple[EndpointLoad, ...] errors: tuple[LoadError, ...] generator_warnings: tuple[str, ...] @@ -65,8 +78,15 @@ class LoadResult: def latency_summary(self) -> str: return f"p50 {self.p50_seconds:.3f}s, p90 {self.p90_seconds:.3f}s, p99 {self.p99_seconds:.3f}s" + def endpoint_summary(self) -> str: + return ", ".join( + f"{endpoint.name} {endpoint.requests} requests, {endpoint.failures} failures, " + f"p50 {endpoint.p50_seconds:.3f}s" + for endpoint in self.endpoints + ) -def percentile_seconds(entries: list[LocustStatEntry], fraction: float) -> float: + +def percentile_seconds(entries: Sequence[LocustStatEntry], fraction: float) -> float: """The response time at `fraction` of the merged histograms, in seconds. Locust buckets response times by millisecond, so this reads the first bucket whose @@ -82,13 +102,29 @@ def percentile_seconds(entries: list[LocustStatEntry], fraction: float) -> float return next(milliseconds for (milliseconds, _), seen in zip(samples, running) if seen >= rank) / 1000.0 +def per_endpoint(entries: Sequence[LocustStatEntry]) -> tuple[EndpointLoad, ...]: + """Each locust request name's own totals, in the order the names first appear.""" + names: Final = tuple(dict.fromkeys(entry.name for entry in entries)) + grouped: Final = ((name, tuple(entry for entry in entries if entry.name == name)) for name in names) + return tuple( + EndpointLoad( + name=name, + requests=sum(entry.num_requests for entry in group), + failures=sum(entry.num_failures for entry in group), + p50_seconds=percentile_seconds(group, 0.5), + ) + for name, group in grouped + ) + + def aggregate_stats( - entries: list[LocustStatEntry], + entries: Sequence[LocustStatEntry], errors: tuple[LoadError, ...], generator_warnings: tuple[str, ...], ) -> LoadResult: requests = sum(entry.num_requests for entry in entries) failures = sum(entry.num_failures for entry in entries) + endpoints = per_endpoint(entries) if not entries or requests == 0: return LoadResult( requests=requests, @@ -97,6 +133,7 @@ def aggregate_stats( p50_seconds=0.0, p90_seconds=0.0, p99_seconds=0.0, + endpoints=endpoints, errors=errors, generator_warnings=generator_warnings, ) @@ -108,6 +145,7 @@ def aggregate_stats( p50_seconds=percentile_seconds(entries, 0.5), p90_seconds=percentile_seconds(entries, 0.9), p99_seconds=percentile_seconds(entries, 0.99), + endpoints=endpoints, errors=errors, generator_warnings=generator_warnings, ) @@ -142,19 +180,21 @@ def read_generator_warnings(stderr: str) -> tuple[str, ...]: return tuple(dict.fromkeys(saturated)) -def run_chat_load( +def run_gateway_load( *, base_url: str, api_keys: tuple[str, ...], model: str, + endpoints: tuple[str, ...], users: int, spawn_rate: float, duration_seconds: float, ) -> LoadResult: - """Drive /chat/completions from headless locust and aggregate what it reported. + """Drive `endpoints` from headless locust and aggregate what it reported. Each simulated user picks one of `api_keys`, so auth and budget lookups spread over a - pool of virtual keys instead of keeping one key's cache entry permanently warm. + pool of virtual keys instead of keeping one key's cache entry permanently warm, and one + of `endpoints` round robin, so the run covers every route the caller asked for. """ with tempfile.TemporaryDirectory(prefix="e2e-load-") as report_dir: csv_prefix = Path(report_dir) / _CSV_PREFIX @@ -180,7 +220,12 @@ def run_chat_load( "--exit-code-on-error", "0", ], - env={**os.environ, "LOAD_API_KEYS": ",".join(api_keys), "LOAD_MODEL": model}, + env={ + **os.environ, + "LOAD_API_KEYS": ",".join(api_keys), + "LOAD_MODEL": model, + "LOAD_ENDPOINTS": ",".join(endpoints), + }, capture_output=True, text=True, timeout=duration_seconds + 120, diff --git a/tests/e2e/load/locustfile.py b/tests/e2e/load/locustfile.py index 3d9ef7c83ea..a4e396aa8d7 100644 --- a/tests/e2e/load/locustfile.py +++ b/tests/e2e/load/locustfile.py @@ -3,16 +3,22 @@ from __future__ import annotations import os import random import uuid +from itertools import cycle from typing import Final from locust import FastHttpUser, constant, task _MODEL: Final = os.environ["LOAD_MODEL"] _API_KEYS: Final = tuple(os.environ["LOAD_API_KEYS"].split(",")) +_NEXT_ENDPOINT: Final = cycle(os.environ["LOAD_ENDPOINTS"].split(",")) def _payload() -> dict[str, object]: - """A prompt no other request sent, so the response cache never answers for the deployment.""" + """A prompt no other request sent, so the response cache never answers for the deployment. + + Both endpoints take the same body: /v1/messages requires max_tokens, which /chat/completions + also accepts, so one payload serves the whole round robin. + """ return { "model": _MODEL, "messages": [{"role": "user", "content": f"load test ping {uuid.uuid4().hex}"}], @@ -20,17 +26,24 @@ def _payload() -> dict[str, object]: } -class ChatUser(FastHttpUser): +class GatewayUser(FastHttpUser): + """One simulated user, pinned to one endpoint for its lifetime. + + Endpoints are handed out round robin as users spawn, so a run spreads evenly over them + while each user's traffic stays on a single route, the way a real client behaves. + """ + wait_time = constant(0) def on_start(self) -> None: self.headers = {"Authorization": f"Bearer {random.choice(_API_KEYS)}"} + self.endpoint = next(_NEXT_ENDPOINT) @task - def chat(self) -> None: + def call(self) -> None: self.client.post( # pyright: ignore[reportUnknownMemberType] # locust FastHttpSession.post types json/**kwargs as Any - "/chat/completions", + self.endpoint, json=_payload(), headers=self.headers, - name="/chat/completions", + name=self.endpoint, ) diff --git a/tests/e2e/load/test_locust_load.py b/tests/e2e/load/test_locust_load.py index 40a238ef624..af3e1483099 100644 --- a/tests/e2e/load/test_locust_load.py +++ b/tests/e2e/load/test_locust_load.py @@ -1,6 +1,7 @@ from __future__ import annotations from pathlib import Path +from typing import Final from locust_load import ( LoadError, @@ -18,12 +19,14 @@ _FAILURES_HEADER = "Method,Name,Error,Occurrences,First Seen,Last Seen\n" def _entry( *, num_requests: int, + name: str = "/chat/completions", num_failures: int = 0, start_time: float = 1000.0, last_request_timestamp: float = 1010.0, response_times: dict[int, int] | None = None, ) -> LocustStatEntry: return LocustStatEntry( + name=name, num_requests=num_requests, num_failures=num_failures, start_time=start_time, @@ -44,6 +47,7 @@ def _result( p50_seconds=0.05, p90_seconds=0.08, p99_seconds=0.1, + endpoints=(), errors=errors, generator_warnings=generator_warnings, ) @@ -126,6 +130,49 @@ class TestAggregate: assert result.requests == 0 assert result.requests_per_second == 0.0 assert result.failure_ratio == 1.0 + assert result.endpoints == () + + +class TestPerEndpoint: + def test_each_route_keeps_its_own_requests_failures_and_median(self) -> None: + entries: Final = ( + _entry(name="/chat/completions", num_requests=100, response_times={20: 100}), + _entry(name="/v1/messages", num_requests=40, num_failures=3, response_times={900: 40}), + ) + + result: Final = aggregate_stats(entries, (), ()) + + assert tuple((one.name, one.requests, one.failures, one.p50_seconds) for one in result.endpoints) == ( + ("/chat/completions", 100, 0, 0.02), + ("/v1/messages", 40, 3, 0.9), + ) + + def test_several_stats_entries_for_one_route_fold_into_a_single_row(self) -> None: + entries: Final = ( + _entry(name="/v1/messages", num_requests=10, response_times={30: 10}), + _entry(name="/v1/messages", num_requests=30, num_failures=1, response_times={30: 30}), + ) + + result: Final = aggregate_stats(entries, (), ()) + + assert tuple((one.name, one.requests, one.failures) for one in result.endpoints) == (("/v1/messages", 40, 1),) + + def test_a_route_that_never_ran_is_absent_so_a_one_sided_run_cannot_pass_unnoticed(self) -> None: + result: Final = aggregate_stats((_entry(name="/chat/completions", num_requests=10),), (), ()) + + assert tuple(one.name for one in result.endpoints) == ("/chat/completions",) + + def test_the_summary_names_every_route_with_its_counts(self) -> None: + entries: Final = ( + _entry(name="/chat/completions", num_requests=2, response_times={20: 2}), + _entry(name="/v1/messages", num_requests=1, num_failures=1, response_times={500: 1}), + ) + + result: Final = aggregate_stats(entries, (), ()) + + assert result.endpoint_summary() == ( + "/chat/completions 2 requests, 0 failures, p50 0.020s, /v1/messages 1 requests, 1 failures, p50 0.500s" + ) class TestErrorBreakdown: diff --git a/tests/e2e/load/test_redis_chaos_e2e.py b/tests/e2e/load/test_redis_chaos_e2e.py index 950712309dc..646429bbdd7 100644 --- a/tests/e2e/load/test_redis_chaos_e2e.py +++ b/tests/e2e/load/test_redis_chaos_e2e.py @@ -11,6 +11,11 @@ retries on the failing pair (a 500 is retryable, so retries keep re-picking insi order) and the router's order-based fallback then re-targets order 2. Every request is expected to succeed, and each one carries retry breadcrumbs into cost tracking. +Traffic is split round robin between /chat/completions and /v1/messages, one endpoint per +simulated user: the Redis touchpoints and the cost-tracking callback are shared by both, but +the Anthropic Messages route reaches them through its own request path, so a regression that +only shows up there would not surface from chat completions alone. + Phase A is a baseline with Redis healthy; phase B holds Redis in CLIENT PAUSE ALL for the length of the phase, simulating Redis being down outright rather than merely slow to write. Every touchpoint times out: the auth cache read falls back to Postgres, the response cache @@ -40,7 +45,7 @@ from e2e_config import PROXY_BASE_URL, unique_marker from e2e_http import NoBody from lifecycle import ResourceManager from load_client import LoadClient -from locust_load import LoadResult, run_chat_load +from locust_load import LoadResult, run_gateway_load from models import KeyGenerateBody, LiteLLMParamsBody from phase_budget import Budget, violations from proxy_client import ProxyClient @@ -55,6 +60,7 @@ SERVING_DEPLOYMENTS: Final = 1 FAILING_ORDER: Final = 1 SERVING_ORDER: Final = 2 KEY_POOL_SIZE: Final = 8 +LOAD_ENDPOINTS: Final = ("/chat/completions", "/v1/messages") LOCUST_USERS: Final = 50 LOCUST_SPAWN_RATE: Final = 50.0 BASELINE_SECONDS: Final = 60.0 @@ -119,7 +125,8 @@ class Phase: f"{self.name}: {self.load.requests} requests, {self.load.failures} failures, " f"{self.load.requests_per_second:.0f} rps, {self.load.latency_summary()}; {self.usage.summary()}; " f"{self.cpu_seconds_per_request * 1000:.1f} ms CPU per request; " - f"{self.timeouts_per_request:.2f} Redis timeouts per request" + f"{self.timeouts_per_request:.2f} Redis timeouts per request; " + f"by endpoint: {self.load.endpoint_summary()}" ) @@ -241,10 +248,11 @@ def _generate_key_pool(proxy: ProxyClient, resources: ResourceManager) -> tuple[ def _drive(keys: tuple[str, ...], seconds: float) -> LoadResult: - return run_chat_load( + return run_gateway_load( base_url=PROXY_BASE_URL, api_keys=keys, model=MODEL_GROUP, + endpoints=LOAD_ENDPOINTS, users=LOCUST_USERS, spawn_rate=LOCUST_SPAWN_RATE, duration_seconds=seconds, @@ -310,7 +318,7 @@ def _chaos_budgets(baseline: Phase, chaos: Phase) -> tuple[Budget, ...]: class TestRedisChaos: @pytest.mark.covers( "reliability.circuit_breaker.redis_timeout.stays_responsive", - exercised_on=("chat_completions",), + exercised_on=("chat_completions", "messages"), ) def test_load_survives_redis_being_down( self, @@ -356,6 +364,11 @@ class TestRedisChaos: assert phase.load.requests > 0, ( f"{phase.name} drove no traffic at all, so it proved nothing: {phase.load.diagnosis()}. {report}" ) + assert frozenset(endpoint.name for endpoint in phase.load.endpoints) == frozenset(LOAD_ENDPOINTS), ( + f"{phase.name} drove {tuple(endpoint.name for endpoint in phase.load.endpoints)} rather than every " + f"endpoint in {LOAD_ENDPOINTS}; the round robin hands one endpoint to each simulated user, so a " + f"missing one means a route never ran and its request path was never exercised. {report}" + ) assert phase.load.failures == 0, ( f"{phase.name} had {phase.load.failures} of {phase.load.requests} requests fail. Every request " f"must succeed: the failing deployments sit at order {FAILING_ORDER} and the serving one at order " From 66980bbb873b4db3d6fd070bc8471feae52efe26 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 12:40:25 -0700 Subject: [PATCH 12/27] test(e2e): address Redis chaos PR review, add log-bytes budget Pad the locust payload to tens of KB so per-request bookkeeping cost scales with body size instead of hiding behind a 40-byte prompt. Turn on use_redis_transaction_buffer in the chaos config and JSON_LOGS in the workflow so the spend buffer, pod lock, and JSON-encoded breaker tracebacks are all part of the measured chaos cost. Add a log-bytes-per-request budget alongside latency, RSS, and CPU, reading the proxy's log file size at each phase split; its ceiling is uncalibrated since no chaos run has measured it yet. Co-Authored-By: Claude Code --- .github/workflows/test-e2e-redis-chaos.yml | 2 + tests/e2e/CLAUDE.md | 2 +- tests/e2e/gateway/redis_chaos_ci_config.yml | 1 + tests/e2e/load/locustfile.py | 7 ++- tests/e2e/load/test_redis_chaos_e2e.py | 59 ++++++++++++++++++--- 5 files changed, 62 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-e2e-redis-chaos.yml b/.github/workflows/test-e2e-redis-chaos.yml index 2ce20836db7..0880c1fb464 100644 --- a/.github/workflows/test-e2e-redis-chaos.yml +++ b/.github/workflows/test-e2e-redis-chaos.yml @@ -40,6 +40,7 @@ jobs: DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm LITELLM_MASTER_KEY: sk-redis-chaos-e2e LITELLM_LOG: WARNING + JSON_LOGS: "true" steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: @@ -73,6 +74,7 @@ jobs: run: | nohup uv run --no-sync litellm --config tests/e2e/gateway/redis_chaos_ci_config.yml --port 4000 --num_workers 4 > proxy.log 2>&1 & echo "E2E_PROXY_PID=$!" >> "$GITHUB_ENV" + echo "E2E_PROXY_LOG=$(pwd)/proxy.log" >> "$GITHUB_ENV" for _ in $(seq 1 90); do if curl -fs http://localhost:4000/health/liveliness > /dev/null; then exit 0 diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 80e78dd4cdd..ed57dd71759 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -18,7 +18,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) -- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint and budgeting p50/p90/p99 latency, RSS, and CPU-per-request as ratios against the same run's healthy phase; needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven weekly by `.github/workflows/test-e2e-redis-chaos.yml`), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic +- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint and budgeting p50/p90/p99 latency, RSS, CPU-per-request, and log-bytes-per-request as ratios against the same run's healthy phase; needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven weekly by `.github/workflows/test-e2e-redis-chaos.yml`), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke diff --git a/tests/e2e/gateway/redis_chaos_ci_config.yml b/tests/e2e/gateway/redis_chaos_ci_config.yml index 37b024502f7..f7a71c50a71 100644 --- a/tests/e2e/gateway/redis_chaos_ci_config.yml +++ b/tests/e2e/gateway/redis_chaos_ci_config.yml @@ -1,6 +1,7 @@ general_settings: master_key: os.environ/LITELLM_MASTER_KEY store_model_in_db: true + use_redis_transaction_buffer: true litellm_settings: callbacks: ["prometheus"] diff --git a/tests/e2e/load/locustfile.py b/tests/e2e/load/locustfile.py index a4e396aa8d7..9b7bdf2ee1e 100644 --- a/tests/e2e/load/locustfile.py +++ b/tests/e2e/load/locustfile.py @@ -11,17 +11,20 @@ from locust import FastHttpUser, constant, task _MODEL: Final = os.environ["LOAD_MODEL"] _API_KEYS: Final = tuple(os.environ["LOAD_API_KEYS"].split(",")) _NEXT_ENDPOINT: Final = cycle(os.environ["LOAD_ENDPOINTS"].split(",")) +_FILLER: Final = "x" * 40_000 def _payload() -> dict[str, object]: """A prompt no other request sent, so the response cache never answers for the deployment. Both endpoints take the same body: /v1/messages requires max_tokens, which /chat/completions - also accepts, so one payload serves the whole round robin. + also accepts, so one payload serves the whole round robin. Padded to tens of KB so a + per-request bookkeeping cost that scales with body size (string formatting, hashing) shows + up in the CPU and log-size budgets instead of hiding behind a 40-byte prompt. """ return { "model": _MODEL, - "messages": [{"role": "user", "content": f"load test ping {uuid.uuid4().hex}"}], + "messages": [{"role": "user", "content": f"load test ping {uuid.uuid4().hex} {_FILLER}"}], "max_tokens": 16, } diff --git a/tests/e2e/load/test_redis_chaos_e2e.py b/tests/e2e/load/test_redis_chaos_e2e.py index 646429bbdd7..153be656566 100644 --- a/tests/e2e/load/test_redis_chaos_e2e.py +++ b/tests/e2e/load/test_redis_chaos_e2e.py @@ -22,11 +22,13 @@ Every touchpoint times out: the auth cache read falls back to Postgres, the resp read and write both fail, and the spend counter increment times out and the callback stringifies the request metadata, breadcrumbs included, into a failed-tracking alert. On v1.100.0 that string doubled per request until the worker hung (LIT-6780), which is what the -per-phase RSS and CPU percentiles are here to catch. +per-phase RSS, CPU, and log-bytes budgets are here to catch. Needs the proxy on the same host, since RSS and CPU come from psutil on its process tree: a multi-worker proxy serves /metrics from the prometheus multiprocess collector, which drops -the process collector's memory and CPU series. Deselected unless E2E_REDIS_CHAOS is set. +the process collector's memory and CPU series. Log bytes are read from the file the proxy's +stdout/stderr was redirected to, so the same host requirement covers that too. Deselected +unless E2E_REDIS_CHAOS is set. """ from __future__ import annotations @@ -37,6 +39,7 @@ import time from collections.abc import Iterator from dataclasses import dataclass from itertools import pairwise +from pathlib import Path from typing import Final import pytest @@ -79,6 +82,11 @@ REDIS_PAUSE_MS: Final = int(CHAOS_SECONDS * 1000) CHAOS_LATENCY_RATIO_CEILING: Final = 12.0 CHAOS_RSS_RATIO_CEILING: Final = 1.5 CHAOS_CPU_PER_REQUEST_RATIO_CEILING: Final = 6.0 +# Uncalibrated: no chaos run has measured this yet, since JSON_LOGS and the padded payload +# landed after the last run this file's other ceilings were calibrated from. Deliberately loose +# until a real run tightens it; the failed-tracking alert body that motivates this test already +# logs the full request metadata per timeout, so a JSON-encoded traceback storm should dwarf this. +CHAOS_LOG_BYTES_PER_REQUEST_RATIO_CEILING: Final = 20.0 DRAIN_TIMEOUT_SECONDS: Final = 30.0 DRAIN_POLL_SECONDS: Final = 1.0 @@ -111,6 +119,7 @@ class Phase: load: LoadResult usage: UsageWindow redis_timeouts: float + log_bytes: int @property def timeouts_per_request(self) -> float: @@ -120,11 +129,16 @@ class Phase: def cpu_seconds_per_request(self) -> float: return self.usage.cpu_seconds_per_request(self.load.requests) + @property + def log_bytes_per_request(self) -> float: + return self.log_bytes / self.load.requests if self.load.requests else 0.0 + def report(self) -> str: return ( f"{self.name}: {self.load.requests} requests, {self.load.failures} failures, " f"{self.load.requests_per_second:.0f} rps, {self.load.latency_summary()}; {self.usage.summary()}; " f"{self.cpu_seconds_per_request * 1000:.1f} ms CPU per request; " + f"{self.log_bytes_per_request:.0f} log bytes per request; " f"{self.timeouts_per_request:.2f} Redis timeouts per request; " f"by endpoint: {self.load.endpoint_summary()}" ) @@ -163,6 +177,22 @@ def proxy_pid() -> int: return int(pid) +@pytest.fixture +def proxy_log() -> Path: + """Path to the proxy's stdout/stderr log, which the workflow captures to a file. + + Required rather than discovered for the same reason as proxy_pid: a developer machine may + have more than one proxy log around. + """ + path: Final = os.environ.get("E2E_PROXY_LOG") + assert path, "E2E_PROXY_LOG must hold the path the proxy's stdout/stderr was redirected to" + return Path(path) + + +def _log_bytes(path: Path) -> int: + return path.stat().st_size + + @pytest.fixture def redis_control() -> Iterator[redis.Redis[bytes]]: """A control connection to the proxy's Redis, which unpauses it in teardown as a safety net. @@ -292,10 +322,13 @@ def _chaos_budgets(baseline: Phase, chaos: Phase) -> tuple[Budget, ...]: tail (or only in the median) cannot hide behind the other. Latency gets the loosest bound because a timing-out Redis legitimately adds its socket_timeout to every request that touches it, several times over on a retried request. RSS gets the tightest: the failure - path has no business allocating more per request. CPU is budgeted once, as CPU seconds per - request rather than per percentile: cores-busy saturates at the worker count under load, so - its percentiles read the same whether a request costs 10 ms of CPU or 40, and cannot budget - anything; seconds per request is the CPU figure that actually moves. + path has no business allocating more per request. CPU and log bytes are each budgeted once, + as an amount per request rather than per percentile: cores-busy saturates at the worker + count under load, so its percentiles read the same whether a request costs 10 ms of CPU or + 40, and cannot budget anything; per-request is the figure that actually moves. Log bytes + isolates the cost of the failed-tracking alert's own noisy error handling from the CPU it + burns doing useful retry work, since the two would otherwise be indistinguishable in one + CPU number. """ return ( _latency_budget("p50", baseline.load.p50_seconds, chaos.load.p50_seconds), @@ -311,6 +344,14 @@ def _chaos_budgets(baseline: Phase, chaos: Phase) -> tuple[Budget, ...]: ratio_ceiling=CHAOS_CPU_PER_REQUEST_RATIO_CEILING, unit=" ms", ), + Budget( + name="log bytes per request", + baseline=baseline.log_bytes_per_request, + degraded=chaos.log_bytes_per_request, + ratio_ceiling=CHAOS_LOG_BYTES_PER_REQUEST_RATIO_CEILING, + unit=" B", + decimals=0, + ), ) @@ -325,6 +366,7 @@ class TestRedisChaos: client: LoadClient, resources: ResourceManager, proxy_pid: int, + proxy_log: Path, redis_control: redis.Redis[bytes], ) -> None: proxy: Final = client.proxy @@ -335,28 +377,33 @@ class TestRedisChaos: cooldown_re: Final = _deployment_metric_re("deployment_cooled_down_total", model_ids) at_start: Final = _scrape(proxy) + log_at_start: Final = _log_bytes(proxy_log) with ProxyUsageSampler(proxy_pid) as sampler: baseline_load: Final = _drive(keys, BASELINE_SECONDS) baseline_usage: Final = sampler.split() after_baseline: Final = _scrape(proxy) + log_after_baseline: Final = _log_bytes(proxy_log) redis_control.client_pause(REDIS_PAUSE_MS, all=True) # pyright: ignore[reportUnknownMemberType] # redis-py stubs return Any chaos_load: Final = _drive(keys, CHAOS_SECONDS) chaos_usage: Final = sampler.split() at_end: Final = _scrape_after_drain(proxy, retries_re) + log_at_end: Final = _log_bytes(proxy_log) baseline: Final = Phase( name="baseline", load=baseline_load, usage=baseline_usage, redis_timeouts=_metric(after_baseline, TIMEOUT_FAILURES_RE) - _metric(at_start, TIMEOUT_FAILURES_RE), + log_bytes=log_after_baseline - log_at_start, ) chaos: Final = Phase( name="chaos", load=chaos_load, usage=chaos_usage, redis_timeouts=_metric(at_end, TIMEOUT_FAILURES_RE) - _metric(after_baseline, TIMEOUT_FAILURES_RE), + log_bytes=log_at_end - log_after_baseline, ) report: Final = f"{baseline.report()} | {chaos.report()}" From cc1d2c66c835df8ee25fb02031ed415c2e936634 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 13:38:01 -0700 Subject: [PATCH 13/27] test(e2e): bound chaos latency and log volume with flat ceilings A ratio against the healthy phase cannot bound either metric. Once the Redis circuit breaker opens, a request skips Redis instead of waiting on its socket timeout, so the chaos phase can measure cheaper than the baseline it is compared against: local runs came in at 0.61x baseline p90 while a log-bytes ratio read 724x. Splitting Budget into RatioBudget and AbsoluteBudget lets RSS and CPU keep the ratio they need, since both are machine-shaped, while latency and log volume get the wall-clock ceiling a user actually cares about. Co-Authored-By: Claude Code --- tests/e2e/CLAUDE.md | 2 +- tests/e2e/load/phase_budget.py | 55 +++++++++++++----- tests/e2e/load/test_phase_budget.py | 53 +++++++++++++++-- tests/e2e/load/test_redis_chaos_e2e.py | 79 ++++++++++++-------------- 4 files changed, 126 insertions(+), 63 deletions(-) diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index ed57dd71759..419cfd6dedd 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -18,7 +18,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) -- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint and budgeting p50/p90/p99 latency, RSS, CPU-per-request, and log-bytes-per-request as ratios against the same run's healthy phase; needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven weekly by `.github/workflows/test-e2e-redis-chaos.yml`), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic +- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven weekly by `.github/workflows/test-e2e-redis-chaos.yml`), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke diff --git a/tests/e2e/load/phase_budget.py b/tests/e2e/load/phase_budget.py index 7d354a678bc..066e2579da8 100644 --- a/tests/e2e/load/phase_budget.py +++ b/tests/e2e/load/phase_budget.py @@ -1,18 +1,27 @@ """Comparing one load phase against another, for tests that degrade a dependency mid-run. -A chaos phase's absolute numbers say very little on their own: RSS scales with worker count, -latency with core count, so a ceiling calibrated on one machine is meaningless on the next. -What travels is the ratio against a healthy phase measured on the same machine in the same run. +Two shapes of ceiling, because the metrics divide into two kinds. RSS and CPU are +machine-shaped: RSS scales with worker count and CPU with core count, so an absolute number +calibrated on one runner means nothing on the next, and what travels is the ratio against a +healthy phase measured on the same machine in the same run. Latency and log volume are not: +a ratio there is actively misleading, because a dependency that fails fast once its breaker +opens can make the degraded phase look cheaper than the healthy one while still being far +slower or noisier than a user should ever see. Those get a flat ceiling, which is the promise +the test is actually making. """ from __future__ import annotations from dataclasses import dataclass -from typing import Final +from typing import Final, TypeAlias + + +def _rendered(value: float, unit: str, decimals: int) -> str: + return f"{value:.{decimals}f}{unit}" @dataclass(frozen=True, slots=True) -class Budget: +class RatioBudget: """One metric's healthy value, its degraded value, and how much growth is allowed.""" name: str @@ -27,26 +36,46 @@ class Budget: """How many times the baseline the degraded value is, or None if there is no baseline.""" return self.degraded / self.baseline if self.baseline > 0 else None - def _rendered(self, value: float) -> str: - return f"{value:.{self.decimals}f}{self.unit}" - def violation(self) -> str | None: """Why this metric fails its budget, or None if it passes.""" ratio: Final = self.ratio if ratio is None: return ( - f"{self.name} measured {self._rendered(self.baseline)} in the healthy phase, so there is nothing " - f"to compare the degraded phase against; the measurement did not happen" + f"{self.name} measured {_rendered(self.baseline, self.unit, self.decimals)} in the healthy phase, " + f"so there is nothing to compare the degraded phase against; the measurement did not happen" ) if ratio > self.ratio_ceiling: return ( - f"{self.name} went from {self._rendered(self.baseline)} healthy to " - f"{self._rendered(self.degraded)} degraded, {ratio:.1f}x the baseline and past the " - f"{self.ratio_ceiling:.1f}x allowed" + f"{self.name} went from {_rendered(self.baseline, self.unit, self.decimals)} healthy to " + f"{_rendered(self.degraded, self.unit, self.decimals)} degraded, {ratio:.1f}x the baseline and past " + f"the {self.ratio_ceiling:.1f}x allowed" ) return None +@dataclass(frozen=True, slots=True) +class AbsoluteBudget: + """One metric's degraded value against a flat ceiling, for metrics a ratio cannot bound.""" + + name: str + measured: float + ceiling: float + unit: str + decimals: int = 1 + + def violation(self) -> str | None: + """Why this metric fails its budget, or None if it passes.""" + if self.measured > self.ceiling: + return ( + f"{self.name} measured {_rendered(self.measured, self.unit, self.decimals)} in the degraded phase, " + f"past the {_rendered(self.ceiling, self.unit, self.decimals)} allowed" + ) + return None + + +Budget: TypeAlias = RatioBudget | AbsoluteBudget + + def violations(budgets: tuple[Budget, ...]) -> tuple[str, ...]: """Every budget the run blew, so one failure reports all of them instead of the first.""" return tuple(violation for budget in budgets if (violation := budget.violation()) is not None) diff --git a/tests/e2e/load/test_phase_budget.py b/tests/e2e/load/test_phase_budget.py index 2355fb7cf4c..ea9e56afb0d 100644 --- a/tests/e2e/load/test_phase_budget.py +++ b/tests/e2e/load/test_phase_budget.py @@ -2,14 +2,16 @@ from __future__ import annotations from typing import Final -from phase_budget import Budget, violations +from phase_budget import AbsoluteBudget, RatioBudget, violations -def _budget(*, baseline: float, degraded: float, ceiling: float = 2.0) -> Budget: - return Budget(name="p99 RSS", baseline=baseline, degraded=degraded, ratio_ceiling=ceiling, unit=" MB", decimals=0) +def _budget(*, baseline: float, degraded: float, ceiling: float = 2.0) -> RatioBudget: + return RatioBudget( + name="p99 RSS", baseline=baseline, degraded=degraded, ratio_ceiling=ceiling, unit=" MB", decimals=0 + ) -class TestBudget: +class TestRatioBudget: def test_growth_within_the_ceiling_is_not_a_violation(self) -> None: assert _budget(baseline=100, degraded=199).violation() is None @@ -37,7 +39,7 @@ class TestBudget: assert "nothing to compare" in violation def test_the_unit_and_decimals_carry_into_the_message(self) -> None: - violation: Final = Budget( + violation: Final = RatioBudget( name="p99 latency", baseline=0.16, degraded=9.5, ratio_ceiling=8.0, unit="s", decimals=3 ).violation() @@ -46,13 +48,40 @@ class TestBudget: assert "9.500s" in violation +class TestAbsoluteBudget: + def test_a_value_under_the_ceiling_is_not_a_violation(self) -> None: + assert AbsoluteBudget(name="p99 latency", measured=1.2, ceiling=5.0, unit="s", decimals=3).violation() is None + + def test_a_value_exactly_at_the_ceiling_is_allowed(self) -> None: + assert AbsoluteBudget(name="p99 latency", measured=5.0, ceiling=5.0, unit="s", decimals=3).violation() is None + + def test_a_value_past_the_ceiling_reports_the_measurement_and_the_ceiling(self) -> None: + violation: Final = AbsoluteBudget( + name="p99 latency", measured=9.5, ceiling=5.0, unit="s", decimals=3 + ).violation() + + assert violation is not None + assert "9.500s" in violation + assert "5.000s allowed" in violation + + def test_a_flat_ceiling_fails_a_degraded_phase_that_is_cheaper_than_its_baseline(self) -> None: + # The whole reason this shape exists: once the breaker opens, requests skip Redis instead + # of waiting on its socket timeout, so the chaos phase can measure faster than the healthy + # one. A ratio against that baseline passes; the user still waited 9.5s. + assert _budget(baseline=20.0, degraded=9.5, ceiling=2.0).violation() is None + assert AbsoluteBudget(name="p99 latency", measured=9.5, ceiling=5.0, unit="s").violation() is not None + + def test_a_zero_measurement_is_not_a_violation(self) -> None: + assert AbsoluteBudget(name="log bytes per request", measured=0, ceiling=12_000, unit=" B").violation() is None + + class TestViolations: def test_every_blown_budget_is_reported_not_just_the_first(self) -> None: blown: Final = violations( ( _budget(baseline=100, degraded=500), _budget(baseline=100, degraded=120), - Budget(name="CPU per request", baseline=10, degraded=90, ratio_ceiling=6.0, unit=" ms"), + RatioBudget(name="CPU per request", baseline=10, degraded=90, ratio_ceiling=6.0, unit=" ms"), ) ) @@ -60,5 +89,17 @@ class TestViolations: assert blown[0].startswith("p99 RSS") assert blown[1].startswith("CPU per request") + def test_both_budget_shapes_report_together(self) -> None: + blown: Final = violations( + ( + _budget(baseline=100, degraded=500), + AbsoluteBudget(name="p99 latency", measured=9.5, ceiling=5.0, unit="s", decimals=3), + ) + ) + + assert len(blown) == 2 + assert blown[0].startswith("p99 RSS") + assert blown[1].startswith("p99 latency") + def test_a_run_inside_every_budget_reports_nothing(self) -> None: assert violations((_budget(baseline=100, degraded=150),)) == () diff --git a/tests/e2e/load/test_redis_chaos_e2e.py b/tests/e2e/load/test_redis_chaos_e2e.py index 153be656566..94c879d29cb 100644 --- a/tests/e2e/load/test_redis_chaos_e2e.py +++ b/tests/e2e/load/test_redis_chaos_e2e.py @@ -50,7 +50,7 @@ from lifecycle import ResourceManager from load_client import LoadClient from locust_load import LoadResult, run_gateway_load from models import KeyGenerateBody, LiteLLMParamsBody -from phase_budget import Budget, violations +from phase_budget import AbsoluteBudget, Budget, RatioBudget, violations from proxy_client import ProxyClient from proxy_usage import ProxyUsageSampler, UsageWindow @@ -70,23 +70,25 @@ BASELINE_SECONDS: Final = 60.0 CHAOS_SECONDS: Final = 90.0 REDIS_PAUSE_MS: Final = int(CHAOS_SECONDS * 1000) -# Chaos-phase ceilings, as a multiple of the same metric in the baseline phase. Ratios rather -# than absolutes because every absolute here is machine-shaped: RSS scales with worker count -# and latency with core count, so a number calibrated on one runner means nothing on another. -# Calibrated from local runs under CLIENT PAUSE ALL that came in around 4x latency at every -# percentile, 1.03x RSS and 4.4x CPU per request, and deliberately loose: the regression these -# guard against grew memory by an order of magnitude, so catching it does not need a tight -# bound, and a tight one would flake on a shared CI runner. Latency gets the most slack because -# it is the metric a Redis outage is legitimately allowed to move, by its socket timeout on -# every call a request attempts. -CHAOS_LATENCY_RATIO_CEILING: Final = 12.0 +# RSS and CPU are budgeted as a multiple of the same metric in the baseline phase, because both +# are machine-shaped: RSS scales with worker count and CPU with core count, so a number +# calibrated on one runner means nothing on another. Calibrated from local runs under CLIENT +# PAUSE ALL that came in around 1.03x RSS and 4.4x CPU per request, and deliberately loose: the +# regression these guard against grew memory by an order of magnitude, so catching it does not +# need a tight bound, and a tight one would flake on a shared CI runner. CHAOS_RSS_RATIO_CEILING: Final = 1.5 CHAOS_CPU_PER_REQUEST_RATIO_CEILING: Final = 6.0 -# Uncalibrated: no chaos run has measured this yet, since JSON_LOGS and the padded payload -# landed after the last run this file's other ceilings were calibrated from. Deliberately loose -# until a real run tightens it; the failed-tracking alert body that motivates this test already -# logs the full request metadata per timeout, so a JSON-encoded traceback storm should dwarf this. -CHAOS_LOG_BYTES_PER_REQUEST_RATIO_CEILING: Final = 20.0 + +# Latency and log volume get flat ceilings instead, because a ratio cannot bound either one. Once +# the breaker opens, a request skips Redis rather than waiting on its socket timeout, so the chaos +# phase can come in faster than baseline (local runs measured p90 at 0.61x) and a ratio passes on a +# phase that was never slow. What a user actually cares about is the wall-clock number, which these +# hold directly. Calibrated from local runs whose worst chaos phase was p50 0.66s, p90 0.74s, p99 +# 1.20s and 3.4 KB of log per request, then left roughly 3x loose for a shared CI runner. +CHAOS_P50_LATENCY_CEILING_SECONDS: Final = 2.0 +CHAOS_P90_LATENCY_CEILING_SECONDS: Final = 3.0 +CHAOS_P99_LATENCY_CEILING_SECONDS: Final = 5.0 +CHAOS_LOG_BYTES_PER_REQUEST_CEILING: Final = 12_000.0 DRAIN_TIMEOUT_SECONDS: Final = 30.0 DRAIN_POLL_SECONDS: Final = 1.0 @@ -289,19 +291,12 @@ def _drive(keys: tuple[str, ...], seconds: float) -> LoadResult: ) -def _latency_budget(percentile: str, baseline: float, degraded: float) -> Budget: - return Budget( - name=f"{percentile} latency", - baseline=baseline, - degraded=degraded, - ratio_ceiling=CHAOS_LATENCY_RATIO_CEILING, - unit="s", - decimals=3, - ) +def _latency_budget(percentile: str, measured: float, ceiling: float) -> Budget: + return AbsoluteBudget(name=f"{percentile} latency", measured=measured, ceiling=ceiling, unit="s", decimals=3) def _rss_budget(percentile: str, baseline: UsageWindow, degraded: UsageWindow, fraction: float) -> Budget: - return Budget( + return RatioBudget( name=f"{percentile} RSS", baseline=baseline.rss_percentile(fraction) / 2**20, degraded=degraded.rss_percentile(fraction) / 2**20, @@ -312,18 +307,18 @@ def _rss_budget(percentile: str, baseline: UsageWindow, degraded: UsageWindow, f def _chaos_budgets(baseline: Phase, chaos: Phase) -> tuple[Budget, ...]: - """What a Redis outage is allowed to cost, measured against the same run's healthy phase. + """What a Redis outage is allowed to cost. Every request still succeeding is the headline assertion, but a proxy can answer every request while leaking: the v1.100.0 regression (LIT-6780) served traffic the whole way up - to a 61 GB worker. These bound the cost of serving it. + to a 61 GB worker. These bound the cost of serving it. RSS and CPU are bounded against the + same run's healthy phase, latency and log bytes against a flat ceiling; see phase_budget + for why the two kinds of metric cannot share one shape. Latency and RSS are budgeted at p50, p90 and p99 so a regression that only shows up in the - tail (or only in the median) cannot hide behind the other. Latency gets the loosest bound - because a timing-out Redis legitimately adds its socket_timeout to every request that - touches it, several times over on a retried request. RSS gets the tightest: the failure - path has no business allocating more per request. CPU and log bytes are each budgeted once, - as an amount per request rather than per percentile: cores-busy saturates at the worker + tail (or only in the median) cannot hide behind the other. RSS gets the tightest bound: the + failure path has no business allocating more per request. CPU and log bytes are each budgeted + once, as an amount per request rather than per percentile: cores-busy saturates at the worker count under load, so its percentiles read the same whether a request costs 10 ms of CPU or 40, and cannot budget anything; per-request is the figure that actually moves. Log bytes isolates the cost of the failed-tracking alert's own noisy error handling from the CPU it @@ -331,24 +326,23 @@ def _chaos_budgets(baseline: Phase, chaos: Phase) -> tuple[Budget, ...]: CPU number. """ return ( - _latency_budget("p50", baseline.load.p50_seconds, chaos.load.p50_seconds), - _latency_budget("p90", baseline.load.p90_seconds, chaos.load.p90_seconds), - _latency_budget("p99", baseline.load.p99_seconds, chaos.load.p99_seconds), + _latency_budget("p50", chaos.load.p50_seconds, CHAOS_P50_LATENCY_CEILING_SECONDS), + _latency_budget("p90", chaos.load.p90_seconds, CHAOS_P90_LATENCY_CEILING_SECONDS), + _latency_budget("p99", chaos.load.p99_seconds, CHAOS_P99_LATENCY_CEILING_SECONDS), _rss_budget("p50", baseline.usage, chaos.usage, 0.5), _rss_budget("p90", baseline.usage, chaos.usage, 0.9), _rss_budget("p99", baseline.usage, chaos.usage, 0.99), - Budget( + RatioBudget( name="CPU per request", baseline=baseline.cpu_seconds_per_request * 1000, degraded=chaos.cpu_seconds_per_request * 1000, ratio_ceiling=CHAOS_CPU_PER_REQUEST_RATIO_CEILING, unit=" ms", ), - Budget( + AbsoluteBudget( name="log bytes per request", - baseline=baseline.log_bytes_per_request, - degraded=chaos.log_bytes_per_request, - ratio_ceiling=CHAOS_LOG_BYTES_PER_REQUEST_RATIO_CEILING, + measured=chaos.log_bytes_per_request, + ceiling=CHAOS_LOG_BYTES_PER_REQUEST_CEILING, unit=" B", decimals=0, ), @@ -447,8 +441,7 @@ class TestRedisChaos: blown: Final = violations(_chaos_budgets(baseline, chaos)) assert not blown, ( - f"pausing Redis cost the proxy more than the socket timeout on the calls it attempts: " - f"{'; '.join(blown)}. {report}" + f"pausing Redis cost the proxy more than a Redis outage is allowed to: {'; '.join(blown)}. {report}" ) rows: Final = proxy.poll_logs_for_key(keys[0], min_rows=1) From 1d71006567311f6608688873db5d670e8fae21af Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 13:43:18 -0700 Subject: [PATCH 14/27] test(e2e): tighten chaos RSS and CPU ceilings to what runs actually measured RSS moved 0.91x-1.40x across three identical local runs, so it stays loose at 2x rather than the arbitrary 1.5x carried over from the pre-padding-payload calibration. CPU per request held steady at 1.33x-1.36x across the same runs, so 4x replaces the looser 6x it inherited from stale numbers. Co-Authored-By: Claude Code --- tests/e2e/load/test_redis_chaos_e2e.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/e2e/load/test_redis_chaos_e2e.py b/tests/e2e/load/test_redis_chaos_e2e.py index 94c879d29cb..c1da41aeb16 100644 --- a/tests/e2e/load/test_redis_chaos_e2e.py +++ b/tests/e2e/load/test_redis_chaos_e2e.py @@ -72,12 +72,11 @@ REDIS_PAUSE_MS: Final = int(CHAOS_SECONDS * 1000) # RSS and CPU are budgeted as a multiple of the same metric in the baseline phase, because both # are machine-shaped: RSS scales with worker count and CPU with core count, so a number -# calibrated on one runner means nothing on another. Calibrated from local runs under CLIENT -# PAUSE ALL that came in around 1.03x RSS and 4.4x CPU per request, and deliberately loose: the -# regression these guard against grew memory by an order of magnitude, so catching it does not -# need a tight bound, and a tight one would flake on a shared CI runner. -CHAOS_RSS_RATIO_CEILING: Final = 1.5 -CHAOS_CPU_PER_REQUEST_RATIO_CEILING: Final = 6.0 +# calibrated on one runner means nothing on another. RSS moved 0.91x-1.40x across three otherwise +# identical local runs, so it stays loose; CPU per request held steady at 1.33x-1.36x across the +# same runs, so it can sit closer to what is actually measured. +CHAOS_RSS_RATIO_CEILING: Final = 2.0 +CHAOS_CPU_PER_REQUEST_RATIO_CEILING: Final = 4.0 # Latency and log volume get flat ceilings instead, because a ratio cannot bound either one. Once # the breaker opens, a request skips Redis rather than waiting on its socket timeout, so the chaos From e81a7887762fff8c4e9b31d38bd77a9c4546a2a4 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 13:43:53 -0700 Subject: [PATCH 15/27] test(e2e): hold chaos CPU per request to 2x Three local runs measured 1.33x-1.36x, so 2x is the tightest bound the data supports and still catches a regression far smaller than 4x would. Noted in the comment that this is the ceiling to loosen first if a weekly run trips it, since core count shifts how much of baseline CPU is fixed per-request work. Co-Authored-By: Claude Code --- tests/e2e/load/test_redis_chaos_e2e.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/e2e/load/test_redis_chaos_e2e.py b/tests/e2e/load/test_redis_chaos_e2e.py index c1da41aeb16..c800ccb0e51 100644 --- a/tests/e2e/load/test_redis_chaos_e2e.py +++ b/tests/e2e/load/test_redis_chaos_e2e.py @@ -74,9 +74,11 @@ REDIS_PAUSE_MS: Final = int(CHAOS_SECONDS * 1000) # are machine-shaped: RSS scales with worker count and CPU with core count, so a number # calibrated on one runner means nothing on another. RSS moved 0.91x-1.40x across three otherwise # identical local runs, so it stays loose; CPU per request held steady at 1.33x-1.36x across the -# same runs, so it can sit closer to what is actually measured. +# same runs, so it sits close to what is actually measured. That makes CPU the likeliest of these +# to flake first on a runner whose core count shifts how much of baseline CPU is fixed per-request +# work: loosen it rather than widening the others if a weekly run trips it without a real cause. CHAOS_RSS_RATIO_CEILING: Final = 2.0 -CHAOS_CPU_PER_REQUEST_RATIO_CEILING: Final = 4.0 +CHAOS_CPU_PER_REQUEST_RATIO_CEILING: Final = 2.0 # Latency and log volume get flat ceilings instead, because a ratio cannot bound either one. Once # the breaker opens, a request skips Redis rather than waiting on its socket timeout, so the chaos From a1f9f4cbe839a8bd4bcf605bbcf44effddd3724f Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 13:45:11 -0700 Subject: [PATCH 16/27] test(e2e): tighten chaos latency ceilings to 1s/2s/3s Local runs measured p50 0.19s, p90 0.23s, p99 0.69s, so 2s/3s/5s left several times that as slack. 1s/2s/3s keeps a comfortable margin while catching a smaller regression than the looser ceilings would have. Co-Authored-By: Claude Code --- tests/e2e/load/test_redis_chaos_e2e.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/e2e/load/test_redis_chaos_e2e.py b/tests/e2e/load/test_redis_chaos_e2e.py index c800ccb0e51..518bbd262a9 100644 --- a/tests/e2e/load/test_redis_chaos_e2e.py +++ b/tests/e2e/load/test_redis_chaos_e2e.py @@ -84,11 +84,11 @@ CHAOS_CPU_PER_REQUEST_RATIO_CEILING: Final = 2.0 # the breaker opens, a request skips Redis rather than waiting on its socket timeout, so the chaos # phase can come in faster than baseline (local runs measured p90 at 0.61x) and a ratio passes on a # phase that was never slow. What a user actually cares about is the wall-clock number, which these -# hold directly. Calibrated from local runs whose worst chaos phase was p50 0.66s, p90 0.74s, p99 -# 1.20s and 3.4 KB of log per request, then left roughly 3x loose for a shared CI runner. -CHAOS_P50_LATENCY_CEILING_SECONDS: Final = 2.0 -CHAOS_P90_LATENCY_CEILING_SECONDS: Final = 3.0 -CHAOS_P99_LATENCY_CEILING_SECONDS: Final = 5.0 +# hold directly. Calibrated from local runs whose worst chaos phase was p50 0.19s, p90 0.23s, p99 +# 0.69s and 3.5 KB of log per request, with several times that left as slack for a shared CI runner. +CHAOS_P50_LATENCY_CEILING_SECONDS: Final = 1.0 +CHAOS_P90_LATENCY_CEILING_SECONDS: Final = 2.0 +CHAOS_P99_LATENCY_CEILING_SECONDS: Final = 3.0 CHAOS_LOG_BYTES_PER_REQUEST_CEILING: Final = 12_000.0 DRAIN_TIMEOUT_SECONDS: Final = 30.0 From 0ea18f28af5d2320fa06aac960d06ce9f6898a4f Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 13:45:42 -0700 Subject: [PATCH 17/27] test(e2e): tighten chaos log-bytes ceiling to 10 KB per request Local runs measured 3.5 KB per request, so 10 KB keeps close to 3x headroom while tightening from the earlier 12 KB. Co-Authored-By: Claude Code --- tests/e2e/load/test_redis_chaos_e2e.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/load/test_redis_chaos_e2e.py b/tests/e2e/load/test_redis_chaos_e2e.py index 518bbd262a9..0732f725f34 100644 --- a/tests/e2e/load/test_redis_chaos_e2e.py +++ b/tests/e2e/load/test_redis_chaos_e2e.py @@ -89,7 +89,7 @@ CHAOS_CPU_PER_REQUEST_RATIO_CEILING: Final = 2.0 CHAOS_P50_LATENCY_CEILING_SECONDS: Final = 1.0 CHAOS_P90_LATENCY_CEILING_SECONDS: Final = 2.0 CHAOS_P99_LATENCY_CEILING_SECONDS: Final = 3.0 -CHAOS_LOG_BYTES_PER_REQUEST_CEILING: Final = 12_000.0 +CHAOS_LOG_BYTES_PER_REQUEST_CEILING: Final = 10_000.0 DRAIN_TIMEOUT_SECONDS: Final = 30.0 DRAIN_POLL_SECONDS: Final = 1.0 From a8d017180079aff46a4e3b9612bdefa5a3e323ac Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 14:38:10 -0700 Subject: [PATCH 18/27] ci: gate stable and RC releases on the Redis chaos load test The chaos test only ran on a weekly cron, so a release could be cut from a commit it had never covered. Making it callable lets create-release.yml run it against the exact commit being tagged and refuse to tag if it fails. Dev, nightly, alpha and beta tags skip the gate: they are cut far more often than stable and RC tags, and the weekly schedule already covers the default branch. Input validation moves into the gate job so a malformed tag or SHA fails before spending a multi-minute chaos run. Co-Authored-By: Claude Code --- .github/workflows/create-release.yml | 42 +++++++++++++++++++--- .github/workflows/test-e2e-redis-chaos.yml | 7 ++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 0ad84cd3ceb..bdc1810f204 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -15,11 +15,14 @@ on: permissions: {} jobs: - release: - name: Create Release + # Stable and RC tags are gated on the Redis chaos load test; dev, nightly, alpha and beta + # tags are cut too often to spend a multi-minute chaos run on each one, and the weekly + # schedule already covers the default branch. + gate: + name: Validate inputs and decide whether this tag is gated runs-on: ubuntu-latest - permissions: - contents: write + outputs: + gated: ${{ steps.decide.outputs.gated }} steps: - name: Validate inputs env: @@ -35,6 +38,37 @@ jobs: exit 1 fi + - name: Decide + id: decide + env: + TAG: ${{ inputs.tag }} + run: | + if echo "${TAG}" | grep -qiE '(nightly|alpha|beta|[-.]dev)'; then + echo "gated=false" >> "$GITHUB_OUTPUT" + echo "${TAG} is a pre-release that skips the chaos gate" + else + echo "gated=true" >> "$GITHUB_OUTPUT" + echo "${TAG} is a stable or RC tag and must pass the chaos gate" + fi + + redis-chaos-gate: + name: Redis Chaos E2E + needs: gate + if: needs.gate.outputs.gated == 'true' + permissions: + contents: read + uses: ./.github/workflows/test-e2e-redis-chaos.yml + with: + ref: ${{ inputs.commit_hash }} + + release: + name: Create Release + needs: [gate, redis-chaos-gate] + if: always() && needs.gate.result == 'success' && (needs.redis-chaos-gate.result == 'success' || needs.redis-chaos-gate.result == 'skipped') + runs-on: ubuntu-latest + permissions: + contents: write + steps: - name: Create release env: TAG: ${{ inputs.tag }} diff --git a/.github/workflows/test-e2e-redis-chaos.yml b/.github/workflows/test-e2e-redis-chaos.yml index 0880c1fb464..9deca013603 100644 --- a/.github/workflows/test-e2e-redis-chaos.yml +++ b/.github/workflows/test-e2e-redis-chaos.yml @@ -4,6 +4,12 @@ on: schedule: - cron: "0 12 * * 6" workflow_dispatch: + workflow_call: + inputs: + ref: + description: "Commit SHA or ref to test. Defaults to the ref the workflow was triggered on" + required: false + type: string permissions: contents: read @@ -45,6 +51,7 @@ jobs: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: persist-credentials: false + ref: ${{ inputs.ref || github.sha }} - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 From 93c8ef1beb446a864eb74ef49ffd0f3229c8cc80 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 14:40:08 -0700 Subject: [PATCH 19/27] docs(e2e): note the release gate in the load/ harness guide Co-Authored-By: Claude Code --- tests/e2e/CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 419cfd6dedd..b8e9c8ccde3 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -18,7 +18,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) -- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven weekly by `.github/workflows/test-e2e-redis-chaos.yml`), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic +- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven weekly by `.github/workflows/test-e2e-redis-chaos.yml`, which `create-release.yml` also calls to gate stable and RC tags on the commit being released), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke From 9d91aff249b71b9f8e08b00e8d729e3fb1985c89 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 14:43:47 -0700 Subject: [PATCH 20/27] ci(release): rename the tag-decision job to stable-release-gate Co-Authored-By: Claude Code --- .github/workflows/create-release.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index bdc1810f204..56f336c258d 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -18,7 +18,7 @@ jobs: # Stable and RC tags are gated on the Redis chaos load test; dev, nightly, alpha and beta # tags are cut too often to spend a multi-minute chaos run on each one, and the weekly # schedule already covers the default branch. - gate: + stable-release-gate: name: Validate inputs and decide whether this tag is gated runs-on: ubuntu-latest outputs: @@ -53,8 +53,8 @@ jobs: redis-chaos-gate: name: Redis Chaos E2E - needs: gate - if: needs.gate.outputs.gated == 'true' + needs: stable-release-gate + if: needs.stable-release-gate.outputs.gated == 'true' permissions: contents: read uses: ./.github/workflows/test-e2e-redis-chaos.yml @@ -63,8 +63,8 @@ jobs: release: name: Create Release - needs: [gate, redis-chaos-gate] - if: always() && needs.gate.result == 'success' && (needs.redis-chaos-gate.result == 'success' || needs.redis-chaos-gate.result == 'skipped') + needs: [stable-release-gate, redis-chaos-gate] + if: always() && needs.stable-release-gate.result == 'success' && (needs.redis-chaos-gate.result == 'success' || needs.redis-chaos-gate.result == 'skipped') runs-on: ubuntu-latest permissions: contents: write From cdf8a8cab8b56b0c70a5eee437e9e97eb002fdf6 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 14:47:57 -0700 Subject: [PATCH 21/27] ci(release): rename the tag-classification job to prepare Co-Authored-By: Claude Code --- .github/workflows/create-release.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 56f336c258d..9100349440e 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -18,7 +18,7 @@ jobs: # Stable and RC tags are gated on the Redis chaos load test; dev, nightly, alpha and beta # tags are cut too often to spend a multi-minute chaos run on each one, and the weekly # schedule already covers the default branch. - stable-release-gate: + prepare: name: Validate inputs and decide whether this tag is gated runs-on: ubuntu-latest outputs: @@ -53,8 +53,8 @@ jobs: redis-chaos-gate: name: Redis Chaos E2E - needs: stable-release-gate - if: needs.stable-release-gate.outputs.gated == 'true' + needs: prepare + if: needs.prepare.outputs.gated == 'true' permissions: contents: read uses: ./.github/workflows/test-e2e-redis-chaos.yml @@ -63,8 +63,8 @@ jobs: release: name: Create Release - needs: [stable-release-gate, redis-chaos-gate] - if: always() && needs.stable-release-gate.result == 'success' && (needs.redis-chaos-gate.result == 'success' || needs.redis-chaos-gate.result == 'skipped') + needs: [prepare, redis-chaos-gate] + if: always() && needs.prepare.result == 'success' && (needs.redis-chaos-gate.result == 'success' || needs.redis-chaos-gate.result == 'skipped') runs-on: ubuntu-latest permissions: contents: write From 7ebf60f33e16cefe6a9cbf500870436059015c37 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 14:49:59 -0700 Subject: [PATCH 22/27] ci(release): rename the tag-classification job to classify-if-stable-release Co-Authored-By: Claude Code --- .github/workflows/create-release.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 9100349440e..7fd95badb0a 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -18,7 +18,7 @@ jobs: # Stable and RC tags are gated on the Redis chaos load test; dev, nightly, alpha and beta # tags are cut too often to spend a multi-minute chaos run on each one, and the weekly # schedule already covers the default branch. - prepare: + classify-if-stable-release: name: Validate inputs and decide whether this tag is gated runs-on: ubuntu-latest outputs: @@ -53,8 +53,8 @@ jobs: redis-chaos-gate: name: Redis Chaos E2E - needs: prepare - if: needs.prepare.outputs.gated == 'true' + needs: classify-if-stable-release + if: needs.classify-if-stable-release.outputs.gated == 'true' permissions: contents: read uses: ./.github/workflows/test-e2e-redis-chaos.yml @@ -63,8 +63,8 @@ jobs: release: name: Create Release - needs: [prepare, redis-chaos-gate] - if: always() && needs.prepare.result == 'success' && (needs.redis-chaos-gate.result == 'success' || needs.redis-chaos-gate.result == 'skipped') + needs: [classify-if-stable-release, redis-chaos-gate] + if: always() && needs.classify-if-stable-release.result == 'success' && (needs.redis-chaos-gate.result == 'success' || needs.redis-chaos-gate.result == 'skipped') runs-on: ubuntu-latest permissions: contents: write From 445b45f261184ac58c7ddad0ef3749cd7bb6d220 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 14:55:59 -0700 Subject: [PATCH 23/27] ci(release): rename the tag-classification job to run-stable-release-checks Co-Authored-By: Claude Code --- .github/workflows/create-release.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 7fd95badb0a..8d971001348 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -18,7 +18,7 @@ jobs: # Stable and RC tags are gated on the Redis chaos load test; dev, nightly, alpha and beta # tags are cut too often to spend a multi-minute chaos run on each one, and the weekly # schedule already covers the default branch. - classify-if-stable-release: + run-stable-release-checks: name: Validate inputs and decide whether this tag is gated runs-on: ubuntu-latest outputs: @@ -53,8 +53,8 @@ jobs: redis-chaos-gate: name: Redis Chaos E2E - needs: classify-if-stable-release - if: needs.classify-if-stable-release.outputs.gated == 'true' + needs: run-stable-release-checks + if: needs.run-stable-release-checks.outputs.gated == 'true' permissions: contents: read uses: ./.github/workflows/test-e2e-redis-chaos.yml @@ -63,8 +63,8 @@ jobs: release: name: Create Release - needs: [classify-if-stable-release, redis-chaos-gate] - if: always() && needs.classify-if-stable-release.result == 'success' && (needs.redis-chaos-gate.result == 'success' || needs.redis-chaos-gate.result == 'skipped') + needs: [run-stable-release-checks, redis-chaos-gate] + if: always() && needs.run-stable-release-checks.result == 'success' && (needs.redis-chaos-gate.result == 'success' || needs.redis-chaos-gate.result == 'skipped') runs-on: ubuntu-latest permissions: contents: write From 39324dc96038ff8a9f0b43fb07da9cec26d315b1 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 14:59:35 -0700 Subject: [PATCH 24/27] ci(release): rename redis-chaos-gate to redis-chaos-check Co-Authored-By: Claude Code --- .github/workflows/create-release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 8d971001348..c51bf02bdc7 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -51,7 +51,7 @@ jobs: echo "${TAG} is a stable or RC tag and must pass the chaos gate" fi - redis-chaos-gate: + redis-chaos-check: name: Redis Chaos E2E needs: run-stable-release-checks if: needs.run-stable-release-checks.outputs.gated == 'true' @@ -63,8 +63,8 @@ jobs: release: name: Create Release - needs: [run-stable-release-checks, redis-chaos-gate] - if: always() && needs.run-stable-release-checks.result == 'success' && (needs.redis-chaos-gate.result == 'success' || needs.redis-chaos-gate.result == 'skipped') + needs: [run-stable-release-checks, redis-chaos-check] + if: always() && needs.run-stable-release-checks.result == 'success' && (needs.redis-chaos-check.result == 'success' || needs.redis-chaos-check.result == 'skipped') runs-on: ubuntu-latest permissions: contents: write From 7d8f2c9ad3fa9c4ed0409ffb1a36663ab6f0e97a Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 15:00:57 -0700 Subject: [PATCH 25/27] ci(e2e): drop the weekly cron for the Redis chaos test Now that create-release.yml gates stable and RC releases on this test directly, the weekly schedule is redundant: every release gets a run against its own commit instead of whatever happened to be on the default branch that Saturday. workflow_dispatch stays for manual runs. Co-Authored-By: Claude Code --- .github/workflows/create-release.yml | 3 +-- .github/workflows/test-e2e-redis-chaos.yml | 5 +---- tests/e2e/CLAUDE.md | 2 +- tests/e2e/CONTRIBUTING.md | 2 +- tests/e2e/load/test_redis_chaos_e2e.py | 2 +- 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index c51bf02bdc7..5bf10ef324f 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -16,8 +16,7 @@ permissions: {} jobs: # Stable and RC tags are gated on the Redis chaos load test; dev, nightly, alpha and beta - # tags are cut too often to spend a multi-minute chaos run on each one, and the weekly - # schedule already covers the default branch. + # tags are cut too often to spend a multi-minute chaos run on each one. run-stable-release-checks: name: Validate inputs and decide whether this tag is gated runs-on: ubuntu-latest diff --git a/.github/workflows/test-e2e-redis-chaos.yml b/.github/workflows/test-e2e-redis-chaos.yml index 9deca013603..c7412a63334 100644 --- a/.github/workflows/test-e2e-redis-chaos.yml +++ b/.github/workflows/test-e2e-redis-chaos.yml @@ -1,8 +1,6 @@ -name: "Weekly Redis Chaos E2E" +name: "Redis Chaos E2E" on: - schedule: - - cron: "0 12 * * 6" workflow_dispatch: workflow_call: inputs: @@ -16,7 +14,6 @@ permissions: jobs: redis-chaos-e2e: - if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest-16-cores timeout-minutes: 30 services: diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index b8e9c8ccde3..20cd0c35465 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -18,7 +18,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) -- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven weekly by `.github/workflows/test-e2e-redis-chaos.yml`, which `create-release.yml` also calls to gate stable and RC tags on the commit being released), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic +- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven by `.github/workflows/test-e2e-redis-chaos.yml`, which `create-release.yml` calls to gate stable and RC tags on the commit being released), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index b1331de190a..b6faebed867 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -65,7 +65,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots on a weekly schedule +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots when `create-release.yml` calls it to gate a stable or RC release Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/load/test_redis_chaos_e2e.py b/tests/e2e/load/test_redis_chaos_e2e.py index 0732f725f34..9a9e8082e5b 100644 --- a/tests/e2e/load/test_redis_chaos_e2e.py +++ b/tests/e2e/load/test_redis_chaos_e2e.py @@ -76,7 +76,7 @@ REDIS_PAUSE_MS: Final = int(CHAOS_SECONDS * 1000) # identical local runs, so it stays loose; CPU per request held steady at 1.33x-1.36x across the # same runs, so it sits close to what is actually measured. That makes CPU the likeliest of these # to flake first on a runner whose core count shifts how much of baseline CPU is fixed per-request -# work: loosen it rather than widening the others if a weekly run trips it without a real cause. +# work: loosen it rather than widening the others if a CI run trips it without a real cause. CHAOS_RSS_RATIO_CEILING: Final = 2.0 CHAOS_CPU_PER_REQUEST_RATIO_CEILING: Final = 2.0 From d4e083348ca5edfbd45142fd23dbe08f3f3cb9d0 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 15:14:19 -0700 Subject: [PATCH 26/27] revert: drop the create-release.yml gating and E2E_REDIS_CHAOS opt-in create-release.yml is back to calling the chaos test through no mechanism at all; it never called it. Also drops the E2E_REDIS_CHAOS opt-in gate itself: the redis_chaos marker still exists for -m selection and is still excluded from the per-PR selector by path (tests/e2e/(ui|claude_code|load)/), but the test no longer needs an env var to run once its file is targeted. Co-Authored-By: Claude Code --- .github/workflows/create-release.yml | 41 +++------------------- .github/workflows/test-e2e-redis-chaos.yml | 1 - tests/e2e/CLAUDE.md | 2 +- tests/e2e/CONTRIBUTING.md | 2 +- tests/e2e/conftest.py | 2 +- tests/e2e/e2e_config.py | 1 - tests/e2e/load/conftest.py | 8 ++--- tests/e2e/load/test_redis_chaos_e2e.py | 3 +- tests/e2e/pytest.ini | 2 +- 9 files changed, 11 insertions(+), 51 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 5bf10ef324f..0ad84cd3ceb 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -15,13 +15,11 @@ on: permissions: {} jobs: - # Stable and RC tags are gated on the Redis chaos load test; dev, nightly, alpha and beta - # tags are cut too often to spend a multi-minute chaos run on each one. - run-stable-release-checks: - name: Validate inputs and decide whether this tag is gated + release: + name: Create Release runs-on: ubuntu-latest - outputs: - gated: ${{ steps.decide.outputs.gated }} + permissions: + contents: write steps: - name: Validate inputs env: @@ -37,37 +35,6 @@ jobs: exit 1 fi - - name: Decide - id: decide - env: - TAG: ${{ inputs.tag }} - run: | - if echo "${TAG}" | grep -qiE '(nightly|alpha|beta|[-.]dev)'; then - echo "gated=false" >> "$GITHUB_OUTPUT" - echo "${TAG} is a pre-release that skips the chaos gate" - else - echo "gated=true" >> "$GITHUB_OUTPUT" - echo "${TAG} is a stable or RC tag and must pass the chaos gate" - fi - - redis-chaos-check: - name: Redis Chaos E2E - needs: run-stable-release-checks - if: needs.run-stable-release-checks.outputs.gated == 'true' - permissions: - contents: read - uses: ./.github/workflows/test-e2e-redis-chaos.yml - with: - ref: ${{ inputs.commit_hash }} - - release: - name: Create Release - needs: [run-stable-release-checks, redis-chaos-check] - if: always() && needs.run-stable-release-checks.result == 'success' && (needs.redis-chaos-check.result == 'success' || needs.redis-chaos-check.result == 'skipped') - runs-on: ubuntu-latest - permissions: - contents: write - steps: - name: Create release env: TAG: ${{ inputs.tag }} diff --git a/.github/workflows/test-e2e-redis-chaos.yml b/.github/workflows/test-e2e-redis-chaos.yml index c7412a63334..6b066739c95 100644 --- a/.github/workflows/test-e2e-redis-chaos.yml +++ b/.github/workflows/test-e2e-redis-chaos.yml @@ -91,7 +91,6 @@ jobs: - name: Run the Redis chaos load test env: - E2E_REDIS_CHAOS: "1" LITELLM_PROXY_URL: http://localhost:4000 REDIS_HOST: 127.0.0.1 REDIS_PORT: "6379" diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 20cd0c35465..f9ad73b4508 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -18,7 +18,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) -- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven by `.github/workflows/test-e2e-redis-chaos.yml`, which `create-release.yml` calls to gate stable and RC tags on the commit being released), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic +- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml`), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index b6faebed867..a82e8ea43f5 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -65,7 +65,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots when `create-release.yml` calls it to gate a stable or RC release +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 700dc822d59..72448201f8b 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -61,7 +61,7 @@ def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line( "markers", "redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from " - "gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set", + "gateway/redis_chaos_ci_config.yml on the same host", ) diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 7344a2b6cec..86f0c616c04 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -143,7 +143,6 @@ LOAD_MIN_CONCURRENCY_EFFICIENCY = float(os.environ.get("E2E_LOAD_MIN_CONCURRENCY WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY" MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK" -REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) diff --git a/tests/e2e/load/conftest.py b/tests/e2e/load/conftest.py index e6f3c7aa538..7c749837225 100644 --- a/tests/e2e/load/conftest.py +++ b/tests/e2e/load/conftest.py @@ -3,15 +3,11 @@ from __future__ import annotations import os import pytest - -from e2e_config import REDIS_CHAOS_OPT_IN_ENV, WEEKLY_ANOMALY_OPT_IN_ENV +from e2e_config import WEEKLY_ANOMALY_OPT_IN_ENV from load_client import LoadClient, build_client from proxy_client import ProxyClient -_OPT_IN_MARKERS = ( - ("weekly", WEEKLY_ANOMALY_OPT_IN_ENV), - ("redis_chaos", REDIS_CHAOS_OPT_IN_ENV), -) +_OPT_IN_MARKERS = (("weekly", WEEKLY_ANOMALY_OPT_IN_ENV),) def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: diff --git a/tests/e2e/load/test_redis_chaos_e2e.py b/tests/e2e/load/test_redis_chaos_e2e.py index 9a9e8082e5b..2e1e2a92bd3 100644 --- a/tests/e2e/load/test_redis_chaos_e2e.py +++ b/tests/e2e/load/test_redis_chaos_e2e.py @@ -27,8 +27,7 @@ per-phase RSS, CPU, and log-bytes budgets are here to catch. Needs the proxy on the same host, since RSS and CPU come from psutil on its process tree: a multi-worker proxy serves /metrics from the prometheus multiprocess collector, which drops the process collector's memory and CPU series. Log bytes are read from the file the proxy's -stdout/stderr was redirected to, so the same host requirement covers that too. Deselected -unless E2E_REDIS_CHAOS is set. +stdout/stderr was redirected to, so the same host requirement covers that too. """ from __future__ import annotations diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index 774d9644497..28f7fc011f7 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -9,4 +9,4 @@ markers = load: heavy throughput/load test; collected last so it never perturbs latency-sensitive suites weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set - redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set + redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host From ffc16a4b0e467005b8e4387fb0cc449f57cf009c Mon Sep 17 00:00:00 2001 From: kerry Date: Fri, 11 Sep 2026 22:46:31 +0000 Subject: [PATCH 27/27] test(e2e): restore the E2E_REDIS_CHAOS opt-in for the redis chaos test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-e2e-redis-chaos.yml | 1 + tests/e2e/CLAUDE.md | 2 +- tests/e2e/CONTRIBUTING.md | 2 +- tests/e2e/conftest.py | 2 +- tests/e2e/e2e_config.py | 1 + tests/e2e/load/conftest.py | 7 +++++-- tests/e2e/load/test_redis_chaos_e2e.py | 3 ++- tests/e2e/pytest.ini | 2 +- 8 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-e2e-redis-chaos.yml b/.github/workflows/test-e2e-redis-chaos.yml index 6b066739c95..c7412a63334 100644 --- a/.github/workflows/test-e2e-redis-chaos.yml +++ b/.github/workflows/test-e2e-redis-chaos.yml @@ -91,6 +91,7 @@ jobs: - name: Run the Redis chaos load test env: + E2E_REDIS_CHAOS: "1" LITELLM_PROXY_URL: http://localhost:4000 REDIS_HOST: 127.0.0.1 REDIS_PORT: "6379" diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index f9ad73b4508..8be2e7b4ce2 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -18,7 +18,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) -- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml`), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic +- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index a82e8ea43f5..f1f7b17d86b 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -65,7 +65,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 72448201f8b..700dc822d59 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -61,7 +61,7 @@ def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line( "markers", "redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from " - "gateway/redis_chaos_ci_config.yml on the same host", + "gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set", ) diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 86f0c616c04..7344a2b6cec 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -143,6 +143,7 @@ LOAD_MIN_CONCURRENCY_EFFICIENCY = float(os.environ.get("E2E_LOAD_MIN_CONCURRENCY WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY" MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK" +REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) diff --git a/tests/e2e/load/conftest.py b/tests/e2e/load/conftest.py index 7c749837225..fa608d157cc 100644 --- a/tests/e2e/load/conftest.py +++ b/tests/e2e/load/conftest.py @@ -3,11 +3,14 @@ from __future__ import annotations import os import pytest -from e2e_config import WEEKLY_ANOMALY_OPT_IN_ENV +from e2e_config import REDIS_CHAOS_OPT_IN_ENV, WEEKLY_ANOMALY_OPT_IN_ENV from load_client import LoadClient, build_client from proxy_client import ProxyClient -_OPT_IN_MARKERS = (("weekly", WEEKLY_ANOMALY_OPT_IN_ENV),) +_OPT_IN_MARKERS = ( + ("weekly", WEEKLY_ANOMALY_OPT_IN_ENV), + ("redis_chaos", REDIS_CHAOS_OPT_IN_ENV), +) def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: diff --git a/tests/e2e/load/test_redis_chaos_e2e.py b/tests/e2e/load/test_redis_chaos_e2e.py index 2e1e2a92bd3..9a9e8082e5b 100644 --- a/tests/e2e/load/test_redis_chaos_e2e.py +++ b/tests/e2e/load/test_redis_chaos_e2e.py @@ -27,7 +27,8 @@ per-phase RSS, CPU, and log-bytes budgets are here to catch. Needs the proxy on the same host, since RSS and CPU come from psutil on its process tree: a multi-worker proxy serves /metrics from the prometheus multiprocess collector, which drops the process collector's memory and CPU series. Log bytes are read from the file the proxy's -stdout/stderr was redirected to, so the same host requirement covers that too. +stdout/stderr was redirected to, so the same host requirement covers that too. Deselected +unless E2E_REDIS_CHAOS is set. """ from __future__ import annotations diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index 28f7fc011f7..774d9644497 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -9,4 +9,4 @@ markers = load: heavy throughput/load test; collected last so it never perturbs latency-sensitive suites weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set - redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host + redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set