From 058ff8c63c222d52392814b0fda84d5b030a84ec Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 9 Sep 2026 15:33:44 -0700 Subject: [PATCH] test(e2e): keep answering while every Redis command times out Add tests/e2e/router/test_redis_timeout_e2e.py against a proxy booted from tests/e2e/gateway/redis_timeout_ci_config.yml: a real Redis with socket_timeout 0.001, so every command times out and the circuit breaker opens, plus a primary deployment that always fails and falls back to a healthy one, so every request carries retry breadcrumbs into cost tracking. The test drives twenty chat requests through the proxy and asserts each answers within ten seconds, the last third is no slower than the first, /health/liveliness stays fast, and every request still reaches the spend log. Gate it behind the redis_timeout marker and E2E_REDIS_TIMEOUT, exclude it from the per-PR e2e-changed selector, register the reliability.circuit_breaker.redis_timeout.stays_responsive cell, and run it as its own job in the weekly load anomaly workflow with a Postgres and Valkey service. Against a v1.100.0 proxy the run wedges the worker: requests time out and liveliness stops answering (LIT-6780). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014ZDULyJPp17ZFiJenRxs2T --- .github/e2e-stack/select_tests.py | 1 + .github/workflows/weekly_load_anomaly.yml | 84 +++++++++++++++++ tests/e2e/CLAUDE.md | 2 +- tests/e2e/CONTRIBUTING.md | 2 +- tests/e2e/conftest.py | 9 +- tests/e2e/coverage_registry/reliability.yaml | 1 + tests/e2e/e2e_config.py | 5 +- tests/e2e/gateway/redis_timeout_ci_config.yml | 29 ++++++ tests/e2e/pytest.ini | 1 + tests/e2e/router/conftest.py | 21 +++-- tests/e2e/router/test_redis_timeout_e2e.py | 93 +++++++++++++++++++ 11 files changed, 234 insertions(+), 14 deletions(-) create mode 100644 tests/e2e/gateway/redis_timeout_ci_config.yml create mode 100644 tests/e2e/router/test_redis_timeout_e2e.py diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py index a62358f81ff..10f26bd3b8a 100644 --- a/.github/e2e-stack/select_tests.py +++ b/.github/e2e-stack/select_tests.py @@ -8,6 +8,7 @@ UNSUPPORTED: Final = re.compile( r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$" r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$" r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$" + r"|^tests/e2e/router/test_redis_timeout_e2e\.py$" ) HARNESS: Final = re.compile( r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" diff --git a/.github/workflows/weekly_load_anomaly.yml b/.github/workflows/weekly_load_anomaly.yml index 3e1fca89645..6e564cf7e78 100644 --- a/.github/workflows/weekly_load_anomaly.yml +++ b/.github/workflows/weekly_load_anomaly.yml @@ -83,3 +83,87 @@ jobs: - name: Show proxy log on failure if: failure() run: tail -n 300 proxy.log + + redis-timeout-e2e: + if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + postgres: + image: postgres:16.6 + env: + POSTGRES_USER: llmproxy + POSTGRES_PASSWORD: dbpassword9090 + POSTGRES_DB: litellm + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U llmproxy" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + valkey: + image: valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa + ports: + - 6379:6379 + options: >- + --health-cmd "valkey-cli ping" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm + LITELLM_MASTER_KEY: sk-redis-timeout-e2e + LITELLM_LOG: WARNING + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build + + - name: Install dependencies + run: | + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy + + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + + - name: Generate Prisma client + run: | + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + - name: Start the proxy with a Redis that times out on every command + run: | + nohup uv run --no-sync litellm --config tests/e2e/gateway/redis_timeout_ci_config.yml --port 4000 > proxy.log 2>&1 & + for _ in $(seq 1 90); do + if curl -fs http://localhost:4000/health/liveliness > /dev/null; then + exit 0 + fi + sleep 2 + done + echo "proxy never became live" + tail -n 100 proxy.log + exit 1 + + - name: Run the Redis timeout e2e test + env: + E2E_REDIS_TIMEOUT: "1" + LITELLM_PROXY_URL: http://localhost:4000 + run: | + uv run --no-sync pytest tests/e2e/router/test_redis_timeout_e2e.py -v --tb=short -rA + + - name: Show proxy log on failure + if: failure() + run: tail -n 300 proxy.log diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 89c04208d65..53342f66b67 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -17,7 +17,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server (see "MCP suite: real Datadog only" below); plus the gateway-managed OAuth (authorization_code) path exercised through `/chat/completions`, the one behavior Datadog's static-header auth cannot reach, seeding the per-user upstream token via the interactive authorize dance driven with the mcp SDK's own OAuth client (headless-browser consent from a saved session) and asserting the completion lists and executes the server's tools with the stored per-user token - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection -- `router/` - routing and reliability behavior (fallbacks, cooldowns) +- `router/` - routing and reliability behavior (fallbacks, cooldowns). Also holds `test_redis_timeout_e2e.py`, which needs a proxy booted from `gateway/redis_timeout_ci_config.yml` (real Redis, `socket_timeout: 0.001`, so every command times out); it is marked `redis_timeout`, deselected unless `E2E_REDIS_TIMEOUT` is set, excluded from the per-PR check, and driven by `.github/workflows/weekly_load_anomaly.yml` - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What remains here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`) and markerless harness unit tests for the Locust/session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 1183096b81e..ccd27df6f46 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -65,7 +65,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, `guardrails/test_presidio_masking_e2e.py`, and `router/test_redis_timeout_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start, and the Redis timeout test needs a proxy whose Redis times out on every command (`gateway/redis_timeout_ci_config.yml`), which the weekly load workflow boots Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index e1b987cbfd9..ff3d2dcc6bd 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -20,16 +20,14 @@ from datetime import datetime, timezone import pytest import requests - from e2e_config import CONTROL_PLANE_BASE_URL, FIXTURE_DIR, FIXTURE_MODE_RAW, PROXY_BASE_URL from e2e_db import RESET_OPT_IN_ENV, reset_spend_logs, run_spend_log_cleanup from fixture_mode import fixture_mode_collection_error, fixture_report_lines -from provider_edge import replay_leftover_error from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager +from provider_edge import replay_leftover_error from proxy_client import ProxyClient, build_proxy_client - _E2E_TEST_RAN = pytest.StashKey[bool]() _CALL_PASSED = pytest.StashKey[bool]() @@ -60,6 +58,11 @@ def pytest_configure(config: pytest.Config) -> None: "markers", "managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set", ) + config.addinivalue_line( + "markers", + "redis_timeout: needs a proxy booted from gateway/redis_timeout_ci_config.yml whose Redis times out on every " + "command; deselected unless E2E_REDIS_TIMEOUT is set", + ) def pytest_sessionstart(session: pytest.Session) -> None: diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 6b69677d490..d62f0105931 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -31,6 +31,7 @@ - {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"} - {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix"} - {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"} +- {id: reliability.circuit_breaker.redis_timeout.stays_responsive, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis_timeout, assertions: [stays_responsive], exercised_on: [chat_completions], source: "litellm/proxy/hooks/proxy_track_cost_callback.py:386", fail_before_fix: proven, rationale: "With every Redis command timing out and every request retrying then falling back, per-request latency stays flat, liveliness stays fast, and spend rows still land; on v1.100.0 the failed-tracking alert body doubled per request until the worker OOMed (LIT-6780)"} - {id: reliability.timeout.request_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: request_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions, messages], source: "litellm/router.py:545-551", rationale: "Per-request timeout raises Timeout"} - {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"} - {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 691335ffdd5..6cb59ea4e90 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -13,7 +13,6 @@ from pathlib import Path from typing import Final from dotenv import load_dotenv - from fixture_mode import deterministic_marker, parse_fixture_mode from provider_edge import provider_edge_api_base @@ -144,6 +143,7 @@ LOAD_MIN_CONCURRENCY_EFFICIENCY = float(os.environ.get("E2E_LOAD_MIN_CONCURRENCY WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY" MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK" +REDIS_TIMEOUT_OPT_IN_ENV = "E2E_REDIS_TIMEOUT" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) @@ -181,8 +181,7 @@ def datadog_mcp_url(*, toolsets: str = "core") -> str: site = ( os.environ.get("DD_SITE", DD_SITE) or "datadoghq.com" ).strip().removeprefix("https://").removeprefix("http://").rstrip("/") - if site.startswith("app."): - site = site[len("app.") :] + site = site.removeprefix("app.") host = "mcp.datadoghq.com" if site in ("", "datadoghq.com") else f"mcp.{site}" base = f"https://{host}/v1/mcp" return f"{base}?toolsets={toolsets}" if toolsets else base diff --git a/tests/e2e/gateway/redis_timeout_ci_config.yml b/tests/e2e/gateway/redis_timeout_ci_config.yml new file mode 100644 index 00000000000..69ab2ee14dc --- /dev/null +++ b/tests/e2e/gateway/redis_timeout_ci_config.yml @@ -0,0 +1,29 @@ +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + store_model_in_db: true + +litellm_settings: + cache: true + cache_params: + type: redis + host: 127.0.0.1 + port: 6379 + socket_timeout: 0.001 + +router_settings: + num_retries: 1 + fallbacks: + - redis-timeout-primary: + - redis-timeout-backup + +model_list: + - model_name: redis-timeout-primary + litellm_params: + model: openai/gpt-5-mini + api_key: sk-redis-timeout-primary-not-used + mock_response: "litellm.InternalServerError" + - model_name: redis-timeout-backup + litellm_params: + model: openai/gpt-5-mini + api_key: sk-redis-timeout-backup-not-used + mock_response: "ok" diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index c3f8865f218..f69d5d058f3 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -9,3 +9,4 @@ markers = load: heavy throughput/load test; collected last so it never perturbs latency-sensitive suites weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set + redis_timeout: needs a proxy booted from gateway/redis_timeout_ci_config.yml whose Redis times out on every command; deselected unless E2E_REDIS_TIMEOUT is set diff --git a/tests/e2e/router/conftest.py b/tests/e2e/router/conftest.py index 98501f9bd7c..57fde729f41 100644 --- a/tests/e2e/router/conftest.py +++ b/tests/e2e/router/conftest.py @@ -10,13 +10,12 @@ proxy does not already list it (compose has it in static config; stage does not) from __future__ import annotations +import os from collections.abc import Iterator import pytest -from requests import RequestException - from complexity_router_client import ComplexityRouterClient, build_client -from proxy_client import ProxyClient +from e2e_config import REDIS_TIMEOUT_OPT_IN_ENV from e2e_http import NoBody, Success from lifecycle import ResourceManager from models import ( @@ -26,6 +25,8 @@ from models import ( LiteLLMParamsBody, ModelsListResponse, ) +from proxy_client import ProxyClient +from requests import RequestException ROUTER_MODEL = "complexity-smart-router" ROUTER_PARAMS = LiteLLMParamsBody( @@ -45,6 +46,16 @@ ROUTER_PARAMS = LiteLLMParamsBody( ROUTER_KEY_MODELS = [ROUTER_MODEL, "gpt-5.5", "claude-haiku-4-5"] +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + if os.environ.get(REDIS_TIMEOUT_OPT_IN_ENV): + return + deselected = [item for item in items if item.get_closest_marker("redis_timeout") is not None] + if not deselected: + return + config.hook.pytest_deselected(items=deselected) + items[:] = [item for item in items if item.get_closest_marker("redis_timeout") is None] + + @pytest.fixture(scope="session") def client(proxy: ProxyClient) -> ComplexityRouterClient: return build_client(proxy) @@ -120,8 +131,6 @@ def _ensure_complexity_smart_router( # pyright: ignore[reportUnusedFunction] # @pytest.fixture def complexity_key(resources: ResourceManager, client: ComplexityRouterClient) -> str: """Per-test key allowed to call the complexity router and its tier backends.""" - key = client.proxy.generate_key( - KeyGenerateBody(models=ROUTER_KEY_MODELS, user_id="e2e-complexity-router") - ) + key = client.proxy.generate_key(KeyGenerateBody(models=ROUTER_KEY_MODELS, user_id="e2e-complexity-router")) resources.defer(lambda: client.proxy.delete_key(key)) return key diff --git a/tests/e2e/router/test_redis_timeout_e2e.py b/tests/e2e/router/test_redis_timeout_e2e.py new file mode 100644 index 00000000000..eeb5224e051 --- /dev/null +++ b/tests/e2e/router/test_redis_timeout_e2e.py @@ -0,0 +1,93 @@ +"""Live e2e: the proxy keeps answering while every Redis command times out. + +Runs only against a proxy booted from tests/e2e/gateway/redis_timeout_ci_config.yml, which +points cache_params at a real Redis with socket_timeout 0.001 so every command times out and +the circuit breaker opens. Each request fails its primary deployment, retries, falls back to the +backup and succeeds, so it carries retry breadcrumbs; its cost tracking then fails on the spend +counter increment and stringifies the request metadata into a failed-tracking alert. On v1.100.0 +that string doubled per request until the worker hung (LIT-6780). Deselected unless +E2E_REDIS_TIMEOUT is set, since it needs that dedicated proxy. +""" + +from __future__ import annotations + +import time +from typing import Final + +import pytest +from complexity_router_client import ComplexityRouterClient +from e2e_config import unique_marker +from e2e_http import NoBody, Success +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, ChatResponse, KeyGenerateBody + +pytestmark = [pytest.mark.e2e, pytest.mark.redis_timeout] + +PRIMARY_MODEL: Final = "redis-timeout-primary" +BACKUP_MODEL: Final = "redis-timeout-backup" +REQUESTS: Final = 20 +MAX_SECONDS_PER_REQUEST: Final = 10.0 +MAX_LATENCY_GROWTH_RATIO: Final = 3.0 +MAX_LIVELINESS_SECONDS: Final = 2.0 + + +class TestRedisTimeout: + @pytest.mark.covers( + "reliability.circuit_breaker.redis_timeout.stays_responsive", + exercised_on=["chat_completions"], + ) + def test_retries_under_redis_timeouts_keep_answering( + self, client: ComplexityRouterClient, resources: ResourceManager + ) -> None: + proxy = client.proxy + key = proxy.generate_key( + KeyGenerateBody(models=[PRIMARY_MODEL, BACKUP_MODEL], key_alias=f"e2e-redis-timeout-{unique_marker()}") + ) + resources.defer(lambda: proxy.delete_key(key)) + + latencies: list[float] = [] + for request_number in range(1, REQUESTS + 1): + started = time.monotonic() + result = proxy.transport.post( + "/chat/completions", + headers=proxy.transport.bearer(key), + json=ChatBody( + model=PRIMARY_MODEL, + messages=[ChatMessage(role="user", content=f"redis timeout {unique_marker()} {request_number}")], + max_tokens=5, + ), + response_type=ChatResponse, + timeout=MAX_SECONDS_PER_REQUEST, + ) + elapsed = time.monotonic() - started + assert isinstance(result, Success), ( + f"request {request_number} failed after {elapsed:.1f}s with Redis timing out: {result}; " + f"earlier requests took {[round(seconds, 2) for seconds in latencies]}" + ) + assert result.data.choices, f"request {request_number}: fallback to {BACKUP_MODEL} returned no choices" + assert elapsed < MAX_SECONDS_PER_REQUEST, ( + f"request {request_number} took {elapsed:.1f}s with Redis timing out; " + f"earlier requests took {[round(seconds, 2) for seconds in latencies]}" + ) + latencies.append(elapsed) + + third = REQUESTS // 3 + early = sum(latencies[:third]) / third + late = sum(latencies[-third:]) / third + assert late <= max(early * MAX_LATENCY_GROWTH_RATIO, 0.5), ( + f"per-request latency grew from {early:.2f}s to {late:.2f}s across {REQUESTS} requests " + "while Redis timed out; the proxy is paying more for each failed request" + ) + + started = time.monotonic() + probe = proxy.transport.probe("/health/liveliness", params=NoBody()) + liveliness_seconds = time.monotonic() - started + assert probe.healthy, f"/health/liveliness returned {probe.status_code} after the Redis timeout loop" + assert liveliness_seconds < MAX_LIVELINESS_SECONDS, ( + f"/health/liveliness took {liveliness_seconds:.1f}s after the loop; the worker is stalled" + ) + + rows = proxy.poll_logs_for_key(key, min_rows=REQUESTS) + assert len(rows) >= REQUESTS, ( + f"only {len(rows)} of {REQUESTS} requests reached the spend log; a Redis outage must not lose spend rows" + )