test(e2e): add reliability suite covering fallback, timeout, and cache behavior (#34023)

* test(e2e): add reliability suite covering fallback, timeout, and cache behavior

* test(e2e): move reliability suite under router and drive it with real deployments

* test(e2e): make the router complexity fixture opt-in so reliability tests can coexist

tests/e2e/models.py

@@ -166,6 +166,27 @@ class ChatBody(BaseModel):
     guardrails: list[str] | None = None
 
 
+class RouterSettingsOverride(BaseModel):
+    """Per-request `router_settings_override` in a /chat/completions body: the
+    reliability knobs (fallbacks by trigger, retry count) the reliability suite
+    drives per call instead of via static router config. Serialized exclude_none, so
+    an override sets only the strategies a test exercises. Each fallbacks map is
+    model_name -> the ordered fallback model_names to try."""
+
+    fallbacks: list[dict[str, list[str]]] | None = None
+    context_window_fallbacks: list[dict[str, list[str]]] | None = None
+    content_policy_fallbacks: list[dict[str, list[str]]] | None = None
+    num_retries: int | None = None
+
+
+class ReliabilityChatBody(ChatBody):
+    """A /chat/completions body carrying a per-request router_settings_override.
+    Composes ChatBody (no attribute repetition) and adds the override; serialized
+    exclude_none so an absent override never leaks into the request."""
+
+    router_settings_override: RouterSettingsOverride | None = None
+
+
 class OutMessage(BaseModel):
     content: str | None = None
     reasoning_content: str | None = None
@@ -526,6 +547,7 @@ class LiteLLMParamsBody(BaseModel):
     use_in_pass_through: bool | None = None
     complexity_router_config: dict[str, object] | None = None
     mock_response: str | None = None
+    timeout: float | None = None
 
 
 ModelMode = Literal["batch", "realtime", "image_generation"]

tests/e2e/router/conftest.py

@@ -79,8 +79,8 @@ def _router_is_callable(proxy: ProxyClient) -> bool:
     return isinstance(result, Success)
 
 
-@pytest.fixture(scope="session", autouse=True)
-def _ensure_complexity_smart_router(  # pyright: ignore[reportUnusedFunction]  # pytest autouse session fixture, wired by name
+@pytest.fixture(scope="session")
+def _ensure_complexity_smart_router(  # pyright: ignore[reportUnusedFunction]  # requested by the complexity test via usefixtures, wired by name
     client: ComplexityRouterClient,
 ) -> Iterator[None]:
     """Ensure the complexity router virtual model exists for this session.

tests/e2e/router/reliability_support.py

@@ -0,0 +1,77 @@
+"""Shared helpers for the reliability e2e tests (fallbacks, timeouts, cache).
+
+These are plain functions over the router suite's shared ProxyClient, not a
+fixture/client class: the tests reuse the router `client` fixture and pass
+`client.proxy`. Fallbacks and timeouts are driven by REAL deployments that all
+point at the real `openai/gpt-5.5`; a bad base URL yields a real connection
+error and a 1ms deadline yields a real timeout, and each test wires the
+reroute per request through a `router_settings_override` in the /chat/completions
+body, so a single long-lived proxy serves every reliability behavior.
+"""
+
+from __future__ import annotations
+
+from pydantic import ValidationError
+
+from proxy_client import ProxyClient
+from e2e_http import StreamingResponse
+from models import (
+    ChatMessage,
+    ChatResponse,
+    LiteLLMParamsBody,
+    ReliabilityChatBody,
+    RouterSettingsOverride,
+)
+
+REAL_MODEL = "openai/gpt-5.5"
+REAL_KEY = "os.environ/OPENAI_API_KEY"
+
+
+def create_bad_base_deployment(proxy: ProxyClient, name: str) -> str:
+    """Register a deployment pointing at an unreachable base, so every call to it
+    fails with a real connection error the fallback can reroute around."""
+    return proxy.create_model(
+        name, LiteLLMParamsBody(model=REAL_MODEL, api_key=REAL_KEY, api_base="http://127.0.0.1:9/v1")
+    )
+
+
+def create_timeout_deployment(proxy: ProxyClient, name: str) -> str:
+    """Register a deployment with a 1ms deadline the real backend always exceeds."""
+    return proxy.create_model(name, LiteLLMParamsBody(model=REAL_MODEL, api_key=REAL_KEY, timeout=0.001))
+
+
+def chat_override(
+    proxy: ProxyClient,
+    key: str,
+    model: str,
+    content: str,
+    override: RouterSettingsOverride | None = None,
+    stream: bool = False,
+) -> StreamingResponse:
+    """POST /chat/completions with an optional per-request router_settings_override,
+    returning the raw outcome so tests read status, body, and reliability headers."""
+    return proxy.transport.send(
+        "/chat/completions",
+        headers=proxy.transport.bearer(key),
+        json=ReliabilityChatBody(
+            model=model,
+            messages=[ChatMessage(role="user", content=content)],
+            max_tokens=16,
+            stream=stream,
+            router_settings_override=override,
+        ),
+        stream=stream,
+    )
+
+
+def content_of(resp: StreamingResponse) -> str | None:
+    """The assistant message content of a successful chat response, or None when the
+    body is not a success shape (an error body, or an elided streamed body)."""
+    try:
+        parsed = ChatResponse.model_validate_json(resp.body)
+    except ValidationError:
+        return None
+    if not parsed.choices:
+        return None
+    message = parsed.choices[0].message
+    return message.content if message is not None else None

tests/e2e/router/test_complexity_router_e2e.py

@@ -38,6 +38,7 @@ HEURISTIC_TIER_MODELS = frozenset({"openai/gpt-5.5", "gpt-5.5"})
 LLM_TIER_MODELS = frozenset({"anthropic/claude-haiku-4-5", "claude-haiku-4-5"})
 
 
+@pytest.mark.usefixtures("_ensure_complexity_smart_router")
 class TestComplexityRouterLlmClassifier:
     @pytest.mark.skip(
         reason="product bug LIT-4521: LLM classifier returns SIMPLE for short hard prompts "

tests/e2e/router/test_reliability_cache_e2e.py

@@ -0,0 +1,37 @@
+"""Live e2e: the response cache returns a cached answer on an exact repeat.
+
+The same unique prompt is sent twice to the real `gpt-5.5` deployment under the
+same key: the first call is a cache miss (the proxy computes and stores the entry,
+and returns no x-litellm-cache-key), the second is an exact hit (the proxy serves
+from cache and returns x-litellm-cache-key). This relies on the standard Redis
+response cache being enabled on the proxy under test.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from complexity_router_client import ComplexityRouterClient
+from e2e_config import unique_marker
+from reliability_support import chat_override
+
+pytestmark = pytest.mark.e2e
+
+
+class TestReliabilityCache:
+    @pytest.mark.covers("reliability.cache.exact.returns_cached")
+    def test_exact_cache_returns_cached(self, client: ComplexityRouterClient, scoped_key: str) -> None:
+        prompt = f"cache probe {unique_marker()}"
+
+        first = chat_override(client.proxy, scoped_key, "gpt-5.5", prompt)
+        assert first.status_code == 200, f"first call should succeed, got {first.status_code}: {first.body[:300]}"
+        assert "x-litellm-cache-key" not in first.headers, (
+            "first (uncached) call must not report a cache-key header"
+        )
+
+        second = chat_override(client.proxy, scoped_key, "gpt-5.5", prompt)
+        assert second.status_code == 200, f"second call should succeed, got {second.status_code}: {second.body[:300]}"
+        assert "x-litellm-cache-key" in second.headers, (
+            "second identical call should hit the response cache and report a cache-key header "
+            "(requires the proxy's Redis response cache to be enabled)"
+        )

tests/e2e/router/test_reliability_fallbacks_e2e.py

@@ -0,0 +1,69 @@
+"""Live e2e: per-request fallbacks reroute a failing deployment's traffic to a
+healthy one.
+
+Each test registers a primary deployment that fails (an unreachable base URL, or
+a 1ms deadline) and calls it with a `router_settings_override` mapping it to the
+real `gpt-5.5`. The proof the fallback fired is twofold: the response is a real
+completion from `gpt-5.5` (a non-empty content string), and the proxy reports at
+least one attempted fallback in the x-litellm-attempted-fallbacks header.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from complexity_router_client import ComplexityRouterClient
+from e2e_config import unique_marker
+from e2e_http import StreamingResponse
+from lifecycle import ResourceManager
+from models import RouterSettingsOverride
+from reliability_support import (
+    chat_override,
+    content_of,
+    create_bad_base_deployment,
+    create_timeout_deployment,
+)
+
+pytestmark = pytest.mark.e2e
+
+
+def _assert_served_by_fallback(resp: StreamingResponse) -> None:
+    assert resp.status_code == 200, f"expected 200 after fallback, got {resp.status_code}: {resp.body[:300]}"
+    content = content_of(resp)
+    assert isinstance(content, str) and content, (
+        f"the gpt-5.5 fallback should have returned a real completion, got content {content!r} "
+        f"(body={resp.body[:300]})"
+    )
+    attempted = resp.headers.get("x-litellm-attempted-fallbacks")
+    assert attempted is not None, "response is missing the x-litellm-attempted-fallbacks header"
+    assert int(attempted) >= 1, f"x-litellm-attempted-fallbacks should be >= 1, got {attempted!r}"
+
+
+class TestReliabilityFallbacks:
+    @pytest.mark.covers("reliability.fallback.5xx.routes_to_fallback")
+    def test_5xx_routes_to_fallback(
+        self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str
+    ) -> None:
+        primary = f"reliability-fail-{unique_marker()}"
+        model_id = create_bad_base_deployment(client.proxy, primary)
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+
+        resp = chat_override(
+            client.proxy, scoped_key, primary, "say hi",
+            override=RouterSettingsOverride(fallbacks=[{primary: ["gpt-5.5"]}]),
+        )
+        _assert_served_by_fallback(resp)
+
+    @pytest.mark.covers("reliability.fallback.timeout.routes_to_fallback")
+    def test_timeout_routes_to_fallback(
+        self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str
+    ) -> None:
+        primary = f"reliability-tofail-{unique_marker()}"
+        model_id = create_timeout_deployment(client.proxy, primary)
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+
+        resp = chat_override(
+            client.proxy, scoped_key, primary, "say hi",
+            override=RouterSettingsOverride(fallbacks=[{primary: ["gpt-5.5"]}]),
+        )
+        _assert_served_by_fallback(resp)

tests/e2e/router/test_reliability_timeouts_e2e.py

@@ -0,0 +1,53 @@
+"""Live e2e: a per-request timeout surfaces to the caller instead of hanging.
+
+A deployment created with a 1ms deadline always exceeds it against the real
+backend. With no fallback in play, the proxy must return the timeout to the
+caller: a 408 for a non-streamed request, and the same timeout surfaced on the
+streamed path (either a 408 before the stream opens or a timeout error carried in
+the response).
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from complexity_router_client import ComplexityRouterClient
+from e2e_config import unique_marker
+from lifecycle import ResourceManager
+from reliability_support import chat_override, create_timeout_deployment
+
+pytestmark = pytest.mark.e2e
+
+
+class TestReliabilityTimeouts:
+    @pytest.mark.covers("reliability.timeout.request_timeout.exceeds_deadline")
+    def test_request_timeout_exceeds_deadline(
+        self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str
+    ) -> None:
+        name = f"reliability-timeout-{unique_marker()}"
+        model_id = create_timeout_deployment(client.proxy, name)
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+
+        resp = chat_override(client.proxy, scoped_key, name, "hello")
+        assert resp.status_code == 408, (
+            f"a timed-out request should return 408, got {resp.status_code}: {resp.body[:300]}"
+        )
+        assert "timeout" in resp.body.lower(), f"the 408 body should name the timeout, got: {resp.body[:300]}"
+
+    @pytest.mark.covers("reliability.timeout.stream_timeout.exceeds_deadline")
+    def test_stream_timeout_exceeds_deadline(
+        self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str
+    ) -> None:
+        name = f"reliability-stream-timeout-{unique_marker()}"
+        model_id = create_timeout_deployment(client.proxy, name)
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+
+        resp = chat_override(client.proxy, scoped_key, name, "hello", stream=True)
+        surfaced = f"{resp.body} {resp.stream_error or ''}".lower()
+        assert resp.status_code >= 400, (
+            f"a timed-out streaming request should surface an error status, got {resp.status_code}: {resp.body[:300]}"
+        )
+        assert "timeout" in surfaced, (
+            f"the streamed timeout error should name the timeout, got body={resp.body[:300]}, "
+            f"stream_error={resp.stream_error!r}"
+        )