diff --git a/tests/e2e/models.py b/tests/e2e/models.py index e07814f2568..22aedb1cfbe 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -166,6 +166,27 @@ class ChatBody(BaseModel): guardrails: list[str] | None = None +class RouterSettingsOverride(BaseModel): + """Per-request `router_settings_override` in a /chat/completions body: the + reliability knobs (fallbacks by trigger, retry count) the reliability suite + drives per call instead of via static router config. Serialized exclude_none, so + an override sets only the strategies a test exercises. Each fallbacks map is + model_name -> the ordered fallback model_names to try.""" + + fallbacks: list[dict[str, list[str]]] | None = None + context_window_fallbacks: list[dict[str, list[str]]] | None = None + content_policy_fallbacks: list[dict[str, list[str]]] | None = None + num_retries: int | None = None + + +class ReliabilityChatBody(ChatBody): + """A /chat/completions body carrying a per-request router_settings_override. + Composes ChatBody (no attribute repetition) and adds the override; serialized + exclude_none so an absent override never leaks into the request.""" + + router_settings_override: RouterSettingsOverride | None = None + + class OutMessage(BaseModel): content: str | None = None reasoning_content: str | None = None @@ -526,6 +547,7 @@ class LiteLLMParamsBody(BaseModel): use_in_pass_through: bool | None = None complexity_router_config: dict[str, object] | None = None mock_response: str | None = None + timeout: float | None = None ModelMode = Literal["batch", "realtime", "image_generation"] diff --git a/tests/e2e/router/conftest.py b/tests/e2e/router/conftest.py index 8ddc19aa94f..98501f9bd7c 100644 --- a/tests/e2e/router/conftest.py +++ b/tests/e2e/router/conftest.py @@ -79,8 +79,8 @@ def _router_is_callable(proxy: ProxyClient) -> bool: return isinstance(result, Success) -@pytest.fixture(scope="session", autouse=True) -def _ensure_complexity_smart_router( # pyright: ignore[reportUnusedFunction] # pytest autouse session fixture, wired by name +@pytest.fixture(scope="session") +def _ensure_complexity_smart_router( # pyright: ignore[reportUnusedFunction] # requested by the complexity test via usefixtures, wired by name client: ComplexityRouterClient, ) -> Iterator[None]: """Ensure the complexity router virtual model exists for this session. diff --git a/tests/e2e/router/reliability_support.py b/tests/e2e/router/reliability_support.py new file mode 100644 index 00000000000..4dab0aaa3fa --- /dev/null +++ b/tests/e2e/router/reliability_support.py @@ -0,0 +1,77 @@ +"""Shared helpers for the reliability e2e tests (fallbacks, timeouts, cache). + +These are plain functions over the router suite's shared ProxyClient, not a +fixture/client class: the tests reuse the router `client` fixture and pass +`client.proxy`. Fallbacks and timeouts are driven by REAL deployments that all +point at the real `openai/gpt-5.5`; a bad base URL yields a real connection +error and a 1ms deadline yields a real timeout, and each test wires the +reroute per request through a `router_settings_override` in the /chat/completions +body, so a single long-lived proxy serves every reliability behavior. +""" + +from __future__ import annotations + +from pydantic import ValidationError + +from proxy_client import ProxyClient +from e2e_http import StreamingResponse +from models import ( + ChatMessage, + ChatResponse, + LiteLLMParamsBody, + ReliabilityChatBody, + RouterSettingsOverride, +) + +REAL_MODEL = "openai/gpt-5.5" +REAL_KEY = "os.environ/OPENAI_API_KEY" + + +def create_bad_base_deployment(proxy: ProxyClient, name: str) -> str: + """Register a deployment pointing at an unreachable base, so every call to it + fails with a real connection error the fallback can reroute around.""" + return proxy.create_model( + name, LiteLLMParamsBody(model=REAL_MODEL, api_key=REAL_KEY, api_base="http://127.0.0.1:9/v1") + ) + + +def create_timeout_deployment(proxy: ProxyClient, name: str) -> str: + """Register a deployment with a 1ms deadline the real backend always exceeds.""" + return proxy.create_model(name, LiteLLMParamsBody(model=REAL_MODEL, api_key=REAL_KEY, timeout=0.001)) + + +def chat_override( + proxy: ProxyClient, + key: str, + model: str, + content: str, + override: RouterSettingsOverride | None = None, + stream: bool = False, +) -> StreamingResponse: + """POST /chat/completions with an optional per-request router_settings_override, + returning the raw outcome so tests read status, body, and reliability headers.""" + return proxy.transport.send( + "/chat/completions", + headers=proxy.transport.bearer(key), + json=ReliabilityChatBody( + model=model, + messages=[ChatMessage(role="user", content=content)], + max_tokens=16, + stream=stream, + router_settings_override=override, + ), + stream=stream, + ) + + +def content_of(resp: StreamingResponse) -> str | None: + """The assistant message content of a successful chat response, or None when the + body is not a success shape (an error body, or an elided streamed body).""" + try: + parsed = ChatResponse.model_validate_json(resp.body) + except ValidationError: + return None + if not parsed.choices: + return None + message = parsed.choices[0].message + return message.content if message is not None else None diff --git a/tests/e2e/router/test_complexity_router_e2e.py b/tests/e2e/router/test_complexity_router_e2e.py index a495e2fdf4d..e8508c963b8 100644 --- a/tests/e2e/router/test_complexity_router_e2e.py +++ b/tests/e2e/router/test_complexity_router_e2e.py @@ -38,6 +38,7 @@ HEURISTIC_TIER_MODELS = frozenset({"openai/gpt-5.5", "gpt-5.5"}) LLM_TIER_MODELS = frozenset({"anthropic/claude-haiku-4-5", "claude-haiku-4-5"}) +@pytest.mark.usefixtures("_ensure_complexity_smart_router") class TestComplexityRouterLlmClassifier: @pytest.mark.skip( reason="product bug LIT-4521: LLM classifier returns SIMPLE for short hard prompts " diff --git a/tests/e2e/router/test_reliability_cache_e2e.py b/tests/e2e/router/test_reliability_cache_e2e.py new file mode 100644 index 00000000000..78d8fcdc08f --- /dev/null +++ b/tests/e2e/router/test_reliability_cache_e2e.py @@ -0,0 +1,37 @@ +"""Live e2e: the response cache returns a cached answer on an exact repeat. + +The same unique prompt is sent twice to the real `gpt-5.5` deployment under the +same key: the first call is a cache miss (the proxy computes and stores the entry, +and returns no x-litellm-cache-key), the second is an exact hit (the proxy serves +from cache and returns x-litellm-cache-key). This relies on the standard Redis +response cache being enabled on the proxy under test. +""" + +from __future__ import annotations + +import pytest + +from complexity_router_client import ComplexityRouterClient +from e2e_config import unique_marker +from reliability_support import chat_override + +pytestmark = pytest.mark.e2e + + +class TestReliabilityCache: + @pytest.mark.covers("reliability.cache.exact.returns_cached") + def test_exact_cache_returns_cached(self, client: ComplexityRouterClient, scoped_key: str) -> None: + prompt = f"cache probe {unique_marker()}" + + first = chat_override(client.proxy, scoped_key, "gpt-5.5", prompt) + assert first.status_code == 200, f"first call should succeed, got {first.status_code}: {first.body[:300]}" + assert "x-litellm-cache-key" not in first.headers, ( + "first (uncached) call must not report a cache-key header" + ) + + second = chat_override(client.proxy, scoped_key, "gpt-5.5", prompt) + assert second.status_code == 200, f"second call should succeed, got {second.status_code}: {second.body[:300]}" + assert "x-litellm-cache-key" in second.headers, ( + "second identical call should hit the response cache and report a cache-key header " + "(requires the proxy's Redis response cache to be enabled)" + ) diff --git a/tests/e2e/router/test_reliability_fallbacks_e2e.py b/tests/e2e/router/test_reliability_fallbacks_e2e.py new file mode 100644 index 00000000000..5b7d21c6ef7 --- /dev/null +++ b/tests/e2e/router/test_reliability_fallbacks_e2e.py @@ -0,0 +1,69 @@ +"""Live e2e: per-request fallbacks reroute a failing deployment's traffic to a +healthy one. + +Each test registers a primary deployment that fails (an unreachable base URL, or +a 1ms deadline) and calls it with a `router_settings_override` mapping it to the +real `gpt-5.5`. The proof the fallback fired is twofold: the response is a real +completion from `gpt-5.5` (a non-empty content string), and the proxy reports at +least one attempted fallback in the x-litellm-attempted-fallbacks header. +""" + +from __future__ import annotations + +import pytest + +from complexity_router_client import ComplexityRouterClient +from e2e_config import unique_marker +from e2e_http import StreamingResponse +from lifecycle import ResourceManager +from models import RouterSettingsOverride +from reliability_support import ( + chat_override, + content_of, + create_bad_base_deployment, + create_timeout_deployment, +) + +pytestmark = pytest.mark.e2e + + +def _assert_served_by_fallback(resp: StreamingResponse) -> None: + assert resp.status_code == 200, f"expected 200 after fallback, got {resp.status_code}: {resp.body[:300]}" + content = content_of(resp) + assert isinstance(content, str) and content, ( + f"the gpt-5.5 fallback should have returned a real completion, got content {content!r} " + f"(body={resp.body[:300]})" + ) + attempted = resp.headers.get("x-litellm-attempted-fallbacks") + assert attempted is not None, "response is missing the x-litellm-attempted-fallbacks header" + assert int(attempted) >= 1, f"x-litellm-attempted-fallbacks should be >= 1, got {attempted!r}" + + +class TestReliabilityFallbacks: + @pytest.mark.covers("reliability.fallback.5xx.routes_to_fallback") + def test_5xx_routes_to_fallback( + self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str + ) -> None: + primary = f"reliability-fail-{unique_marker()}" + model_id = create_bad_base_deployment(client.proxy, primary) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + resp = chat_override( + client.proxy, scoped_key, primary, "say hi", + override=RouterSettingsOverride(fallbacks=[{primary: ["gpt-5.5"]}]), + ) + _assert_served_by_fallback(resp) + + @pytest.mark.covers("reliability.fallback.timeout.routes_to_fallback") + def test_timeout_routes_to_fallback( + self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str + ) -> None: + primary = f"reliability-tofail-{unique_marker()}" + model_id = create_timeout_deployment(client.proxy, primary) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + resp = chat_override( + client.proxy, scoped_key, primary, "say hi", + override=RouterSettingsOverride(fallbacks=[{primary: ["gpt-5.5"]}]), + ) + _assert_served_by_fallback(resp) diff --git a/tests/e2e/router/test_reliability_timeouts_e2e.py b/tests/e2e/router/test_reliability_timeouts_e2e.py new file mode 100644 index 00000000000..f24d5139e66 --- /dev/null +++ b/tests/e2e/router/test_reliability_timeouts_e2e.py @@ -0,0 +1,53 @@ +"""Live e2e: a per-request timeout surfaces to the caller instead of hanging. + +A deployment created with a 1ms deadline always exceeds it against the real +backend. With no fallback in play, the proxy must return the timeout to the +caller: a 408 for a non-streamed request, and the same timeout surfaced on the +streamed path (either a 408 before the stream opens or a timeout error carried in +the response). +""" + +from __future__ import annotations + +import pytest + +from complexity_router_client import ComplexityRouterClient +from e2e_config import unique_marker +from lifecycle import ResourceManager +from reliability_support import chat_override, create_timeout_deployment + +pytestmark = pytest.mark.e2e + + +class TestReliabilityTimeouts: + @pytest.mark.covers("reliability.timeout.request_timeout.exceeds_deadline") + def test_request_timeout_exceeds_deadline( + self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"reliability-timeout-{unique_marker()}" + model_id = create_timeout_deployment(client.proxy, name) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + resp = chat_override(client.proxy, scoped_key, name, "hello") + assert resp.status_code == 408, ( + f"a timed-out request should return 408, got {resp.status_code}: {resp.body[:300]}" + ) + assert "timeout" in resp.body.lower(), f"the 408 body should name the timeout, got: {resp.body[:300]}" + + @pytest.mark.covers("reliability.timeout.stream_timeout.exceeds_deadline") + def test_stream_timeout_exceeds_deadline( + self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"reliability-stream-timeout-{unique_marker()}" + model_id = create_timeout_deployment(client.proxy, name) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + resp = chat_override(client.proxy, scoped_key, name, "hello", stream=True) + surfaced = f"{resp.body} {resp.stream_error or ''}".lower() + assert resp.status_code >= 400, ( + f"a timed-out streaming request should surface an error status, got {resp.status_code}: {resp.body[:300]}" + ) + assert "timeout" in surfaced, ( + f"the streamed timeout error should name the timeout, got body={resp.body[:300]}, " + f"stream_error={resp.stream_error!r}" + )