diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 967144dfb16..79d9e011f7e 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -806,6 +806,7 @@ class LiteLLMParamsBody(BaseModel): mock_response: str | None = None timeout: float | None = None tpm: int | None = None + weight: int | None = None ModelMode = Literal["batch", "realtime", "image_generation"] @@ -820,6 +821,7 @@ class ModelInfoBody(BaseModel): mode: ModelMode | None = None access_groups: list[str] | None = None team_id: str | None = None + allowed_fails_policy: dict[str, int] | None = None class ModelNewBody(BaseModel): diff --git a/tests/e2e/router/reliability_support.py b/tests/e2e/router/reliability_support.py index e342aa363ca..5822058003c 100644 --- a/tests/e2e/router/reliability_support.py +++ b/tests/e2e/router/reliability_support.py @@ -19,6 +19,8 @@ from models import ( ChatMessage, ChatResponse, LiteLLMParamsBody, + ModelInfoBody, + ModelNewBody, ReliabilityChatBody, RouterSettingsOverride, ) @@ -26,6 +28,18 @@ from models import ( REAL_MODEL = "openai/gpt-5.5" REAL_KEY = "os.environ/OPENAI_API_KEY" +# The smallest-context chat model OpenAI still serves (16385 tokens). A prompt +# past that limit comes back as a real `context_length_exceeded` 400, which is +# what litellm maps to ContextWindowExceededError. +SMALL_CONTEXT_MODEL = "openai/gpt-3.5-turbo" +SMALL_CONTEXT_LIMIT_TOKENS = 16385 + + +def oversized_prompt(marker: str) -> str: + """A prompt comfortably past SMALL_CONTEXT_MODEL's context limit, so the + provider refuses it on length rather than answering a truncated version.""" + return f"{marker} " + ("token " * (SMALL_CONTEXT_LIMIT_TOKENS + 4000)) + def create_bad_base_deployment(proxy: ProxyClient, name: str) -> str: """Register a deployment pointing at an unreachable base, so every call to it @@ -40,6 +54,38 @@ def create_timeout_deployment(proxy: ProxyClient, name: str) -> str: return proxy.create_model(name, LiteLLMParamsBody(model=REAL_MODEL, api_key=REAL_KEY, timeout=0.001)) +def create_small_context_deployment(proxy: ProxyClient, name: str) -> str: + """Register a deployment on the smallest-context model OpenAI still serves, so an + oversized prompt earns a real context-window refusal from the provider.""" + return proxy.create_model(name, LiteLLMParamsBody(model=SMALL_CONTEXT_MODEL, api_key=REAL_KEY)) + + +def create_always_timing_out_deployment(proxy: ProxyClient, name: str) -> str: + """The always-picked half of a retry pair: a 1ms deadline the backend always + exceeds, all of the model group's shuffle weight, and a cooldown policy that + benches it on its first Timeout so the retry cannot land on it again.""" + return proxy.register_model( + ModelNewBody( + model_name=name, + litellm_params=LiteLLMParamsBody(model=REAL_MODEL, api_key=REAL_KEY, timeout=0.001, weight=1), + model_info=ModelInfoBody(allowed_fails_policy={"TimeoutErrorAllowedFails": 0}), + ) + ) + + +def create_zero_weight_backup_deployment(proxy: ProxyClient, name: str) -> str: + """The other half of a retry pair: healthy, but weight 0, so the weighted shuffle + never opens on it. It is reachable only once its sibling is benched and the + weighted pick falls through to a uniform one over what is left.""" + return proxy.register_model( + ModelNewBody( + model_name=name, + litellm_params=LiteLLMParamsBody(model=REAL_MODEL, api_key=REAL_KEY, weight=0), + model_info=ModelInfoBody(), + ) + ) + + def chat_override( proxy: ProxyClient, key: str, diff --git a/tests/e2e/router/test_reliability_fallbacks_e2e.py b/tests/e2e/router/test_reliability_fallbacks_e2e.py index d3ce62f8f95..8cece41ce2d 100644 --- a/tests/e2e/router/test_reliability_fallbacks_e2e.py +++ b/tests/e2e/router/test_reliability_fallbacks_e2e.py @@ -9,6 +9,10 @@ in the x-litellm-attempted-fallbacks header. Empty content is accepted only when `finish_reason == "length"` and the response billed completion tokens, since gpt-5.5 counts reasoning against max_tokens and can consume the whole budget before emitting any text; a fallback that produced nothing at all still fails. + +The context-window case is a different reroute from a plain failure: the provider +refuses the prompt on length, and `context_window_fallbacks` is the setting that +reroutes it, not `fallbacks`. """ from __future__ import annotations @@ -25,8 +29,10 @@ from reliability_support import ( completion_tokens_of, content_of, create_bad_base_deployment, + create_small_context_deployment, create_timeout_deployment, finish_reason_of, + oversized_prompt, reasoning_tokens_of, ) @@ -82,3 +88,17 @@ class TestReliabilityFallbacks: override=RouterSettingsOverride(fallbacks=[{primary: ["gpt-5.5"]}]), ) _assert_served_by_fallback(resp) + + @pytest.mark.covers("reliability.fallback.context_window.routes_to_fallback") + def test_context_window_routes_to_fallback( + self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str + ) -> None: + primary = f"reliability-ctxfail-{unique_marker()}" + model_id = create_small_context_deployment(client.proxy, primary) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + resp = chat_override( + client.proxy, scoped_key, primary, oversized_prompt(unique_marker()), + override=RouterSettingsOverride(context_window_fallbacks=[{primary: ["gpt-5.5"]}]), + ) + _assert_served_by_fallback(resp) diff --git a/tests/e2e/router/test_reliability_retries_e2e.py b/tests/e2e/router/test_reliability_retries_e2e.py new file mode 100644 index 00000000000..5441412935c --- /dev/null +++ b/tests/e2e/router/test_reliability_retries_e2e.py @@ -0,0 +1,73 @@ +"""Live e2e: a request that fails on its first deployment is retried inside its own +model group and still comes back a completion. + +The model group is a pair: an always-timing-out deployment that holds all of the +group's shuffle weight, and a healthy backup at weight 0. The weighted pick always +opens on the timing-out one, its first Timeout benches it (an +`allowed_fails_policy` of `TimeoutErrorAllowedFails: 0`), and the retry falls +through to the only deployment left. So the customer sees a completion and the +proxy reports that it took a retry to get there, with no random first pick in the +middle of it. +""" + +from __future__ import annotations + +import pytest + +from complexity_router_client import ComplexityRouterClient +from e2e_config import unique_marker +from lifecycle import ResourceManager +from models import RouterSettingsOverride +from reliability_support import ( + chat_override, + completion_tokens_of, + content_of, + create_always_timing_out_deployment, + create_zero_weight_backup_deployment, + finish_reason_of, +) + +pytestmark = pytest.mark.e2e + + +class TestReliabilityRetries: + @pytest.mark.covers("reliability.retry.timeout.succeeds_within_retries") + def test_timeout_on_first_deployment_succeeds_on_retry( + self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str + ) -> None: + group = f"reliability-retry-{unique_marker()}" + timing_out = create_always_timing_out_deployment(client.proxy, group) + resources.defer(lambda: client.proxy.delete_model(timing_out)) + backup = create_zero_weight_backup_deployment(client.proxy, group) + resources.defer(lambda: client.proxy.delete_model(backup)) + + resp = chat_override( + client.proxy, + scoped_key, + group, + f"say hi {unique_marker()}", + override=RouterSettingsOverride(num_retries=2), + ) + + assert resp.status_code == 200, ( + f"the retry should have landed on the healthy backup, got {resp.status_code}: {resp.body[:300]}" + ) + + attempted = resp.headers.get("x-litellm-attempted-retries") + assert attempted is not None, "response is missing the x-litellm-attempted-retries header" + assert int(attempted) >= 1, ( + f"x-litellm-attempted-retries is {attempted!r}; a 200 with no retry means the request never " + "opened on the timing-out deployment, so this proves nothing about retries" + ) + + content = content_of(resp) + finish_reason = finish_reason_of(resp) + completion_tokens = completion_tokens_of(resp) or 0 + assert isinstance(content, str), ( + f"the retry should have returned a completion body, got content {content!r} (body={resp.body[:300]})" + ) + assert content or (finish_reason == "length" and completion_tokens > 0), ( + f"the retry returned empty content with finish_reason={finish_reason!r}, " + f"completion_tokens={completion_tokens}; empty content is only acceptable when the budget " + f"was spent on non-visible reasoning (body={resp.body[:300]})" + )