diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 56b6a7a7055..6d9ccad9a24 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -333,6 +333,7 @@ class OutMessage(BaseModel): class ChatChoice(BaseModel): message: OutMessage | None = None + finish_reason: str | None = None class PromptTokensDetails(BaseModel): diff --git a/tests/e2e/router/reliability_support.py b/tests/e2e/router/reliability_support.py index cc1c91c635b..e342aa363ca 100644 --- a/tests/e2e/router/reliability_support.py +++ b/tests/e2e/router/reliability_support.py @@ -57,7 +57,7 @@ def chat_override( json=ReliabilityChatBody( model=model, messages=[ChatMessage(role="user", content=content)], - max_tokens=64, + max_tokens=512, stream=stream, router_settings_override=override, cache=cache, @@ -66,14 +66,39 @@ def chat_override( ) +def _parsed(resp: StreamingResponse) -> ChatResponse | None: + try: + return ChatResponse.model_validate_json(resp.body) + except ValidationError: + return None + + def content_of(resp: StreamingResponse) -> str | None: """The assistant message content of a successful chat response, or None when the body is not a success shape (an error body, or an elided streamed body).""" - try: - parsed = ChatResponse.model_validate_json(resp.body) - except ValidationError: - return None - if not parsed.choices: + parsed = _parsed(resp) + if parsed is None or not parsed.choices: return None message = parsed.choices[0].message return message.content if message is not None else None + + +def finish_reason_of(resp: StreamingResponse) -> str | None: + parsed = _parsed(resp) + if parsed is None or not parsed.choices: + return None + return parsed.choices[0].finish_reason + + +def completion_tokens_of(resp: StreamingResponse) -> int | None: + parsed = _parsed(resp) + if parsed is None or parsed.usage is None: + return None + return parsed.usage.completion_tokens + + +def reasoning_tokens_of(resp: StreamingResponse) -> int | None: + parsed = _parsed(resp) + if parsed is None or parsed.usage is None or parsed.usage.completion_tokens_details is None: + return None + return parsed.usage.completion_tokens_details.reasoning_tokens diff --git a/tests/e2e/router/test_reliability_fallbacks_e2e.py b/tests/e2e/router/test_reliability_fallbacks_e2e.py index fe2d924ae2c..d3ce62f8f95 100644 --- a/tests/e2e/router/test_reliability_fallbacks_e2e.py +++ b/tests/e2e/router/test_reliability_fallbacks_e2e.py @@ -3,9 +3,12 @@ healthy one. Each test registers a primary deployment that fails (an unreachable base URL, or a 1ms deadline) and calls it with a `router_settings_override` mapping it to the -real `gpt-5.5`. The proof the fallback fired is twofold: the response is a real -completion from `gpt-5.5` (a non-empty content string), and the proxy reports at -least one attempted fallback in the x-litellm-attempted-fallbacks header. +real `gpt-5.5`. The proof the fallback fired is twofold: the response is a +completion from `gpt-5.5`, and the proxy reports at least one attempted fallback +in the x-litellm-attempted-fallbacks header. Empty content is accepted only when +`finish_reason == "length"` and the response billed completion tokens, since +gpt-5.5 counts reasoning against max_tokens and can consume the whole budget +before emitting any text; a fallback that produced nothing at all still fails. """ from __future__ import annotations @@ -19,9 +22,12 @@ from lifecycle import ResourceManager from models import RouterSettingsOverride from reliability_support import ( chat_override, + completion_tokens_of, content_of, create_bad_base_deployment, create_timeout_deployment, + finish_reason_of, + reasoning_tokens_of, ) pytestmark = pytest.mark.e2e @@ -30,8 +36,17 @@ pytestmark = pytest.mark.e2e def _assert_served_by_fallback(resp: StreamingResponse) -> None: assert resp.status_code == 200, f"expected 200 after fallback, got {resp.status_code}: {resp.body[:300]}" content = content_of(resp) - assert isinstance(content, str) and content, ( - f"the gpt-5.5 fallback should have returned a real completion, got content {content!r} " + finish_reason = finish_reason_of(resp) + completion_tokens = completion_tokens_of(resp) or 0 + reasoning_tokens = reasoning_tokens_of(resp) or 0 + assert isinstance(content, str), ( + f"the gpt-5.5 fallback should have returned a completion body, got content {content!r} " + f"(body={resp.body[:300]})" + ) + assert content or (finish_reason == "length" and completion_tokens > 0), ( + f"the gpt-5.5 fallback returned empty content with finish_reason={finish_reason!r}, " + f"completion_tokens={completion_tokens}, reasoning_tokens={reasoning_tokens}; empty " + f"content is only acceptable when the budget was spent on non-visible reasoning " f"(body={resp.body[:300]})" ) attempted = resp.headers.get("x-litellm-attempted-fallbacks")