Merge pull request #38833 from BerriAI/litellm_deflake_reliability_fallbacks

test(e2e): stop the reliability fallback tests flaking on gpt-5.5's reasoning budget

tests/e2e/models.py

@@ -333,6 +333,7 @@ class OutMessage(BaseModel):
 
 class ChatChoice(BaseModel):
     message: OutMessage | None = None
+    finish_reason: str | None = None
 
 
 class PromptTokensDetails(BaseModel):

tests/e2e/router/reliability_support.py

@@ -57,7 +57,7 @@ def chat_override(
         json=ReliabilityChatBody(
             model=model,
             messages=[ChatMessage(role="user", content=content)],
-            max_tokens=64,
+            max_tokens=512,
             stream=stream,
             router_settings_override=override,
             cache=cache,
@@ -66,14 +66,39 @@ def chat_override(
     )
 
 
+def _parsed(resp: StreamingResponse) -> ChatResponse | None:
+    try:
+        return ChatResponse.model_validate_json(resp.body)
+    except ValidationError:
+        return None
+
+
 def content_of(resp: StreamingResponse) -> str | None:
     """The assistant message content of a successful chat response, or None when the
     body is not a success shape (an error body, or an elided streamed body)."""
-    try:
-        parsed = ChatResponse.model_validate_json(resp.body)
-    except ValidationError:
-        return None
-    if not parsed.choices:
+    parsed = _parsed(resp)
+    if parsed is None or not parsed.choices:
         return None
     message = parsed.choices[0].message
     return message.content if message is not None else None
+
+
+def finish_reason_of(resp: StreamingResponse) -> str | None:
+    parsed = _parsed(resp)
+    if parsed is None or not parsed.choices:
+        return None
+    return parsed.choices[0].finish_reason
+
+
+def completion_tokens_of(resp: StreamingResponse) -> int | None:
+    parsed = _parsed(resp)
+    if parsed is None or parsed.usage is None:
+        return None
+    return parsed.usage.completion_tokens
+
+
+def reasoning_tokens_of(resp: StreamingResponse) -> int | None:
+    parsed = _parsed(resp)
+    if parsed is None or parsed.usage is None or parsed.usage.completion_tokens_details is None:
+        return None
+    return parsed.usage.completion_tokens_details.reasoning_tokens

tests/e2e/router/test_reliability_fallbacks_e2e.py

@@ -3,9 +3,12 @@ healthy one.
 
 Each test registers a primary deployment that fails (an unreachable base URL, or
 a 1ms deadline) and calls it with a `router_settings_override` mapping it to the
-real `gpt-5.5`. The proof the fallback fired is twofold: the response is a real
-completion from `gpt-5.5` (a non-empty content string), and the proxy reports at
-least one attempted fallback in the x-litellm-attempted-fallbacks header.
+real `gpt-5.5`. The proof the fallback fired is twofold: the response is a
+completion from `gpt-5.5`, and the proxy reports at least one attempted fallback
+in the x-litellm-attempted-fallbacks header. Empty content is accepted only when
+`finish_reason == "length"` and the response billed completion tokens, since
+gpt-5.5 counts reasoning against max_tokens and can consume the whole budget
+before emitting any text; a fallback that produced nothing at all still fails.
 """
 
 from __future__ import annotations
@@ -19,9 +22,12 @@ from lifecycle import ResourceManager
 from models import RouterSettingsOverride
 from reliability_support import (
     chat_override,
+    completion_tokens_of,
     content_of,
     create_bad_base_deployment,
     create_timeout_deployment,
+    finish_reason_of,
+    reasoning_tokens_of,
 )
 
 pytestmark = pytest.mark.e2e
@@ -30,8 +36,17 @@ pytestmark = pytest.mark.e2e
 def _assert_served_by_fallback(resp: StreamingResponse) -> None:
     assert resp.status_code == 200, f"expected 200 after fallback, got {resp.status_code}: {resp.body[:300]}"
     content = content_of(resp)
-    assert isinstance(content, str) and content, (
-        f"the gpt-5.5 fallback should have returned a real completion, got content {content!r} "
+    finish_reason = finish_reason_of(resp)
+    completion_tokens = completion_tokens_of(resp) or 0
+    reasoning_tokens = reasoning_tokens_of(resp) or 0
+    assert isinstance(content, str), (
+        f"the gpt-5.5 fallback should have returned a completion body, got content {content!r} "
+        f"(body={resp.body[:300]})"
+    )
+    assert content or (finish_reason == "length" and completion_tokens > 0), (
+        f"the gpt-5.5 fallback returned empty content with finish_reason={finish_reason!r}, "
+        f"completion_tokens={completion_tokens}, reasoning_tokens={reasoning_tokens}; empty "
+        f"content is only acceptable when the budget was spent on non-visible reasoning "
         f"(body={resp.body[:300]})"
     )
     attempted = resp.headers.get("x-litellm-attempted-fallbacks")