This commit is contained in:
SteeveCrypto 2026-10-27 02:30:29 +00:00 • committed by GitHub
commit 21e1d332fb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 107 additions and 1 deletions

View file

@ -3791,6 +3791,26 @@ class Router:
return completed.response.usage
return None
@staticmethod
def _extract_partial_responses_usage_best_effort(
source_iterator: "BaseResponsesAPIStreamingIterator",
) -> Optional["ResponseAPIUsage"]:
"""
Wrap _extract_partial_responses_usage so an unexpected stream shape
cannot abort a fallback that has already qualified. On failure the
primary's usage is treated as unknown (None), so the fallback's usage
is passed through unchanged instead of being merged with a made-up zero.
"""
try:
return Router._extract_partial_responses_usage(source_iterator)
except Exception as exc: # noqa: BLE001 # usage recovery is accounting only, never a reason to skip the fallback
verbose_router_logger.warning(
"aresponses fallback: could not extract partial usage from the failed stream, "
"continuing without it: %s",
exc,
)
return None
@staticmethod
def _combine_responses_fallback_usage(
fallback_item: "BaseLiteLLMOpenAIResponseObject",
@ -4215,7 +4235,7 @@ class Router:
"""
from litellm.exceptions import MidStreamFallbackError
partial_usage: Final = Router._extract_partial_responses_usage(source_iterator)
partial_usage: Final = Router._extract_partial_responses_usage_best_effort(source_iterator)
fallback_response = None # rebind-ok: pre-init so finally can close it if a fallback was actually attempted
fallback_yielded = False # rebind-ok: flipped on the first fallback item so a fallback that dies before its first event still replays the primary's held announcement
try:

View file

@ -7155,6 +7155,92 @@ async def test_aresponses_streaming_iterator_combines_partial_usage():
assert merged.total_tokens == 49
@pytest.mark.asyncio
async def test_aresponses_streaming_iterator_falls_back_when_partial_usage_extraction_raises():
"""If reading the failed primary's partial usage raises, the already eligible
fallback still runs and its usage is passed through untouched (primary usage is
unknown, not zero)."""
from litellm.exceptions import MidStreamFallbackError
from litellm.types.llms.openai import (
ResponseAPIUsage,
ResponseCompletedEvent,
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
)
router: Final = _make_router_with_fallback()
src: Final = _make_responses_iterator(
chunks=[MagicMock(type="response.output_text.delta")],
error=MidStreamFallbackError(
message="boom",
model="gpt-4",
llm_provider="openai",
is_pre_first_chunk=False,
generated_content="hello",
),
)
fallback_response_object: Final = ResponsesAPIResponse(
id="resp_test", created_at=0, model="gpt-4", object="response", output=[]
)
fallback_response_object.usage = ResponseAPIUsage(input_tokens=20, output_tokens=15, total_tokens=35)
fallback_event: Final = ResponseCompletedEvent(
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
response=fallback_response_object,
)
with (
patch.object(
Router,
"_extract_partial_responses_usage",
side_effect=AttributeError("'dict' object has no attribute 'usage'"),
) as mock_extract,
patch.object(
router,
"async_function_with_fallbacks_common_utils",
return_value=_AsyncList([fallback_event]),
) as mock_fallback_utils,
):
wrapped: Final = await router._aresponses_streaming_iterator(
response=src,
initial_kwargs={
"model": "gpt-4",
"stream": True,
"input": "hi",
"original_generic_function": litellm.aresponses,
},
)
collected: Final = [chunk async for chunk in wrapped]
mock_extract.assert_called_once()
mock_fallback_utils.assert_awaited_once()
assert collected[-1] is fallback_event
usage: Final = fallback_response_object.usage
assert isinstance(usage, ResponseAPIUsage)
assert (usage.input_tokens, usage.output_tokens, usage.total_tokens) == (20, 15, 35)
def test_extract_partial_responses_usage_best_effort_returns_none_on_error():
"""The best-effort wrapper swallows extraction errors and reports unknown usage."""
src: Final = _make_responses_iterator()
with patch.object(
Router,
"_extract_partial_responses_usage",
side_effect=AttributeError("unexpected stream shape"),
) as mock_extract:
assert Router._extract_partial_responses_usage_best_effort(src) is None
mock_extract.assert_called_once_with(src)
def test_extract_partial_responses_usage_best_effort_passes_usage_through():
"""When extraction succeeds, the wrapper returns the extracted usage unchanged."""
from litellm.types.llms.openai import ResponseAPIUsage
src: Final = _make_responses_iterator()
usage: Final = ResponseAPIUsage(input_tokens=3, output_tokens=4, total_tokens=7)
with patch.object(Router, "_extract_partial_responses_usage", return_value=usage):
assert Router._extract_partial_responses_usage_best_effort(src) is usage
def _midstream_rate_limit_error():
rate_limit_error = litellm.RateLimitError(
message="vertex_ai_betaException - Resource exhausted.",