diff --git a/litellm/router.py b/litellm/router.py index 8d95e3e91d4..035c3a6ee8f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3791,6 +3791,26 @@ class Router: return completed.response.usage return None + @staticmethod + def _extract_partial_responses_usage_best_effort( + source_iterator: "BaseResponsesAPIStreamingIterator", + ) -> Optional["ResponseAPIUsage"]: + """ + Wrap _extract_partial_responses_usage so an unexpected stream shape + cannot abort a fallback that has already qualified. On failure the + primary's usage is treated as unknown (None), so the fallback's usage + is passed through unchanged instead of being merged with a made-up zero. + """ + try: + return Router._extract_partial_responses_usage(source_iterator) + except Exception as exc: # noqa: BLE001 # usage recovery is accounting only, never a reason to skip the fallback + verbose_router_logger.warning( + "aresponses fallback: could not extract partial usage from the failed stream, " + "continuing without it: %s", + exc, + ) + return None + @staticmethod def _combine_responses_fallback_usage( fallback_item: "BaseLiteLLMOpenAIResponseObject", @@ -4215,7 +4235,7 @@ class Router: """ from litellm.exceptions import MidStreamFallbackError - partial_usage: Final = Router._extract_partial_responses_usage(source_iterator) + partial_usage: Final = Router._extract_partial_responses_usage_best_effort(source_iterator) fallback_response = None # rebind-ok: pre-init so finally can close it if a fallback was actually attempted fallback_yielded = False # rebind-ok: flipped on the first fallback item so a fallback that dies before its first event still replays the primary's held announcement try: diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 3004f83b1df..a7b3b81964e 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -7155,6 +7155,92 @@ async def test_aresponses_streaming_iterator_combines_partial_usage(): assert merged.total_tokens == 49 +@pytest.mark.asyncio +async def test_aresponses_streaming_iterator_falls_back_when_partial_usage_extraction_raises(): + """If reading the failed primary's partial usage raises, the already eligible + fallback still runs and its usage is passed through untouched (primary usage is + unknown, not zero).""" + from litellm.exceptions import MidStreamFallbackError + from litellm.types.llms.openai import ( + ResponseAPIUsage, + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + router: Final = _make_router_with_fallback() + src: Final = _make_responses_iterator( + chunks=[MagicMock(type="response.output_text.delta")], + error=MidStreamFallbackError( + message="boom", + model="gpt-4", + llm_provider="openai", + is_pre_first_chunk=False, + generated_content="hello", + ), + ) + fallback_response_object: Final = ResponsesAPIResponse( + id="resp_test", created_at=0, model="gpt-4", object="response", output=[] + ) + fallback_response_object.usage = ResponseAPIUsage(input_tokens=20, output_tokens=15, total_tokens=35) + fallback_event: Final = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=fallback_response_object, + ) + + with ( + patch.object( + Router, + "_extract_partial_responses_usage", + side_effect=AttributeError("'dict' object has no attribute 'usage'"), + ) as mock_extract, + patch.object( + router, + "async_function_with_fallbacks_common_utils", + return_value=_AsyncList([fallback_event]), + ) as mock_fallback_utils, + ): + wrapped: Final = await router._aresponses_streaming_iterator( + response=src, + initial_kwargs={ + "model": "gpt-4", + "stream": True, + "input": "hi", + "original_generic_function": litellm.aresponses, + }, + ) + collected: Final = [chunk async for chunk in wrapped] + + mock_extract.assert_called_once() + mock_fallback_utils.assert_awaited_once() + assert collected[-1] is fallback_event + usage: Final = fallback_response_object.usage + assert isinstance(usage, ResponseAPIUsage) + assert (usage.input_tokens, usage.output_tokens, usage.total_tokens) == (20, 15, 35) + + +def test_extract_partial_responses_usage_best_effort_returns_none_on_error(): + """The best-effort wrapper swallows extraction errors and reports unknown usage.""" + src: Final = _make_responses_iterator() + with patch.object( + Router, + "_extract_partial_responses_usage", + side_effect=AttributeError("unexpected stream shape"), + ) as mock_extract: + assert Router._extract_partial_responses_usage_best_effort(src) is None + mock_extract.assert_called_once_with(src) + + +def test_extract_partial_responses_usage_best_effort_passes_usage_through(): + """When extraction succeeds, the wrapper returns the extracted usage unchanged.""" + from litellm.types.llms.openai import ResponseAPIUsage + + src: Final = _make_responses_iterator() + usage: Final = ResponseAPIUsage(input_tokens=3, output_tokens=4, total_tokens=7) + with patch.object(Router, "_extract_partial_responses_usage", return_value=usage): + assert Router._extract_partial_responses_usage_best_effort(src) is usage + + def _midstream_rate_limit_error(): rate_limit_error = litellm.RateLimitError( message="vertex_ai_betaException - Resource exhausted.",