mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge f5a86e4e41 into e16d79d635
This commit is contained in:
commit
21e1d332fb
2 changed files with 107 additions and 1 deletions
|
|
@ -3791,6 +3791,26 @@ class Router:
|
|||
return completed.response.usage
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_partial_responses_usage_best_effort(
|
||||
source_iterator: "BaseResponsesAPIStreamingIterator",
|
||||
) -> Optional["ResponseAPIUsage"]:
|
||||
"""
|
||||
Wrap _extract_partial_responses_usage so an unexpected stream shape
|
||||
cannot abort a fallback that has already qualified. On failure the
|
||||
primary's usage is treated as unknown (None), so the fallback's usage
|
||||
is passed through unchanged instead of being merged with a made-up zero.
|
||||
"""
|
||||
try:
|
||||
return Router._extract_partial_responses_usage(source_iterator)
|
||||
except Exception as exc: # noqa: BLE001 # usage recovery is accounting only, never a reason to skip the fallback
|
||||
verbose_router_logger.warning(
|
||||
"aresponses fallback: could not extract partial usage from the failed stream, "
|
||||
"continuing without it: %s",
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _combine_responses_fallback_usage(
|
||||
fallback_item: "BaseLiteLLMOpenAIResponseObject",
|
||||
|
|
@ -4215,7 +4235,7 @@ class Router:
|
|||
"""
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
|
||||
partial_usage: Final = Router._extract_partial_responses_usage(source_iterator)
|
||||
partial_usage: Final = Router._extract_partial_responses_usage_best_effort(source_iterator)
|
||||
fallback_response = None # rebind-ok: pre-init so finally can close it if a fallback was actually attempted
|
||||
fallback_yielded = False # rebind-ok: flipped on the first fallback item so a fallback that dies before its first event still replays the primary's held announcement
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -7155,6 +7155,92 @@ async def test_aresponses_streaming_iterator_combines_partial_usage():
|
|||
assert merged.total_tokens == 49
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_streaming_iterator_falls_back_when_partial_usage_extraction_raises():
|
||||
"""If reading the failed primary's partial usage raises, the already eligible
|
||||
fallback still runs and its usage is passed through untouched (primary usage is
|
||||
unknown, not zero)."""
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
from litellm.types.llms.openai import (
|
||||
ResponseAPIUsage,
|
||||
ResponseCompletedEvent,
|
||||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamEvents,
|
||||
)
|
||||
|
||||
router: Final = _make_router_with_fallback()
|
||||
src: Final = _make_responses_iterator(
|
||||
chunks=[MagicMock(type="response.output_text.delta")],
|
||||
error=MidStreamFallbackError(
|
||||
message="boom",
|
||||
model="gpt-4",
|
||||
llm_provider="openai",
|
||||
is_pre_first_chunk=False,
|
||||
generated_content="hello",
|
||||
),
|
||||
)
|
||||
fallback_response_object: Final = ResponsesAPIResponse(
|
||||
id="resp_test", created_at=0, model="gpt-4", object="response", output=[]
|
||||
)
|
||||
fallback_response_object.usage = ResponseAPIUsage(input_tokens=20, output_tokens=15, total_tokens=35)
|
||||
fallback_event: Final = ResponseCompletedEvent(
|
||||
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
|
||||
response=fallback_response_object,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
Router,
|
||||
"_extract_partial_responses_usage",
|
||||
side_effect=AttributeError("'dict' object has no attribute 'usage'"),
|
||||
) as mock_extract,
|
||||
patch.object(
|
||||
router,
|
||||
"async_function_with_fallbacks_common_utils",
|
||||
return_value=_AsyncList([fallback_event]),
|
||||
) as mock_fallback_utils,
|
||||
):
|
||||
wrapped: Final = await router._aresponses_streaming_iterator(
|
||||
response=src,
|
||||
initial_kwargs={
|
||||
"model": "gpt-4",
|
||||
"stream": True,
|
||||
"input": "hi",
|
||||
"original_generic_function": litellm.aresponses,
|
||||
},
|
||||
)
|
||||
collected: Final = [chunk async for chunk in wrapped]
|
||||
|
||||
mock_extract.assert_called_once()
|
||||
mock_fallback_utils.assert_awaited_once()
|
||||
assert collected[-1] is fallback_event
|
||||
usage: Final = fallback_response_object.usage
|
||||
assert isinstance(usage, ResponseAPIUsage)
|
||||
assert (usage.input_tokens, usage.output_tokens, usage.total_tokens) == (20, 15, 35)
|
||||
|
||||
|
||||
def test_extract_partial_responses_usage_best_effort_returns_none_on_error():
|
||||
"""The best-effort wrapper swallows extraction errors and reports unknown usage."""
|
||||
src: Final = _make_responses_iterator()
|
||||
with patch.object(
|
||||
Router,
|
||||
"_extract_partial_responses_usage",
|
||||
side_effect=AttributeError("unexpected stream shape"),
|
||||
) as mock_extract:
|
||||
assert Router._extract_partial_responses_usage_best_effort(src) is None
|
||||
mock_extract.assert_called_once_with(src)
|
||||
|
||||
|
||||
def test_extract_partial_responses_usage_best_effort_passes_usage_through():
|
||||
"""When extraction succeeds, the wrapper returns the extracted usage unchanged."""
|
||||
from litellm.types.llms.openai import ResponseAPIUsage
|
||||
|
||||
src: Final = _make_responses_iterator()
|
||||
usage: Final = ResponseAPIUsage(input_tokens=3, output_tokens=4, total_tokens=7)
|
||||
with patch.object(Router, "_extract_partial_responses_usage", return_value=usage):
|
||||
assert Router._extract_partial_responses_usage_best_effort(src) is usage
|
||||
|
||||
|
||||
def _midstream_rate_limit_error():
|
||||
rate_limit_error = litellm.RateLimitError(
|
||||
message="vertex_ai_betaException - Resource exhausted.",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue