From 33cebf85e265f7623cca6b854df2db48008f2df3 Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Mon, 26 Oct 2026 20:46:30 -0700 Subject: [PATCH] fix(router): prevent Responses tool-call replay (#42955) --- litellm/router.py | 13 +- .../base_llm/responses/test_transformation.py | 117 ++++---------- tests/unit/test_router/test_router.py | 149 +++++++++++++----- 3 files changed, 148 insertions(+), 131 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 8d95e3e91d4..f95fc949594 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4006,9 +4006,8 @@ class Router: Full parity with the chat-completions path: - Pre-first-chunk: retry with the original input unchanged. - - Partial content: inject a developer instruction + prior - assistant message carrying the generated text so the fallback - model continues rather than restarts. + - Delivered output: surface the original error rather than + restarting an output item or tool call on a fallback. - Usage combining: merge partial-stream usage onto the fallback's response.completed event so accounting reflects both attempts. - Stream cleanup: shielded aclose() on both source and fallback @@ -4158,6 +4157,7 @@ class Router: async def stream_with_fallbacks(): held_lifecycle_events: tuple[object, ...] = () # rebind-ok: flushed at first output, dropped on fallback + has_forwarded_output = False # rebind-ok: updated as output events reach the client try: async for item in cast( # cast-ok: response iterators are async streams AsyncIterator[object], source_iterator @@ -4168,10 +4168,17 @@ class Router: for held_event in held_lifecycle_events: yield held_event held_lifecycle_events = () + if isinstance(getattr(item, "output_index", None), int) or ( + getattr(item, "type", None) + == _openai_types.ResponsesAPIStreamEvents.IMAGE_GENERATION_PARTIAL_IMAGE + ): + has_forwarded_output = True yield item for held_event in held_lifecycle_events: yield held_event except MidStreamFallbackError as e: + if has_forwarded_output: + raise e.original_exception or e async with contextlib.aclosing( self._aresponses_fallback_attempt( e, source_iterator, initial_kwargs, wrapper.adopt_fallback_headers, held_lifecycle_events diff --git a/tests/unit/llms/base_llm/responses/test_transformation.py b/tests/unit/llms/base_llm/responses/test_transformation.py index 8d21af83668..0c98b7c2590 100644 --- a/tests/unit/llms/base_llm/responses/test_transformation.py +++ b/tests/unit/llms/base_llm/responses/test_transformation.py @@ -12,7 +12,7 @@ from litellm import Router from litellm.constants import STREAM_SSE_DONE_STRING from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig -from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator +from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator, ResponsesAPIStreamingIterator from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import( OutputTextDeltaEvent, @@ -24,6 +24,7 @@ from litellm.types.llms.openai import( ResponsesAPIStreamEvents, ) from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome +from datetime import datetime from typing import Any, AsyncIterator, Final, List from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -1267,97 +1268,35 @@ async def test_aresponses_fallback_on_in_stream_error_event(): @pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") @pytest.mark.asyncio -async def test_aresponses_fallback_uses_continuation_input_after_partial_content(): - """When output text was already streamed before the error, the fallback re-entry - must carry a continuation input with the partial assistant text instead of - retrying the original input from scratch (which would duplicate streamed content).""" - import json - from unittest.mock import Mock - - from litellm.exceptions import MidStreamFallbackError - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig - from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator - from litellm.types.llms.openai import ErrorEvent, ErrorEventError - - router = _make_router() - - events = [ - {"type": "response.output_text.delta", "delta": "partial answer"}, - {"type": "error", "error": {"type": "server_error", "code": "internal_error", "message": "boom"}}, - ] - sse_payload = b"".join(f"data: {json.dumps(event)}\n\n".encode() for event in events) - - async def mock_aiter_bytes(): - yield sse_payload - - mock_response = Mock() - mock_response.headers = {} - mock_response.aiter_bytes = mock_aiter_bytes - mock_logging_obj = MagicMock(spec=LiteLLMLoggingObj) - mock_logging_obj.model_call_details = {"litellm_params": {}} - mock_logging_obj.completion_start_time = None - mock_config = Mock(spec=BaseResponsesAPIConfig) - - def transform(model, parsed_chunk, logging_obj): - if parsed_chunk.get("type") == "error": - return ErrorEvent( - type=ResponsesAPIStreamEvents.ERROR, - sequence_number=0, - error=ErrorEventError(**parsed_chunk["error"]), - ) - delta_event = Mock() - delta_event.type = ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA - delta_event.delta = parsed_chunk["delta"] - return delta_event - - mock_config.transform_streaming_response.side_effect = transform - - source = ResponsesAPIStreamingIterator( - response=mock_response, +async def test_aresponses_partial_text_surfaces_provider_error(): + router: Final = _make_router() + payload: Final = ( + 'data: {"type":"response.output_text.delta","output_index":0,"content_index":0,"item_id":"msg_primary","sequence_number":0,"delta":"partial answer"}\n\n' + 'data: {"type":"error","sequence_number":1,"error":{"type":"server_error","code":"internal_error","message":"boom"}}\n\n' + ) + source: Final = ResponsesAPIStreamingIterator( + response=httpx.Response(200, content=payload), model="gpt-5", - responses_api_provider_config=mock_config, - logging_obj=mock_logging_obj, + responses_api_provider_config=OpenAIResponsesAPIConfig(), + logging_obj=LiteLLMLoggingObj( + model="gpt-5", + messages=[], + stream=True, + call_type="aresponses", + start_time=datetime(2026, 1, 1), + litellm_call_id="partial-text-regression", + function_id="partial-text-regression", + ), custom_llm_provider="openai", ) - - fallback_event = _make_completed_event(1, 1, 2) - - class _FallbackStream: - def __init__(self) -> None: - self._done = False - - def __aiter__(self): - return self - - async def __anext__(self): - if self._done: - raise StopAsyncIteration - self._done = True - return fallback_event - - with patch.object( - router, - "async_function_with_fallbacks_common_utils", - new=AsyncMock(return_value=_FallbackStream()), - ) as mock_fallback: - wrapped = await router._aresponses_streaming_iterator( - response=source, - initial_kwargs={"model": "primary", "input": "original question"}, - ) - collected = [ev async for ev in wrapped] - - assert collected[-1] == fallback_event - raised = mock_fallback.await_args.kwargs["e"] - assert isinstance(raised, MidStreamFallbackError) - assert raised.is_pre_first_chunk is False - assert raised.generated_content == "partial answer" - continuation = mock_fallback.await_args.kwargs["kwargs"]["input"] - assert isinstance(continuation, list) - assert continuation[0]["content"][0]["text"] == "original question" - assert continuation[-2]["role"] == "developer" - assert continuation[-1]["role"] == "assistant" - assert continuation[-1]["content"][0]["text"] == "partial answer" + wrapped: Final = await router._aresponses_streaming_iterator( + response=source, + initial_kwargs={"model": "primary", "input": "original question"}, + ) + first: Final = await anext(wrapped) + assert first.delta == "partial answer" + with pytest.raises(litellm.InternalServerError, match="boom"): + await anext(wrapped) @pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") @pytest.mark.asyncio diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 3004f83b1df..f938f15a91f 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -7038,50 +7038,121 @@ async def test_aresponses_fallback_attempt_does_not_replay_held_lifecycle_events @pytest.mark.asyncio -async def test_aresponses_streaming_iterator_partial_content_injects_continuation(): - """Mid-stream error: input is rewritten to include user prompt + - developer instruction + prior assistant message with partial output.""" - from litellm.exceptions import MidStreamFallbackError +@pytest.mark.parametrize("failure_type", ["error", "response.failed"]) +@pytest.mark.parametrize( + "output_sse", + [ + "", + '{"type":"response.output_item.added","output_index":0,"sequence_number":1,"item":{"type":"function_call","id":"fc_primary","call_id":"call_primary","name":"record_run","arguments":"","status":"in_progress"}}', + '{"type":"response.output_item.done","output_index":0,"sequence_number":1,"item":{"type":"function_call","id":"fc_primary","call_id":"call_primary","name":"record_run","arguments":"{}","status":"completed"}}', + '{"type":"response.output_text.delta","output_index":0,"content_index":0,"item_id":"msg_primary","sequence_number":1,"delta":"hello"}', + '{"type":"image_generation.partial_image","partial_image_index":0,"b64_json":"aW1hZ2U="}', + ], + ids=["before-output", "tool-started", "tool-completed", "text", "partial-image"], +) +async def test_aresponses_does_not_replay_delivered_output( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + output_sse: str, + failure_type: str, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + primary_response: Final = { + "id": "resp_primary", + "object": "response", + "created_at": 0, + "model": "primary", + "status": "in_progress", + "output": [], + } + provider_error: Final = {"type": "server_error", "code": "server_error", "message": "primary failed"} + failure: Final = ( + {"type": "error", "sequence_number": 2, "error": provider_error} + if failure_type == "error" + else { + "type": "response.failed", + "sequence_number": 2, + "response": {**primary_response, "status": "failed", "error": provider_error}, + } + ) + primary_body: Final = ( + f"data: {json.dumps({'type': 'response.created', 'sequence_number': 0, 'response': primary_response})}\n\n" + + (f"data: {output_sse}\n\n" if output_sse else "") + + f"data: {json.dumps(failure)}\n\n" + ) + fallback_response: Final = {**primary_response, "id": "resp_fallback", "model": "fallback", "status": "completed"} + fallback_body: Final = ( + f"data: {json.dumps({'type': 'response.created', 'sequence_number': 0, 'response': fallback_response})}\n\n" + f"data: {json.dumps({'type': 'response.completed', 'sequence_number': 1, 'response': fallback_response})}\n\n" + ) + primary: Final = respx_mock.post("https://primary.responses.test/v1/responses").mock( + return_value=httpx.Response(200, text=primary_body, headers={"content-type": "text/event-stream"}) + ) + fallback: Final = respx_mock.post("https://fallback.responses.test/v1/responses").mock( + return_value=httpx.Response(200, text=fallback_body, headers={"content-type": "text/event-stream"}) + ) + router: Final = Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "openai/primary", + "api_base": "https://primary.responses.test/v1", + "api_key": "local-placeholder", + }, + }, + { + "model_name": "fallback", + "litellm_params": { + "model": "openai/fallback", + "api_base": "https://fallback.responses.test/v1", + "api_key": "local-placeholder", + }, + }, + ], + fallbacks=[{"primary": ["fallback"]}], + num_retries=0, + ) + stream: Final = await router.aresponses(model="primary", input="run the tool", stream=True) + outcome: Final = [event async for event in _events_until_error(stream)] - router = _make_router_with_fallback() - src = _make_responses_iterator( - chunks=[MagicMock(type="response.output_text.delta")], + assert primary.call_count == 1 + if not output_sse: + assert fallback.call_count == 1 + assert [getattr(event, "type", None) for event in outcome] == ["response.created", "response.completed"] + return + assert fallback.call_count == 0 + assert [getattr(event, "type", None) for event in outcome[:-1]] == [ + "response.created", + json.loads(output_sse)["type"], + ] + assert isinstance(outcome[-1], litellm.InternalServerError) + assert "primary failed" in str(outcome[-1]) + + +@pytest.mark.asyncio +async def test_aresponses_streaming_iterator_partial_content_surfaces_original_error(): + router: Final = _make_router_with_fallback() + provider_error: Final = litellm.InternalServerError(message="primary failed", model="gpt-4", llm_provider="openai") + output: Final = SimpleNamespace(type="response.output_text.delta", output_index=0, delta="partial answer") + source: Final = _make_responses_iterator( + chunks=[output], error=MidStreamFallbackError( - message="socket reset mid-stream", + message="primary failed", model="gpt-4", - llm_provider="anthropic", + llm_provider="openai", is_pre_first_chunk=False, - generated_content="The capital of France is", + generated_content="partial answer", + original_exception=provider_error, ), ) + wrapped: Final = await router._aresponses_streaming_iterator( + response=source, + initial_kwargs={"model": "gpt-4", "stream": True, "input": "Hello"}, + ) + outcome: Final = [event async for event in _events_until_error(wrapped)] - with patch.object( - router, - "async_function_with_fallbacks_common_utils", - return_value=_AsyncList(), - ) as mock_fallback_utils: - wrapped = await router._aresponses_streaming_iterator( - response=src, - initial_kwargs={ - "model": "gpt-4", - "stream": True, - "input": "What's the capital of France?", - "original_generic_function": litellm.aresponses, - }, - ) - async for _ in wrapped: - pass - - new_input = mock_fallback_utils.call_args.kwargs["kwargs"]["input"] - assert isinstance(new_input, list) - assert new_input[0]["role"] == "user" - assert new_input[0]["content"][0]["text"] == "What's the capital of France?" - assert new_input[1]["role"] == "developer" - assert "do not repeat" in new_input[1]["content"][0]["text"].lower() - assert new_input[2]["role"] == "assistant" - assert new_input[2]["content"][0]["type"] == "output_text" - assert new_input[2]["content"][0]["text"] == "The capital of France is" - + assert outcome == [output, provider_error] @pytest.mark.asyncio async def test_aresponses_streaming_iterator_combines_partial_usage(): @@ -7102,13 +7173,13 @@ async def test_aresponses_streaming_iterator_combines_partial_usage(): src = _make_responses_iterator( bridge=True, chat_chunks=[MagicMock()], - chunks=[MagicMock(type="response.output_text.delta")], + chunks=[], error=MidStreamFallbackError( message="boom", model="gpt-4", llm_provider="anthropic", is_pre_first_chunk=False, - generated_content="hello", + generated_content="", ), )