fix(router): prevent Responses tool-call replay (#42955)

This commit is contained in:
ishaan-j-berri 2026-10-26 20:46:30 -07:00
parent aa33b052d2
commit 33cebf85e2
3 changed files with 148 additions and 131 deletions

View file

@ -4006,9 +4006,8 @@ class Router:
Full parity with the chat-completions path:
- Pre-first-chunk: retry with the original input unchanged.
- Partial content: inject a developer instruction + prior
assistant message carrying the generated text so the fallback
model continues rather than restarts.
- Delivered output: surface the original error rather than
restarting an output item or tool call on a fallback.
- Usage combining: merge partial-stream usage onto the fallback's
response.completed event so accounting reflects both attempts.
- Stream cleanup: shielded aclose() on both source and fallback
@ -4158,6 +4157,7 @@ class Router:
async def stream_with_fallbacks():
held_lifecycle_events: tuple[object, ...] = () # rebind-ok: flushed at first output, dropped on fallback
has_forwarded_output = False # rebind-ok: updated as output events reach the client
try:
async for item in cast( # cast-ok: response iterators are async streams
AsyncIterator[object], source_iterator
@ -4168,10 +4168,17 @@ class Router:
for held_event in held_lifecycle_events:
yield held_event
held_lifecycle_events = ()
if isinstance(getattr(item, "output_index", None), int) or (
getattr(item, "type", None)
== _openai_types.ResponsesAPIStreamEvents.IMAGE_GENERATION_PARTIAL_IMAGE
):
has_forwarded_output = True
yield item
for held_event in held_lifecycle_events:
yield held_event
except MidStreamFallbackError as e:
if has_forwarded_output:
raise e.original_exception or e
async with contextlib.aclosing(
self._aresponses_fallback_attempt(
e, source_iterator, initial_kwargs, wrapper.adopt_fallback_headers, held_lifecycle_events

View file

@ -12,7 +12,7 @@ from litellm import Router
from litellm.constants import STREAM_SSE_DONE_STRING
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator, ResponsesAPIStreamingIterator
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import(
OutputTextDeltaEvent,
@ -24,6 +24,7 @@ from litellm.types.llms.openai import(
ResponsesAPIStreamEvents,
)
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
from datetime import datetime
from typing import Any, AsyncIterator, Final, List
from unittest.mock import AsyncMock, MagicMock, Mock, patch
@ -1267,97 +1268,35 @@ async def test_aresponses_fallback_on_in_stream_error_event():
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
@pytest.mark.asyncio
async def test_aresponses_fallback_uses_continuation_input_after_partial_content():
"""When output text was already streamed before the error, the fallback re-entry
must carry a continuation input with the partial assistant text instead of
retrying the original input from scratch (which would duplicate streamed content)."""
import json
from unittest.mock import Mock
from litellm.exceptions import MidStreamFallbackError
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
from litellm.types.llms.openai import ErrorEvent, ErrorEventError
router = _make_router()
events = [
{"type": "response.output_text.delta", "delta": "partial answer"},
{"type": "error", "error": {"type": "server_error", "code": "internal_error", "message": "boom"}},
]
sse_payload = b"".join(f"data: {json.dumps(event)}\n\n".encode() for event in events)
async def mock_aiter_bytes():
yield sse_payload
mock_response = Mock()
mock_response.headers = {}
mock_response.aiter_bytes = mock_aiter_bytes
mock_logging_obj = MagicMock(spec=LiteLLMLoggingObj)
mock_logging_obj.model_call_details = {"litellm_params": {}}
mock_logging_obj.completion_start_time = None
mock_config = Mock(spec=BaseResponsesAPIConfig)
def transform(model, parsed_chunk, logging_obj):
if parsed_chunk.get("type") == "error":
return ErrorEvent(
type=ResponsesAPIStreamEvents.ERROR,
sequence_number=0,
error=ErrorEventError(**parsed_chunk["error"]),
)
delta_event = Mock()
delta_event.type = ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA
delta_event.delta = parsed_chunk["delta"]
return delta_event
mock_config.transform_streaming_response.side_effect = transform
source = ResponsesAPIStreamingIterator(
response=mock_response,
async def test_aresponses_partial_text_surfaces_provider_error():
router: Final = _make_router()
payload: Final = (
'data: {"type":"response.output_text.delta","output_index":0,"content_index":0,"item_id":"msg_primary","sequence_number":0,"delta":"partial answer"}\n\n'
'data: {"type":"error","sequence_number":1,"error":{"type":"server_error","code":"internal_error","message":"boom"}}\n\n'
)
source: Final = ResponsesAPIStreamingIterator(
response=httpx.Response(200, content=payload),
model="gpt-5",
responses_api_provider_config=mock_config,
logging_obj=mock_logging_obj,
responses_api_provider_config=OpenAIResponsesAPIConfig(),
logging_obj=LiteLLMLoggingObj(
model="gpt-5",
messages=[],
stream=True,
call_type="aresponses",
start_time=datetime(2026, 1, 1),
litellm_call_id="partial-text-regression",
function_id="partial-text-regression",
),
custom_llm_provider="openai",
)
fallback_event = _make_completed_event(1, 1, 2)
class _FallbackStream:
def __init__(self) -> None:
self._done = False
def __aiter__(self):
return self
async def __anext__(self):
if self._done:
raise StopAsyncIteration
self._done = True
return fallback_event
with patch.object(
router,
"async_function_with_fallbacks_common_utils",
new=AsyncMock(return_value=_FallbackStream()),
) as mock_fallback:
wrapped = await router._aresponses_streaming_iterator(
response=source,
initial_kwargs={"model": "primary", "input": "original question"},
)
collected = [ev async for ev in wrapped]
assert collected[-1] == fallback_event
raised = mock_fallback.await_args.kwargs["e"]
assert isinstance(raised, MidStreamFallbackError)
assert raised.is_pre_first_chunk is False
assert raised.generated_content == "partial answer"
continuation = mock_fallback.await_args.kwargs["kwargs"]["input"]
assert isinstance(continuation, list)
assert continuation[0]["content"][0]["text"] == "original question"
assert continuation[-2]["role"] == "developer"
assert continuation[-1]["role"] == "assistant"
assert continuation[-1]["content"][0]["text"] == "partial answer"
wrapped: Final = await router._aresponses_streaming_iterator(
response=source,
initial_kwargs={"model": "primary", "input": "original question"},
)
first: Final = await anext(wrapped)
assert first.delta == "partial answer"
with pytest.raises(litellm.InternalServerError, match="boom"):
await anext(wrapped)
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
@pytest.mark.asyncio

View file

@ -7038,50 +7038,121 @@ async def test_aresponses_fallback_attempt_does_not_replay_held_lifecycle_events
@pytest.mark.asyncio
async def test_aresponses_streaming_iterator_partial_content_injects_continuation():
"""Mid-stream error: input is rewritten to include user prompt +
developer instruction + prior assistant message with partial output."""
from litellm.exceptions import MidStreamFallbackError
@pytest.mark.parametrize("failure_type", ["error", "response.failed"])
@pytest.mark.parametrize(
"output_sse",
[
"",
'{"type":"response.output_item.added","output_index":0,"sequence_number":1,"item":{"type":"function_call","id":"fc_primary","call_id":"call_primary","name":"record_run","arguments":"","status":"in_progress"}}',
'{"type":"response.output_item.done","output_index":0,"sequence_number":1,"item":{"type":"function_call","id":"fc_primary","call_id":"call_primary","name":"record_run","arguments":"{}","status":"completed"}}',
'{"type":"response.output_text.delta","output_index":0,"content_index":0,"item_id":"msg_primary","sequence_number":1,"delta":"hello"}',
'{"type":"image_generation.partial_image","partial_image_index":0,"b64_json":"aW1hZ2U="}',
],
ids=["before-output", "tool-started", "tool-completed", "text", "partial-image"],
)
async def test_aresponses_does_not_replay_delivered_output(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
output_sse: str,
failure_type: str,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
primary_response: Final = {
"id": "resp_primary",
"object": "response",
"created_at": 0,
"model": "primary",
"status": "in_progress",
"output": [],
}
provider_error: Final = {"type": "server_error", "code": "server_error", "message": "primary failed"}
failure: Final = (
{"type": "error", "sequence_number": 2, "error": provider_error}
if failure_type == "error"
else {
"type": "response.failed",
"sequence_number": 2,
"response": {**primary_response, "status": "failed", "error": provider_error},
}
)
primary_body: Final = (
f"data: {json.dumps({'type': 'response.created', 'sequence_number': 0, 'response': primary_response})}\n\n"
+ (f"data: {output_sse}\n\n" if output_sse else "")
+ f"data: {json.dumps(failure)}\n\n"
)
fallback_response: Final = {**primary_response, "id": "resp_fallback", "model": "fallback", "status": "completed"}
fallback_body: Final = (
f"data: {json.dumps({'type': 'response.created', 'sequence_number': 0, 'response': fallback_response})}\n\n"
f"data: {json.dumps({'type': 'response.completed', 'sequence_number': 1, 'response': fallback_response})}\n\n"
)
primary: Final = respx_mock.post("https://primary.responses.test/v1/responses").mock(
return_value=httpx.Response(200, text=primary_body, headers={"content-type": "text/event-stream"})
)
fallback: Final = respx_mock.post("https://fallback.responses.test/v1/responses").mock(
return_value=httpx.Response(200, text=fallback_body, headers={"content-type": "text/event-stream"})
)
router: Final = Router(
model_list=[
{
"model_name": "primary",
"litellm_params": {
"model": "openai/primary",
"api_base": "https://primary.responses.test/v1",
"api_key": "local-placeholder",
},
},
{
"model_name": "fallback",
"litellm_params": {
"model": "openai/fallback",
"api_base": "https://fallback.responses.test/v1",
"api_key": "local-placeholder",
},
},
],
fallbacks=[{"primary": ["fallback"]}],
num_retries=0,
)
stream: Final = await router.aresponses(model="primary", input="run the tool", stream=True)
outcome: Final = [event async for event in _events_until_error(stream)]
router = _make_router_with_fallback()
src = _make_responses_iterator(
chunks=[MagicMock(type="response.output_text.delta")],
assert primary.call_count == 1
if not output_sse:
assert fallback.call_count == 1
assert [getattr(event, "type", None) for event in outcome] == ["response.created", "response.completed"]
return
assert fallback.call_count == 0
assert [getattr(event, "type", None) for event in outcome[:-1]] == [
"response.created",
json.loads(output_sse)["type"],
]
assert isinstance(outcome[-1], litellm.InternalServerError)
assert "primary failed" in str(outcome[-1])
@pytest.mark.asyncio
async def test_aresponses_streaming_iterator_partial_content_surfaces_original_error():
router: Final = _make_router_with_fallback()
provider_error: Final = litellm.InternalServerError(message="primary failed", model="gpt-4", llm_provider="openai")
output: Final = SimpleNamespace(type="response.output_text.delta", output_index=0, delta="partial answer")
source: Final = _make_responses_iterator(
chunks=[output],
error=MidStreamFallbackError(
message="socket reset mid-stream",
message="primary failed",
model="gpt-4",
llm_provider="anthropic",
llm_provider="openai",
is_pre_first_chunk=False,
generated_content="The capital of France is",
generated_content="partial answer",
original_exception=provider_error,
),
)
wrapped: Final = await router._aresponses_streaming_iterator(
response=source,
initial_kwargs={"model": "gpt-4", "stream": True, "input": "Hello"},
)
outcome: Final = [event async for event in _events_until_error(wrapped)]
with patch.object(
router,
"async_function_with_fallbacks_common_utils",
return_value=_AsyncList(),
) as mock_fallback_utils:
wrapped = await router._aresponses_streaming_iterator(
response=src,
initial_kwargs={
"model": "gpt-4",
"stream": True,
"input": "What's the capital of France?",
"original_generic_function": litellm.aresponses,
},
)
async for _ in wrapped:
pass
new_input = mock_fallback_utils.call_args.kwargs["kwargs"]["input"]
assert isinstance(new_input, list)
assert new_input[0]["role"] == "user"
assert new_input[0]["content"][0]["text"] == "What's the capital of France?"
assert new_input[1]["role"] == "developer"
assert "do not repeat" in new_input[1]["content"][0]["text"].lower()
assert new_input[2]["role"] == "assistant"
assert new_input[2]["content"][0]["type"] == "output_text"
assert new_input[2]["content"][0]["text"] == "The capital of France is"
assert outcome == [output, provider_error]
@pytest.mark.asyncio
async def test_aresponses_streaming_iterator_combines_partial_usage():
@ -7102,13 +7173,13 @@ async def test_aresponses_streaming_iterator_combines_partial_usage():
src = _make_responses_iterator(
bridge=True,
chat_chunks=[MagicMock()],
chunks=[MagicMock(type="response.output_text.delta")],
chunks=[],
error=MidStreamFallbackError(
message="boom",
model="gpt-4",
llm_provider="anthropic",
is_pre_first_chunk=False,
generated_content="hello",
generated_content="",
),
)