mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(router): prevent Responses tool-call replay (#42955)
This commit is contained in:
parent
aa33b052d2
commit
33cebf85e2
3 changed files with 148 additions and 131 deletions
|
|
@ -4006,9 +4006,8 @@ class Router:
|
|||
|
||||
Full parity with the chat-completions path:
|
||||
- Pre-first-chunk: retry with the original input unchanged.
|
||||
- Partial content: inject a developer instruction + prior
|
||||
assistant message carrying the generated text so the fallback
|
||||
model continues rather than restarts.
|
||||
- Delivered output: surface the original error rather than
|
||||
restarting an output item or tool call on a fallback.
|
||||
- Usage combining: merge partial-stream usage onto the fallback's
|
||||
response.completed event so accounting reflects both attempts.
|
||||
- Stream cleanup: shielded aclose() on both source and fallback
|
||||
|
|
@ -4158,6 +4157,7 @@ class Router:
|
|||
|
||||
async def stream_with_fallbacks():
|
||||
held_lifecycle_events: tuple[object, ...] = () # rebind-ok: flushed at first output, dropped on fallback
|
||||
has_forwarded_output = False # rebind-ok: updated as output events reach the client
|
||||
try:
|
||||
async for item in cast( # cast-ok: response iterators are async streams
|
||||
AsyncIterator[object], source_iterator
|
||||
|
|
@ -4168,10 +4168,17 @@ class Router:
|
|||
for held_event in held_lifecycle_events:
|
||||
yield held_event
|
||||
held_lifecycle_events = ()
|
||||
if isinstance(getattr(item, "output_index", None), int) or (
|
||||
getattr(item, "type", None)
|
||||
== _openai_types.ResponsesAPIStreamEvents.IMAGE_GENERATION_PARTIAL_IMAGE
|
||||
):
|
||||
has_forwarded_output = True
|
||||
yield item
|
||||
for held_event in held_lifecycle_events:
|
||||
yield held_event
|
||||
except MidStreamFallbackError as e:
|
||||
if has_forwarded_output:
|
||||
raise e.original_exception or e
|
||||
async with contextlib.aclosing(
|
||||
self._aresponses_fallback_attempt(
|
||||
e, source_iterator, initial_kwargs, wrapper.adopt_fallback_headers, held_lifecycle_events
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from litellm import Router
|
|||
from litellm.constants import STREAM_SSE_DONE_STRING
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator, ResponsesAPIStreamingIterator
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.types.llms.openai import(
|
||||
OutputTextDeltaEvent,
|
||||
|
|
@ -24,6 +24,7 @@ from litellm.types.llms.openai import(
|
|||
ResponsesAPIStreamEvents,
|
||||
)
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
from datetime import datetime
|
||||
from typing import Any, AsyncIterator, Final, List
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
|
|
@ -1267,97 +1268,35 @@ async def test_aresponses_fallback_on_in_stream_error_event():
|
|||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_fallback_uses_continuation_input_after_partial_content():
|
||||
"""When output text was already streamed before the error, the fallback re-entry
|
||||
must carry a continuation input with the partial assistant text instead of
|
||||
retrying the original input from scratch (which would duplicate streamed content)."""
|
||||
import json
|
||||
from unittest.mock import Mock
|
||||
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
|
||||
from litellm.types.llms.openai import ErrorEvent, ErrorEventError
|
||||
|
||||
router = _make_router()
|
||||
|
||||
events = [
|
||||
{"type": "response.output_text.delta", "delta": "partial answer"},
|
||||
{"type": "error", "error": {"type": "server_error", "code": "internal_error", "message": "boom"}},
|
||||
]
|
||||
sse_payload = b"".join(f"data: {json.dumps(event)}\n\n".encode() for event in events)
|
||||
|
||||
async def mock_aiter_bytes():
|
||||
yield sse_payload
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.headers = {}
|
||||
mock_response.aiter_bytes = mock_aiter_bytes
|
||||
mock_logging_obj = MagicMock(spec=LiteLLMLoggingObj)
|
||||
mock_logging_obj.model_call_details = {"litellm_params": {}}
|
||||
mock_logging_obj.completion_start_time = None
|
||||
mock_config = Mock(spec=BaseResponsesAPIConfig)
|
||||
|
||||
def transform(model, parsed_chunk, logging_obj):
|
||||
if parsed_chunk.get("type") == "error":
|
||||
return ErrorEvent(
|
||||
type=ResponsesAPIStreamEvents.ERROR,
|
||||
sequence_number=0,
|
||||
error=ErrorEventError(**parsed_chunk["error"]),
|
||||
)
|
||||
delta_event = Mock()
|
||||
delta_event.type = ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA
|
||||
delta_event.delta = parsed_chunk["delta"]
|
||||
return delta_event
|
||||
|
||||
mock_config.transform_streaming_response.side_effect = transform
|
||||
|
||||
source = ResponsesAPIStreamingIterator(
|
||||
response=mock_response,
|
||||
async def test_aresponses_partial_text_surfaces_provider_error():
|
||||
router: Final = _make_router()
|
||||
payload: Final = (
|
||||
'data: {"type":"response.output_text.delta","output_index":0,"content_index":0,"item_id":"msg_primary","sequence_number":0,"delta":"partial answer"}\n\n'
|
||||
'data: {"type":"error","sequence_number":1,"error":{"type":"server_error","code":"internal_error","message":"boom"}}\n\n'
|
||||
)
|
||||
source: Final = ResponsesAPIStreamingIterator(
|
||||
response=httpx.Response(200, content=payload),
|
||||
model="gpt-5",
|
||||
responses_api_provider_config=mock_config,
|
||||
logging_obj=mock_logging_obj,
|
||||
responses_api_provider_config=OpenAIResponsesAPIConfig(),
|
||||
logging_obj=LiteLLMLoggingObj(
|
||||
model="gpt-5",
|
||||
messages=[],
|
||||
stream=True,
|
||||
call_type="aresponses",
|
||||
start_time=datetime(2026, 1, 1),
|
||||
litellm_call_id="partial-text-regression",
|
||||
function_id="partial-text-regression",
|
||||
),
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
fallback_event = _make_completed_event(1, 1, 2)
|
||||
|
||||
class _FallbackStream:
|
||||
def __init__(self) -> None:
|
||||
self._done = False
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
if self._done:
|
||||
raise StopAsyncIteration
|
||||
self._done = True
|
||||
return fallback_event
|
||||
|
||||
with patch.object(
|
||||
router,
|
||||
"async_function_with_fallbacks_common_utils",
|
||||
new=AsyncMock(return_value=_FallbackStream()),
|
||||
) as mock_fallback:
|
||||
wrapped = await router._aresponses_streaming_iterator(
|
||||
response=source,
|
||||
initial_kwargs={"model": "primary", "input": "original question"},
|
||||
)
|
||||
collected = [ev async for ev in wrapped]
|
||||
|
||||
assert collected[-1] == fallback_event
|
||||
raised = mock_fallback.await_args.kwargs["e"]
|
||||
assert isinstance(raised, MidStreamFallbackError)
|
||||
assert raised.is_pre_first_chunk is False
|
||||
assert raised.generated_content == "partial answer"
|
||||
continuation = mock_fallback.await_args.kwargs["kwargs"]["input"]
|
||||
assert isinstance(continuation, list)
|
||||
assert continuation[0]["content"][0]["text"] == "original question"
|
||||
assert continuation[-2]["role"] == "developer"
|
||||
assert continuation[-1]["role"] == "assistant"
|
||||
assert continuation[-1]["content"][0]["text"] == "partial answer"
|
||||
wrapped: Final = await router._aresponses_streaming_iterator(
|
||||
response=source,
|
||||
initial_kwargs={"model": "primary", "input": "original question"},
|
||||
)
|
||||
first: Final = await anext(wrapped)
|
||||
assert first.delta == "partial answer"
|
||||
with pytest.raises(litellm.InternalServerError, match="boom"):
|
||||
await anext(wrapped)
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -7038,50 +7038,121 @@ async def test_aresponses_fallback_attempt_does_not_replay_held_lifecycle_events
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_streaming_iterator_partial_content_injects_continuation():
|
||||
"""Mid-stream error: input is rewritten to include user prompt +
|
||||
developer instruction + prior assistant message with partial output."""
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
@pytest.mark.parametrize("failure_type", ["error", "response.failed"])
|
||||
@pytest.mark.parametrize(
|
||||
"output_sse",
|
||||
[
|
||||
"",
|
||||
'{"type":"response.output_item.added","output_index":0,"sequence_number":1,"item":{"type":"function_call","id":"fc_primary","call_id":"call_primary","name":"record_run","arguments":"","status":"in_progress"}}',
|
||||
'{"type":"response.output_item.done","output_index":0,"sequence_number":1,"item":{"type":"function_call","id":"fc_primary","call_id":"call_primary","name":"record_run","arguments":"{}","status":"completed"}}',
|
||||
'{"type":"response.output_text.delta","output_index":0,"content_index":0,"item_id":"msg_primary","sequence_number":1,"delta":"hello"}',
|
||||
'{"type":"image_generation.partial_image","partial_image_index":0,"b64_json":"aW1hZ2U="}',
|
||||
],
|
||||
ids=["before-output", "tool-started", "tool-completed", "text", "partial-image"],
|
||||
)
|
||||
async def test_aresponses_does_not_replay_delivered_output(
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
output_sse: str,
|
||||
failure_type: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
primary_response: Final = {
|
||||
"id": "resp_primary",
|
||||
"object": "response",
|
||||
"created_at": 0,
|
||||
"model": "primary",
|
||||
"status": "in_progress",
|
||||
"output": [],
|
||||
}
|
||||
provider_error: Final = {"type": "server_error", "code": "server_error", "message": "primary failed"}
|
||||
failure: Final = (
|
||||
{"type": "error", "sequence_number": 2, "error": provider_error}
|
||||
if failure_type == "error"
|
||||
else {
|
||||
"type": "response.failed",
|
||||
"sequence_number": 2,
|
||||
"response": {**primary_response, "status": "failed", "error": provider_error},
|
||||
}
|
||||
)
|
||||
primary_body: Final = (
|
||||
f"data: {json.dumps({'type': 'response.created', 'sequence_number': 0, 'response': primary_response})}\n\n"
|
||||
+ (f"data: {output_sse}\n\n" if output_sse else "")
|
||||
+ f"data: {json.dumps(failure)}\n\n"
|
||||
)
|
||||
fallback_response: Final = {**primary_response, "id": "resp_fallback", "model": "fallback", "status": "completed"}
|
||||
fallback_body: Final = (
|
||||
f"data: {json.dumps({'type': 'response.created', 'sequence_number': 0, 'response': fallback_response})}\n\n"
|
||||
f"data: {json.dumps({'type': 'response.completed', 'sequence_number': 1, 'response': fallback_response})}\n\n"
|
||||
)
|
||||
primary: Final = respx_mock.post("https://primary.responses.test/v1/responses").mock(
|
||||
return_value=httpx.Response(200, text=primary_body, headers={"content-type": "text/event-stream"})
|
||||
)
|
||||
fallback: Final = respx_mock.post("https://fallback.responses.test/v1/responses").mock(
|
||||
return_value=httpx.Response(200, text=fallback_body, headers={"content-type": "text/event-stream"})
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "primary",
|
||||
"litellm_params": {
|
||||
"model": "openai/primary",
|
||||
"api_base": "https://primary.responses.test/v1",
|
||||
"api_key": "local-placeholder",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "fallback",
|
||||
"litellm_params": {
|
||||
"model": "openai/fallback",
|
||||
"api_base": "https://fallback.responses.test/v1",
|
||||
"api_key": "local-placeholder",
|
||||
},
|
||||
},
|
||||
],
|
||||
fallbacks=[{"primary": ["fallback"]}],
|
||||
num_retries=0,
|
||||
)
|
||||
stream: Final = await router.aresponses(model="primary", input="run the tool", stream=True)
|
||||
outcome: Final = [event async for event in _events_until_error(stream)]
|
||||
|
||||
router = _make_router_with_fallback()
|
||||
src = _make_responses_iterator(
|
||||
chunks=[MagicMock(type="response.output_text.delta")],
|
||||
assert primary.call_count == 1
|
||||
if not output_sse:
|
||||
assert fallback.call_count == 1
|
||||
assert [getattr(event, "type", None) for event in outcome] == ["response.created", "response.completed"]
|
||||
return
|
||||
assert fallback.call_count == 0
|
||||
assert [getattr(event, "type", None) for event in outcome[:-1]] == [
|
||||
"response.created",
|
||||
json.loads(output_sse)["type"],
|
||||
]
|
||||
assert isinstance(outcome[-1], litellm.InternalServerError)
|
||||
assert "primary failed" in str(outcome[-1])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_streaming_iterator_partial_content_surfaces_original_error():
|
||||
router: Final = _make_router_with_fallback()
|
||||
provider_error: Final = litellm.InternalServerError(message="primary failed", model="gpt-4", llm_provider="openai")
|
||||
output: Final = SimpleNamespace(type="response.output_text.delta", output_index=0, delta="partial answer")
|
||||
source: Final = _make_responses_iterator(
|
||||
chunks=[output],
|
||||
error=MidStreamFallbackError(
|
||||
message="socket reset mid-stream",
|
||||
message="primary failed",
|
||||
model="gpt-4",
|
||||
llm_provider="anthropic",
|
||||
llm_provider="openai",
|
||||
is_pre_first_chunk=False,
|
||||
generated_content="The capital of France is",
|
||||
generated_content="partial answer",
|
||||
original_exception=provider_error,
|
||||
),
|
||||
)
|
||||
wrapped: Final = await router._aresponses_streaming_iterator(
|
||||
response=source,
|
||||
initial_kwargs={"model": "gpt-4", "stream": True, "input": "Hello"},
|
||||
)
|
||||
outcome: Final = [event async for event in _events_until_error(wrapped)]
|
||||
|
||||
with patch.object(
|
||||
router,
|
||||
"async_function_with_fallbacks_common_utils",
|
||||
return_value=_AsyncList(),
|
||||
) as mock_fallback_utils:
|
||||
wrapped = await router._aresponses_streaming_iterator(
|
||||
response=src,
|
||||
initial_kwargs={
|
||||
"model": "gpt-4",
|
||||
"stream": True,
|
||||
"input": "What's the capital of France?",
|
||||
"original_generic_function": litellm.aresponses,
|
||||
},
|
||||
)
|
||||
async for _ in wrapped:
|
||||
pass
|
||||
|
||||
new_input = mock_fallback_utils.call_args.kwargs["kwargs"]["input"]
|
||||
assert isinstance(new_input, list)
|
||||
assert new_input[0]["role"] == "user"
|
||||
assert new_input[0]["content"][0]["text"] == "What's the capital of France?"
|
||||
assert new_input[1]["role"] == "developer"
|
||||
assert "do not repeat" in new_input[1]["content"][0]["text"].lower()
|
||||
assert new_input[2]["role"] == "assistant"
|
||||
assert new_input[2]["content"][0]["type"] == "output_text"
|
||||
assert new_input[2]["content"][0]["text"] == "The capital of France is"
|
||||
|
||||
assert outcome == [output, provider_error]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_streaming_iterator_combines_partial_usage():
|
||||
|
|
@ -7102,13 +7173,13 @@ async def test_aresponses_streaming_iterator_combines_partial_usage():
|
|||
src = _make_responses_iterator(
|
||||
bridge=True,
|
||||
chat_chunks=[MagicMock()],
|
||||
chunks=[MagicMock(type="response.output_text.delta")],
|
||||
chunks=[],
|
||||
error=MidStreamFallbackError(
|
||||
message="boom",
|
||||
model="gpt-4",
|
||||
llm_provider="anthropic",
|
||||
is_pre_first_chunk=False,
|
||||
generated_content="hello",
|
||||
generated_content="",
|
||||
),
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue