mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(guardrails): read an error frame's name as clients do and send the error alone when a rewrite cannot be built
A data-less event: error frame with junk after the name is an unknown event to both SDKs, so a healthy stream stays raw. A finished tool call whose input is not JSON cannot be rebuilt, so a failed stream sends its upstream error alone, and a healthy stream still raises as before.
This commit is contained in:
parent
1e11e9c4b9
commit
391a4e66a8
6 changed files with 117 additions and 6 deletions
|
|
@ -578,9 +578,7 @@ def anthropic_sse_event_types(all_chunks: Sequence[object]) -> frozenset[str]:
|
|||
|
||||
|
||||
def _is_named_error_event(event: str) -> bool:
|
||||
return any(
|
||||
line.startswith("event:") and line[len("event:") :].strip() == "error" for line in split_sse_lines(event)
|
||||
)
|
||||
return ("event", "error") in _sse_fields(event)
|
||||
|
||||
|
||||
def _is_error_event(event: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -151,6 +151,10 @@ def _sent_by_the_failed_upstream(chunk: bytes, block_left_open: int | None, end_
|
|||
return block_left_open is None or anthropic_sse_stopped_block_index(event) != block_left_open
|
||||
|
||||
|
||||
def _sse_chunks_of_events(events: Sequence[str]) -> tuple[bytes, ...]:
|
||||
return tuple(f"{event}\n\n".encode() for event in events)
|
||||
|
||||
|
||||
def anthropic_sse_chunks_from_response(
|
||||
assembled: ModelResponse, source_chunks: Sequence[object] = (), *, keep_explanations: bool = False
|
||||
) -> tuple[bytes, ...]:
|
||||
|
|
@ -163,8 +167,21 @@ def anthropic_sse_chunks_from_response(
|
|||
A source stream that ended in an upstream ``error`` event ends with that event again, never
|
||||
with a ``message_delta`` or ``message_stop`` the upstream did not send, with its cut-off text
|
||||
block left open unless another block follows it, and with no finished text block when it read
|
||||
no text, since Claude Code does not retry a stream that shows a finished block.
|
||||
no text, since Claude Code does not retry a stream that shows a finished block. When a tool
|
||||
input cannot be parsed the response cannot be re-emitted, and a failed stream is then its error alone.
|
||||
"""
|
||||
error_events: Final = anthropic_sse_error_events(source_chunks)
|
||||
try:
|
||||
return _rebuilt_sse_chunks(assembled, source_chunks, error_events, keep_explanations)
|
||||
except ValueError:
|
||||
if error_events:
|
||||
return _sse_chunks_of_events(error_events)
|
||||
raise
|
||||
|
||||
|
||||
def _rebuilt_sse_chunks(
|
||||
assembled: ModelResponse, source_chunks: Sequence[object], error_events: Sequence[str], keep_explanations: bool
|
||||
) -> tuple[bytes, ...]:
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
LiteLLMAnthropicMessagesAdapter,
|
||||
)
|
||||
|
|
@ -183,7 +200,6 @@ def anthropic_sse_chunks_from_response(
|
|||
unchanged_tool_use_ids: Final = _unchanged_tool_use_ids(
|
||||
source_content or (), anthropic_response.get("content") or ()
|
||||
)
|
||||
error_events: Final = anthropic_sse_error_events(source_chunks)
|
||||
rewritten_chunks: Final = FakeAnthropicMessagesStreamIterator(
|
||||
response=_without_empty_text(anthropic_response) if error_events else anthropic_response,
|
||||
message_delta_fields=_rewritten_message_delta_fields(source_chunks, unchanged_tool_use_ids, keep_explanations),
|
||||
|
|
@ -194,5 +210,5 @@ def anthropic_sse_chunks_from_response(
|
|||
end_events_sent: Final = _MESSAGE_END_EVENT_TYPES & anthropic_sse_event_types(source_chunks)
|
||||
return (
|
||||
*(chunk for chunk in rewritten_chunks if _sent_by_the_failed_upstream(chunk, block_left_open, end_events_sent)),
|
||||
*(f"{event}\n\n".encode() for event in error_events),
|
||||
*_sse_chunks_of_events(error_events),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -313,6 +313,11 @@ def test_line_separators_inside_a_json_string_do_not_make_frames():
|
|||
assert anthropic_sse_error_events((frame,)) == ()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["error\u2028", "error\u0085", "error\x0b", "error\x0c", "error\x1c", " error"])
|
||||
def test_event_name_no_client_reads_as_error_is_no_error_event(name: str):
|
||||
assert anthropic_sse_error_events((f"event: {name}\n\n".encode(),)) == ()
|
||||
|
||||
|
||||
def test_error_event_with_no_data_stays_a_bare_event_line():
|
||||
assert anthropic_sse_error_events((b"event: error\n\n",)) == ("event: error",)
|
||||
|
||||
|
|
|
|||
|
|
@ -2548,6 +2548,58 @@ async def test_apply_to_output_streaming_anthropic_sse_bytes_cut_off_by_an_upstr
|
|||
assert "event: message_stop" not in joined
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_to_output_streaming_anthropic_sse_bytes_with_a_tool_input_that_is_not_json_send_the_upstream_error_alone():
|
||||
guardrail: Final = OPTIONAL_PresidioPIIMasking(
|
||||
mock_testing=True,
|
||||
apply_to_output=True,
|
||||
mock_redacted_text={"text": "<CREDIT_CARD>"},
|
||||
)
|
||||
upstream_error: Final = {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}
|
||||
byte_chunks: Final = (
|
||||
_anthropic_sse(
|
||||
"message_start",
|
||||
{"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}},
|
||||
),
|
||||
_anthropic_sse(
|
||||
"content_block_start",
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
),
|
||||
_anthropic_sse(
|
||||
"content_block_delta",
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "4111 1111 1111 1111"}},
|
||||
),
|
||||
_anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}),
|
||||
_anthropic_sse(
|
||||
"content_block_start",
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 1,
|
||||
"content_block": {"type": "tool_use", "id": "toolu_bad_input", "name": "Bash", "input": {}},
|
||||
},
|
||||
),
|
||||
_anthropic_sse(
|
||||
"content_block_delta",
|
||||
{"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '{"command" "ls"'}},
|
||||
),
|
||||
_anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 1}),
|
||||
_anthropic_sse("error", upstream_error),
|
||||
)
|
||||
|
||||
async def mock_stream() -> AsyncIterator[bytes]:
|
||||
for chunk in byte_chunks:
|
||||
yield chunk
|
||||
|
||||
collected: Final = [
|
||||
chunk
|
||||
async for chunk in guardrail.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), response=mock_stream(), request_data={}
|
||||
)
|
||||
]
|
||||
|
||||
assert b"".join(collected).decode() == f"event: error\ndata: {json.dumps(upstream_error)}\n\n"
|
||||
|
||||
|
||||
def _anthropic_sse_chunks_led_by_a_non_object_data_line(*, line_in_its_own_chunk: bool) -> tuple[bytes, ...]:
|
||||
non_object_line: Final = b"data: 5\n\n"
|
||||
message_start: Final = _anthropic_sse(
|
||||
|
|
|
|||
|
|
@ -1509,6 +1509,28 @@ class TestToolPermissionGuardrailAnthropicMessages:
|
|||
assert events[-1] == self.UPSTREAM_ERROR, "the client must see the upstream error, not a guardrail block"
|
||||
assert {"message_delta", "message_stop"}.isdisjoint(event["type"] for event in events)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finished_tool_call_whose_input_is_not_json_gets_the_upstream_error_alone(self):
|
||||
sse_chunks: Final = self._sse_chunks("Bash")
|
||||
not_json: Final = {"type": "input_json_delta", "partial_json": '{"command" "ls"'}
|
||||
bad_input: Final = json.dumps({"type": "content_block_delta", "index": 1, "delta": not_json})
|
||||
chunks: Final = self._ended_by_upstream_error(
|
||||
[*sse_chunks[:5], f"event: content_block_delta\ndata: {bad_input}\n\n".encode(), sse_chunks[6]]
|
||||
)
|
||||
|
||||
out: Final = await self._drain(self.blocking, chunks)
|
||||
|
||||
assert self._events(out) == [self.UPSTREAM_ERROR]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("name", ["error\u2028", "error\x0b", " error"], ids=["u2028", "vertical-tab", "second-space"])
|
||||
async def test_healthy_stream_with_a_frame_no_client_reads_as_an_error_is_passed_through_verbatim(self, name: str):
|
||||
chunks: Final = [*self._sse_chunks("Bash"), f"event: {name}\n\n".encode()]
|
||||
|
||||
out: Final = await self._drain(self.blocking, chunks)
|
||||
|
||||
assert out == chunks
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_object_data_frame_in_a_stream_ended_by_an_upstream_error_is_left_out(self):
|
||||
sse_chunks: Final = self._sse_chunks("Bash")
|
||||
|
|
|
|||
|
|
@ -410,3 +410,21 @@ def test_masked_stream_that_failed_after_its_message_delta_still_carries_that_me
|
|||
|
||||
assert [event for event, _ in rewritten][-2:] == ["message_delta", "error"]
|
||||
assert next(payload for event, payload in rewritten if event == "message_delta")["delta"]["stop_reason"] == "end_turn"
|
||||
|
||||
|
||||
def test_healthy_stream_with_a_tool_input_that_is_not_json_still_raises():
|
||||
healthy: Final = _upstream_stream({"stop_reason": "tool_use", "stop_sequence": None})
|
||||
tool_block: Final = {"type": "tool_use", "id": "tu_1", "name": "Bash", "input": {}}
|
||||
tool_input: Final = {"type": "input_json_delta", "partial_json": '{"command" "ls"'}
|
||||
upstream: Final = (
|
||||
*healthy[:-2],
|
||||
_frame("content_block_start", {"type": "content_block_start", "index": 1, "content_block": tool_block}),
|
||||
_frame("content_block_delta", {"type": "content_block_delta", "index": 1, "delta": tool_input}),
|
||||
_frame("content_block_stop", {"type": "content_block_stop", "index": 1}),
|
||||
*healthy[-2:],
|
||||
)
|
||||
assembled: Final = assemble_anthropic_sse_stream(upstream)
|
||||
assert assembled is not None
|
||||
|
||||
with pytest.raises(ValueError, match="Failed to parse tool call arguments"):
|
||||
anthropic_sse_chunks_from_response(assembled, upstream)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue