diff --git a/litellm/llms/anthropic/messages/stream_assembly.py b/litellm/llms/anthropic/messages/stream_assembly.py index 3f350947919..dc5bea90ff5 100644 --- a/litellm/llms/anthropic/messages/stream_assembly.py +++ b/litellm/llms/anthropic/messages/stream_assembly.py @@ -578,9 +578,7 @@ def anthropic_sse_event_types(all_chunks: Sequence[object]) -> frozenset[str]: def _is_named_error_event(event: str) -> bool: - return any( - line.startswith("event:") and line[len("event:") :].strip() == "error" for line in split_sse_lines(event) - ) + return ("event", "error") in _sse_fields(event) def _is_error_event(event: str) -> bool: diff --git a/litellm/proxy/guardrails/anthropic_sse.py b/litellm/proxy/guardrails/anthropic_sse.py index 9a3bcfbec30..ae750d27c9d 100644 --- a/litellm/proxy/guardrails/anthropic_sse.py +++ b/litellm/proxy/guardrails/anthropic_sse.py @@ -151,6 +151,10 @@ def _sent_by_the_failed_upstream(chunk: bytes, block_left_open: int | None, end_ return block_left_open is None or anthropic_sse_stopped_block_index(event) != block_left_open +def _sse_chunks_of_events(events: Sequence[str]) -> tuple[bytes, ...]: + return tuple(f"{event}\n\n".encode() for event in events) + + def anthropic_sse_chunks_from_response( assembled: ModelResponse, source_chunks: Sequence[object] = (), *, keep_explanations: bool = False ) -> tuple[bytes, ...]: @@ -163,8 +167,21 @@ def anthropic_sse_chunks_from_response( A source stream that ended in an upstream ``error`` event ends with that event again, never with a ``message_delta`` or ``message_stop`` the upstream did not send, with its cut-off text block left open unless another block follows it, and with no finished text block when it read - no text, since Claude Code does not retry a stream that shows a finished block. + no text, since Claude Code does not retry a stream that shows a finished block. When a tool + input cannot be parsed the response cannot be re-emitted, and a failed stream is then its error alone. """ + error_events: Final = anthropic_sse_error_events(source_chunks) + try: + return _rebuilt_sse_chunks(assembled, source_chunks, error_events, keep_explanations) + except ValueError: + if error_events: + return _sse_chunks_of_events(error_events) + raise + + +def _rebuilt_sse_chunks( + assembled: ModelResponse, source_chunks: Sequence[object], error_events: Sequence[str], keep_explanations: bool +) -> tuple[bytes, ...]: from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -183,7 +200,6 @@ def anthropic_sse_chunks_from_response( unchanged_tool_use_ids: Final = _unchanged_tool_use_ids( source_content or (), anthropic_response.get("content") or () ) - error_events: Final = anthropic_sse_error_events(source_chunks) rewritten_chunks: Final = FakeAnthropicMessagesStreamIterator( response=_without_empty_text(anthropic_response) if error_events else anthropic_response, message_delta_fields=_rewritten_message_delta_fields(source_chunks, unchanged_tool_use_ids, keep_explanations), @@ -194,5 +210,5 @@ def anthropic_sse_chunks_from_response( end_events_sent: Final = _MESSAGE_END_EVENT_TYPES & anthropic_sse_event_types(source_chunks) return ( *(chunk for chunk in rewritten_chunks if _sent_by_the_failed_upstream(chunk, block_left_open, end_events_sent)), - *(f"{event}\n\n".encode() for event in error_events), + *_sse_chunks_of_events(error_events), ) diff --git a/tests/unit/llms/anthropic/messages/test_stream_assembly.py b/tests/unit/llms/anthropic/messages/test_stream_assembly.py index 8ecdfadb37f..ebb5ee966ea 100644 --- a/tests/unit/llms/anthropic/messages/test_stream_assembly.py +++ b/tests/unit/llms/anthropic/messages/test_stream_assembly.py @@ -313,6 +313,11 @@ def test_line_separators_inside_a_json_string_do_not_make_frames(): assert anthropic_sse_error_events((frame,)) == () +@pytest.mark.parametrize("name", ["error\u2028", "error\u0085", "error\x0b", "error\x0c", "error\x1c", " error"]) +def test_event_name_no_client_reads_as_error_is_no_error_event(name: str): + assert anthropic_sse_error_events((f"event: {name}\n\n".encode(),)) == () + + def test_error_event_with_no_data_stays_a_bare_event_line(): assert anthropic_sse_error_events((b"event: error\n\n",)) == ("event: error",) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py index 8ac51eaa001..563b2a0f2b0 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -2548,6 +2548,58 @@ async def test_apply_to_output_streaming_anthropic_sse_bytes_cut_off_by_an_upstr assert "event: message_stop" not in joined +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_with_a_tool_input_that_is_not_json_send_the_upstream_error_alone(): + guardrail: Final = OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": ""}, + ) + upstream_error: Final = {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}} + byte_chunks: Final = ( + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "4111 1111 1111 1111"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse( + "content_block_start", + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "tool_use", "id": "toolu_bad_input", "name": "Bash", "input": {}}, + }, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '{"command" "ls"'}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 1}), + _anthropic_sse("error", upstream_error), + ) + + async def mock_stream() -> AsyncIterator[bytes]: + for chunk in byte_chunks: + yield chunk + + collected: Final = [ + chunk + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), response=mock_stream(), request_data={} + ) + ] + + assert b"".join(collected).decode() == f"event: error\ndata: {json.dumps(upstream_error)}\n\n" + + def _anthropic_sse_chunks_led_by_a_non_object_data_line(*, line_in_its_own_chunk: bool) -> tuple[bytes, ...]: non_object_line: Final = b"data: 5\n\n" message_start: Final = _anthropic_sse( diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py index 4b8923a604d..a1c2cb7496c 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py @@ -1509,6 +1509,28 @@ class TestToolPermissionGuardrailAnthropicMessages: assert events[-1] == self.UPSTREAM_ERROR, "the client must see the upstream error, not a guardrail block" assert {"message_delta", "message_stop"}.isdisjoint(event["type"] for event in events) + @pytest.mark.asyncio + async def test_finished_tool_call_whose_input_is_not_json_gets_the_upstream_error_alone(self): + sse_chunks: Final = self._sse_chunks("Bash") + not_json: Final = {"type": "input_json_delta", "partial_json": '{"command" "ls"'} + bad_input: Final = json.dumps({"type": "content_block_delta", "index": 1, "delta": not_json}) + chunks: Final = self._ended_by_upstream_error( + [*sse_chunks[:5], f"event: content_block_delta\ndata: {bad_input}\n\n".encode(), sse_chunks[6]] + ) + + out: Final = await self._drain(self.blocking, chunks) + + assert self._events(out) == [self.UPSTREAM_ERROR] + + @pytest.mark.asyncio + @pytest.mark.parametrize("name", ["error\u2028", "error\x0b", " error"], ids=["u2028", "vertical-tab", "second-space"]) + async def test_healthy_stream_with_a_frame_no_client_reads_as_an_error_is_passed_through_verbatim(self, name: str): + chunks: Final = [*self._sse_chunks("Bash"), f"event: {name}\n\n".encode()] + + out: Final = await self._drain(self.blocking, chunks) + + assert out == chunks + @pytest.mark.asyncio async def test_non_object_data_frame_in_a_stream_ended_by_an_upstream_error_is_left_out(self): sse_chunks: Final = self._sse_chunks("Bash") diff --git a/tests/unit/proxy/guardrails/test_anthropic_sse.py b/tests/unit/proxy/guardrails/test_anthropic_sse.py index 57fbf5e52cd..9ebfdf5a7d7 100644 --- a/tests/unit/proxy/guardrails/test_anthropic_sse.py +++ b/tests/unit/proxy/guardrails/test_anthropic_sse.py @@ -410,3 +410,21 @@ def test_masked_stream_that_failed_after_its_message_delta_still_carries_that_me assert [event for event, _ in rewritten][-2:] == ["message_delta", "error"] assert next(payload for event, payload in rewritten if event == "message_delta")["delta"]["stop_reason"] == "end_turn" + + +def test_healthy_stream_with_a_tool_input_that_is_not_json_still_raises(): + healthy: Final = _upstream_stream({"stop_reason": "tool_use", "stop_sequence": None}) + tool_block: Final = {"type": "tool_use", "id": "tu_1", "name": "Bash", "input": {}} + tool_input: Final = {"type": "input_json_delta", "partial_json": '{"command" "ls"'} + upstream: Final = ( + *healthy[:-2], + _frame("content_block_start", {"type": "content_block_start", "index": 1, "content_block": tool_block}), + _frame("content_block_delta", {"type": "content_block_delta", "index": 1, "delta": tool_input}), + _frame("content_block_stop", {"type": "content_block_stop", "index": 1}), + *healthy[-2:], + ) + assembled: Final = assemble_anthropic_sse_stream(upstream) + assert assembled is not None + + with pytest.raises(ValueError, match="Failed to parse tool call arguments"): + anthropic_sse_chunks_from_response(assembled, upstream)