From c5a785f800f0f26d8dfa85be1c7b9b960ffd4755 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 13:22:42 -0700 Subject: [PATCH] fix(guardrails): refuse what Presidio cannot read past a non-object line and keep non-string error types out of the frozenset Presidio now refuses a stream assembly cannot read when it holds a non-object data line, instead of relaying it unmasked. Non-string event types no longer crash a rewrite, and a CRLF stream that opens with a typed error is no longer read as one error, so its tool calls are still checked. --- .../anthropic/messages/stream_assembly.py | 35 ++++++++---- litellm/proxy/guardrails/anthropic_sse.py | 2 +- .../guardrails/guardrail_hooks/presidio.py | 15 ++++-- .../guardrail_hooks/test_presidio.py | 54 +++++++++++++++++++ .../guardrail_hooks/test_tool_permission.py | 18 +++++++ 5 files changed, 107 insertions(+), 17 deletions(-) diff --git a/litellm/llms/anthropic/messages/stream_assembly.py b/litellm/llms/anthropic/messages/stream_assembly.py index fefb4a84465..0f9aec64083 100644 --- a/litellm/llms/anthropic/messages/stream_assembly.py +++ b/litellm/llms/anthropic/messages/stream_assembly.py @@ -428,18 +428,25 @@ _ANTHROPIC_EVENT_TYPES: Final = frozenset( ) +def _is_anthropic_event_type(event_type: object) -> bool: + return isinstance(event_type, str) and event_type in _ANTHROPIC_EVENT_TYPES + + def is_raw_sse_stream(all_chunks: Sequence[object]) -> bool: return any(isinstance(chunk, (str, bytes)) for chunk in all_chunks) -def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None: - raw: Final = b"".join( +def _joined_sse_bytes(all_chunks: Sequence[object]) -> bytes: + return b"".join( chunk if isinstance(chunk, bytes) else chunk.encode("utf-8") for chunk in all_chunks if isinstance(chunk, (str, bytes)) ) + + +def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None: try: - return codecs.getincrementaldecoder("utf-8")().decode(raw, final=False) + return codecs.getincrementaldecoder("utf-8")().decode(_joined_sse_bytes(all_chunks), final=False) except UnicodeDecodeError: return None @@ -448,6 +455,10 @@ def _sse_event_data(event: str) -> Mapping[str, object] | None: return as_str_mapping(extract_sse_data(event)) +def _sse_frames(sse_stream: str) -> Sequence[str]: + return split_sse_chunk_into_events("\n".join(split_sse_lines(sse_stream))) + + def _parsed_sse_events(sse_stream: str) -> tuple[Mapping[str, object], ...]: return tuple( event_data @@ -500,7 +511,7 @@ def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool: sse_stream: Final = _joined_sse_stream(all_chunks) if sse_stream is None: return False - return any(event.get("type") in _ANTHROPIC_EVENT_TYPES for event in _parsed_sse_events(sse_stream)) + return any(_is_anthropic_event_type(event.get("type")) for event in _parsed_sse_events(sse_stream)) def _is_json_value_other_than_object_or_null(payload: str) -> bool: @@ -514,7 +525,7 @@ def has_non_object_json_event(all_chunks: Sequence[object]) -> bool: """Whether a frame's data is JSON that no reader of these streams can take as an event.""" return any( _is_json_value_other_than_object_or_null(_sse_data_payload(event) or "") - for event in split_sse_chunk_into_events(_joined_sse_stream(all_chunks) or "") + for event in _sse_frames(_joined_sse_bytes(all_chunks).decode("utf-8", errors="replace")) ) @@ -522,9 +533,10 @@ def anthropic_sse_event_type(event: str) -> object: return next((event_data.get("type") for event_data in _parsed_sse_events(event)), None) -def anthropic_sse_event_types(all_chunks: Sequence[object]) -> frozenset[object]: - """The ``type`` of every event the stream sent.""" - return frozenset(event.get("type") for event in _parsed_sse_events(_joined_sse_stream(all_chunks) or "")) +def anthropic_sse_event_types(all_chunks: Sequence[object]) -> frozenset[str]: + """The string ``type`` of every event the stream sent.""" + event_types: Final = (event.get("type") for event in _parsed_sse_events(_joined_sse_stream(all_chunks) or "")) + return frozenset(event_type for event_type in event_types if isinstance(event_type, str)) def _is_named_error_event(event: str) -> bool: @@ -535,8 +547,7 @@ def _is_named_error_event(event: str) -> bool: def _is_error_event(event: str) -> bool: event_type: Final = anthropic_sse_event_type(event) - data_is_another_event: Final = isinstance(event_type, str) and event_type in _ANTHROPIC_EVENT_TYPES - return event_type == "error" or (_is_named_error_event(event) and not data_is_another_event) + return event_type == "error" or (_is_named_error_event(event) and not _is_anthropic_event_type(event_type)) def _error_events(sse_stream: str) -> tuple[str, ...]: @@ -706,7 +717,9 @@ def is_sse_error_stream(all_chunks: Sequence[object]) -> bool: sse_stream: Final = _joined_sse_stream(all_chunks) if sse_stream is None: return False - events: Final = _parsed_sse_events(sse_stream) + events: Final = tuple( + event_data for frame in _sse_frames(sse_stream) if (event_data := _sse_event_data(frame)) is not None + ) return len(events) > 0 and all( event.get("type") == "error" or isinstance(event.get("error"), Mapping) for event in events ) diff --git a/litellm/proxy/guardrails/anthropic_sse.py b/litellm/proxy/guardrails/anthropic_sse.py index 890ef38a930..8d7c41b950a 100644 --- a/litellm/proxy/guardrails/anthropic_sse.py +++ b/litellm/proxy/guardrails/anthropic_sse.py @@ -151,7 +151,7 @@ def _text_block_left_open(rewritten_chunks: Sequence[bytes], source_chunks: Sequ return last_index if rewritten_blocks[last_index].get("type") == "text" else None -def _sent_by_the_failed_upstream(chunk: bytes, block_left_open: int | None, end_events_sent: frozenset[object]) -> bool: +def _sent_by_the_failed_upstream(chunk: bytes, block_left_open: int | None, end_events_sent: frozenset[str]) -> bool: event: Final = chunk.decode() event_type: Final = anthropic_sse_event_type(event) if event_type in _MESSAGE_END_EVENT_TYPES: diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 0be55c9d8e4..0eb4dd8b918 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -1581,13 +1581,16 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if is_anthropic_sse_stream((chunk,)): return True if has_non_object_json_event((chunk,)): - raise GuardrailRaisedException( - guardrail_name=self.guardrail_name, - message="Presidio could not read the upstream stream; cannot verify PII when PII protection is configured", - should_wrap_with_default_message=False, - ) + raise self._unreadable_stream_refusal() return False + def _unreadable_stream_refusal(self) -> GuardrailRaisedException: + return GuardrailRaisedException( + guardrail_name=self.guardrail_name, + message="Presidio could not read the upstream stream; cannot verify PII when PII protection is configured", + should_wrap_with_default_message=False, + ) + async def _mask_anthropic_sse_stream( self, first_chunk: bytes, rest: AsyncIterator[object], request_data: dict ) -> tuple[object, ...]: @@ -1595,6 +1598,8 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): chunks: Final = (first_chunk, *rest_chunks) assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True, through_upstream_error=True) if assembled is None: + if has_non_object_json_event(chunks): + raise self._unreadable_stream_refusal() verbose_proxy_logger.warning( "Presidio apply_to_output: raw SSE stream could not be assembled into a response. " "Output PII masking was skipped for this response." diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py index 5ebb0ac4ced..fc46209c539 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -2658,6 +2658,60 @@ async def test_apply_to_output_streaming_non_object_line_with_no_anthropic_frame await anext(stream) +def _anthropic_text_frames_without_a_message_start(text: str) -> tuple[bytes, ...]: + return ( + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + ) + + +_PING_FRAME: Final = _anthropic_sse("ping", {"type": "ping"}) +_TEXT_FRAMES: Final = _anthropic_text_frames_without_a_message_start("4111 1111 1111 1111") +_MESSAGE_START_BESIDE_A_NON_OBJECT_LINE: Final = _anthropic_sse( + "message_start", {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": []}} +).replace(b"data: {", b"data: 5\ndata: {", 1) +STREAMS_ASSEMBLY_CANNOT_READ_WITH_A_NON_OBJECT_LINE: Final = { + "beside-the-message-start": (b"".join((_MESSAGE_START_BESIDE_A_NON_OBJECT_LINE, *_TEXT_FRAMES)),), + "after-a-ping": (_PING_FRAME, b"data: 5\n\n", *_TEXT_FRAMES), + "after-a-ping-in-carriage-return-frames": (_PING_FRAME, b'data: {"foo": 1}\r\rdata: 5\r\r', *_TEXT_FRAMES), + "beside-undecodable-bytes": (b"data: 5\n\n" + b'data: {"text": "\xff"}\n\n' + b"".join(_TEXT_FRAMES),), +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "byte_chunks", + STREAMS_ASSEMBLY_CANNOT_READ_WITH_A_NON_OBJECT_LINE.values(), + ids=STREAMS_ASSEMBLY_CANNOT_READ_WITH_A_NON_OBJECT_LINE, +) +async def test_apply_to_output_streaming_unreadable_stream_with_a_non_object_line_is_refused( + byte_chunks: tuple[bytes, ...], +): + guardrail: Final = OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": ""}, + ) + + async def mock_stream() -> AsyncIterator[bytes]: + for chunk in byte_chunks: + yield chunk + + stream: Final = guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), response=mock_stream(), request_data={} + ) + + with pytest.raises(GuardrailRaisedException, match="cannot verify PII"): + await anext(stream) + + STREAMS_OF_UNKNOWN_SHAPE_MAIN_PASSED: Final = { "null-line": (b"data: null\n\n", b'data: {"foo": 1}\n\n'), "line-that-is-not-json": (b"data: not json\n\n", b'data: {"foo": 1}\n\n'), diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py index f64a0c5a25e..43dd1c6ce0f 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py @@ -1552,6 +1552,24 @@ class TestToolPermissionGuardrailAnthropicMessages: assert b"".join(out) == b"".join(chunks) + @pytest.mark.asyncio + @pytest.mark.parametrize("error_type", [[], {"a": 1}], ids=["list", "dict"]) + async def test_rewrite_keeps_an_error_frame_whose_type_is_not_a_string(self, error_type: object): + error_frame: Final = f"event: error\ndata: {json.dumps({'type': error_type})}\n\n".encode() + + out: Final = await self._drain(self.rewriting, [*self._sse_chunks("Read")[:7], error_frame]) + + assert b"".join(out).endswith(error_frame) + assert self._tool_use_blocks(self._events(out)) == [] + + @pytest.mark.asyncio + async def test_crlf_stream_that_opens_with_an_error_is_still_blocked(self): + frames: Final = self._ended_by_upstream_error([]) + chunks: Final = [frame.replace(b"\n", b"\r\n") for frame in (*frames, *self._sse_chunks("Read"))] + + with pytest.raises(GuardrailRaisedException, match="could not be verified"): + await self._drain(self.blocking, chunks) + @pytest.mark.asyncio async def test_denied_tool_use_delivered_before_an_upstream_error_is_still_blocked(self): chunks: Final = self._ended_by_upstream_error(self._sse_chunks("Read")[:7])