From ac600f3631264bd4b73ff0209b07214dec24152e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:32:04 -0700 Subject: [PATCH] fix(guardrails): refuse a first frame whose type is a list or dict in Presidio Presidio passed PII at 200 where main answered 500 when the first read was a JSON object with a list or dict type, or a JSON array with a character cut at the end of the read. has_unreadable_json_event now reads both as data no reader of these streams can take, so the first-frame check refuses them. --- .../llms/anthropic/messages/stream_assembly.py | 16 ++++++++++------ litellm/proxy/guardrails/anthropic_sse.py | 2 +- .../proxy/guardrails/guardrail_hooks/presidio.py | 6 +++--- .../guardrails/guardrail_hooks/test_presidio.py | 5 +++++ 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/litellm/llms/anthropic/messages/stream_assembly.py b/litellm/llms/anthropic/messages/stream_assembly.py index 0f9aec64083..b8b676aa459 100644 --- a/litellm/llms/anthropic/messages/stream_assembly.py +++ b/litellm/llms/anthropic/messages/stream_assembly.py @@ -514,19 +514,23 @@ def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool: return any(_is_anthropic_event_type(event.get("type")) for event in _parsed_sse_events(sse_stream)) -def _is_json_value_other_than_object_or_null(payload: str) -> bool: +def _is_json_no_reader_can_take(payload: str) -> bool: try: - return not isinstance(json.loads(payload), (dict, type(None))) + value: Final[object] = json.loads(payload) except ValueError: return False + if value is None: + return False + event: Final = as_str_mapping(value) + return event is None or isinstance(event.get("type"), (list, dict)) -def has_non_object_json_event(all_chunks: Sequence[object]) -> bool: +def has_unreadable_json_event(all_chunks: Sequence[object]) -> bool: """Whether a frame's data is JSON that no reader of these streams can take as an event.""" - return any( - _is_json_value_other_than_object_or_null(_sse_data_payload(event) or "") - for event in _sse_frames(_joined_sse_bytes(all_chunks).decode("utf-8", errors="replace")) + sse_stream: Final = codecs.getincrementaldecoder("utf-8")(errors="replace").decode( + _joined_sse_bytes(all_chunks), final=False ) + return any(_is_json_no_reader_can_take(_sse_data_payload(event) or "") for event in _sse_frames(sse_stream)) def anthropic_sse_event_type(event: str) -> object: diff --git a/litellm/proxy/guardrails/anthropic_sse.py b/litellm/proxy/guardrails/anthropic_sse.py index 8d7c41b950a..5761168763c 100644 --- a/litellm/proxy/guardrails/anthropic_sse.py +++ b/litellm/proxy/guardrails/anthropic_sse.py @@ -28,7 +28,7 @@ from litellm.llms.anthropic.messages.stream_assembly import ( assemble_anthropic_sse_stream as assemble_anthropic_sse_stream, # noqa: PLC0414 # public re-export ) from litellm.llms.anthropic.messages.stream_assembly import ( - has_non_object_json_event as has_non_object_json_event, # noqa: PLC0414 # public re-export + has_unreadable_json_event as has_unreadable_json_event, # noqa: PLC0414 # public re-export ) from litellm.llms.anthropic.messages.stream_assembly import ( is_anthropic_sse_stream as is_anthropic_sse_stream, # noqa: PLC0414 # public re-export diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 0eb4dd8b918..458528e7e13 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -48,7 +48,7 @@ from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames from litellm.proxy.guardrails.anthropic_sse import ( anthropic_sse_chunks_from_response, assemble_anthropic_sse_stream, - has_non_object_json_event, + has_unreadable_json_event, is_anthropic_sse_stream, model_response_text, ) @@ -1580,7 +1580,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): def _is_anthropic_first_frame(self, chunk: bytes) -> bool: if is_anthropic_sse_stream((chunk,)): return True - if has_non_object_json_event((chunk,)): + if has_unreadable_json_event((chunk,)): raise self._unreadable_stream_refusal() return False @@ -1598,7 +1598,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): chunks: Final = (first_chunk, *rest_chunks) assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True, through_upstream_error=True) if assembled is None: - if has_non_object_json_event(chunks): + if has_unreadable_json_event(chunks): raise self._unreadable_stream_refusal() verbose_proxy_logger.warning( "Presidio apply_to_output: raw SSE stream could not be assembled into a response. " diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py index fc46209c539..3368be45292 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -2628,6 +2628,10 @@ STREAMS_WITH_A_NON_OBJECT_LINE_AND_NO_ANTHROPIC_FRAME: Final = { "then-broken-json": (b"data: 5\n\n", b'data: {"foo"\n\n'), "two-data-lines-in-one-frame": (b'data: 5\ndata: {"type": "message_start"}\n\n',), "past-the-byte-cap": (b"data: 5\n\n" * 12_000,), + "list-type": (b'data: {"type": []}\n\n', b'data: {"text": "4111"}\n\n'), + "dict-type": (b'data: {"type": {"a": 1}}\n\n', b'data: {"text": "4111"}\n\n'), + "error-frame-with-a-list-type": (b'event: error\ndata: {"type": []}\n\n', b'data: {"text": "4111"}\n\n'), + "cut-character-at-the-end": (b'data: [5]\xe2',), } @@ -2714,6 +2718,7 @@ async def test_apply_to_output_streaming_unreadable_stream_with_a_non_object_lin STREAMS_OF_UNKNOWN_SHAPE_MAIN_PASSED: Final = { "null-line": (b"data: null\n\n", b'data: {"foo": 1}\n\n'), + "number-type": (b'data: {"type": 5}\n\n', b'data: {"foo": 1}\n\n'), "line-that-is-not-json": (b"data: not json\n\n", b'data: {"foo": 1}\n\n'), "done-sentinel": (b"data: [DONE]\n\n", b'data: {"foo": 1}\n\n'), "non-object-line-after-the-shape-is-decided": (b'data: {"foo": 1}\n\n', b"data: 5\n\n"),