mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(guardrails): refuse what Presidio cannot read past a non-object line and keep non-string error types out of the frozenset
Presidio now refuses a stream assembly cannot read when it holds a non-object data line, instead of relaying it unmasked. Non-string event types no longer crash a rewrite, and a CRLF stream that opens with a typed error is no longer read as one error, so its tool calls are still checked.
This commit is contained in:
parent
8cd41755b6
commit
c5a785f800
5 changed files with 107 additions and 17 deletions
|
|
@ -428,18 +428,25 @@ _ANTHROPIC_EVENT_TYPES: Final = frozenset(
|
|||
)
|
||||
|
||||
|
||||
def _is_anthropic_event_type(event_type: object) -> bool:
|
||||
return isinstance(event_type, str) and event_type in _ANTHROPIC_EVENT_TYPES
|
||||
|
||||
|
||||
def is_raw_sse_stream(all_chunks: Sequence[object]) -> bool:
|
||||
return any(isinstance(chunk, (str, bytes)) for chunk in all_chunks)
|
||||
|
||||
|
||||
def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None:
|
||||
raw: Final = b"".join(
|
||||
def _joined_sse_bytes(all_chunks: Sequence[object]) -> bytes:
|
||||
return b"".join(
|
||||
chunk if isinstance(chunk, bytes) else chunk.encode("utf-8")
|
||||
for chunk in all_chunks
|
||||
if isinstance(chunk, (str, bytes))
|
||||
)
|
||||
|
||||
|
||||
def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None:
|
||||
try:
|
||||
return codecs.getincrementaldecoder("utf-8")().decode(raw, final=False)
|
||||
return codecs.getincrementaldecoder("utf-8")().decode(_joined_sse_bytes(all_chunks), final=False)
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
|
||||
|
|
@ -448,6 +455,10 @@ def _sse_event_data(event: str) -> Mapping[str, object] | None:
|
|||
return as_str_mapping(extract_sse_data(event))
|
||||
|
||||
|
||||
def _sse_frames(sse_stream: str) -> Sequence[str]:
|
||||
return split_sse_chunk_into_events("\n".join(split_sse_lines(sse_stream)))
|
||||
|
||||
|
||||
def _parsed_sse_events(sse_stream: str) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(
|
||||
event_data
|
||||
|
|
@ -500,7 +511,7 @@ def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool:
|
|||
sse_stream: Final = _joined_sse_stream(all_chunks)
|
||||
if sse_stream is None:
|
||||
return False
|
||||
return any(event.get("type") in _ANTHROPIC_EVENT_TYPES for event in _parsed_sse_events(sse_stream))
|
||||
return any(_is_anthropic_event_type(event.get("type")) for event in _parsed_sse_events(sse_stream))
|
||||
|
||||
|
||||
def _is_json_value_other_than_object_or_null(payload: str) -> bool:
|
||||
|
|
@ -514,7 +525,7 @@ def has_non_object_json_event(all_chunks: Sequence[object]) -> bool:
|
|||
"""Whether a frame's data is JSON that no reader of these streams can take as an event."""
|
||||
return any(
|
||||
_is_json_value_other_than_object_or_null(_sse_data_payload(event) or "")
|
||||
for event in split_sse_chunk_into_events(_joined_sse_stream(all_chunks) or "")
|
||||
for event in _sse_frames(_joined_sse_bytes(all_chunks).decode("utf-8", errors="replace"))
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -522,9 +533,10 @@ def anthropic_sse_event_type(event: str) -> object:
|
|||
return next((event_data.get("type") for event_data in _parsed_sse_events(event)), None)
|
||||
|
||||
|
||||
def anthropic_sse_event_types(all_chunks: Sequence[object]) -> frozenset[object]:
|
||||
"""The ``type`` of every event the stream sent."""
|
||||
return frozenset(event.get("type") for event in _parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
|
||||
def anthropic_sse_event_types(all_chunks: Sequence[object]) -> frozenset[str]:
|
||||
"""The string ``type`` of every event the stream sent."""
|
||||
event_types: Final = (event.get("type") for event in _parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
|
||||
return frozenset(event_type for event_type in event_types if isinstance(event_type, str))
|
||||
|
||||
|
||||
def _is_named_error_event(event: str) -> bool:
|
||||
|
|
@ -535,8 +547,7 @@ def _is_named_error_event(event: str) -> bool:
|
|||
|
||||
def _is_error_event(event: str) -> bool:
|
||||
event_type: Final = anthropic_sse_event_type(event)
|
||||
data_is_another_event: Final = isinstance(event_type, str) and event_type in _ANTHROPIC_EVENT_TYPES
|
||||
return event_type == "error" or (_is_named_error_event(event) and not data_is_another_event)
|
||||
return event_type == "error" or (_is_named_error_event(event) and not _is_anthropic_event_type(event_type))
|
||||
|
||||
|
||||
def _error_events(sse_stream: str) -> tuple[str, ...]:
|
||||
|
|
@ -706,7 +717,9 @@ def is_sse_error_stream(all_chunks: Sequence[object]) -> bool:
|
|||
sse_stream: Final = _joined_sse_stream(all_chunks)
|
||||
if sse_stream is None:
|
||||
return False
|
||||
events: Final = _parsed_sse_events(sse_stream)
|
||||
events: Final = tuple(
|
||||
event_data for frame in _sse_frames(sse_stream) if (event_data := _sse_event_data(frame)) is not None
|
||||
)
|
||||
return len(events) > 0 and all(
|
||||
event.get("type") == "error" or isinstance(event.get("error"), Mapping) for event in events
|
||||
)
|
||||
|
|
|
|||
|
|
@ -151,7 +151,7 @@ def _text_block_left_open(rewritten_chunks: Sequence[bytes], source_chunks: Sequ
|
|||
return last_index if rewritten_blocks[last_index].get("type") == "text" else None
|
||||
|
||||
|
||||
def _sent_by_the_failed_upstream(chunk: bytes, block_left_open: int | None, end_events_sent: frozenset[object]) -> bool:
|
||||
def _sent_by_the_failed_upstream(chunk: bytes, block_left_open: int | None, end_events_sent: frozenset[str]) -> bool:
|
||||
event: Final = chunk.decode()
|
||||
event_type: Final = anthropic_sse_event_type(event)
|
||||
if event_type in _MESSAGE_END_EVENT_TYPES:
|
||||
|
|
|
|||
|
|
@ -1581,13 +1581,16 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
|
|||
if is_anthropic_sse_stream((chunk,)):
|
||||
return True
|
||||
if has_non_object_json_event((chunk,)):
|
||||
raise GuardrailRaisedException(
|
||||
guardrail_name=self.guardrail_name,
|
||||
message="Presidio could not read the upstream stream; cannot verify PII when PII protection is configured",
|
||||
should_wrap_with_default_message=False,
|
||||
)
|
||||
raise self._unreadable_stream_refusal()
|
||||
return False
|
||||
|
||||
def _unreadable_stream_refusal(self) -> GuardrailRaisedException:
|
||||
return GuardrailRaisedException(
|
||||
guardrail_name=self.guardrail_name,
|
||||
message="Presidio could not read the upstream stream; cannot verify PII when PII protection is configured",
|
||||
should_wrap_with_default_message=False,
|
||||
)
|
||||
|
||||
async def _mask_anthropic_sse_stream(
|
||||
self, first_chunk: bytes, rest: AsyncIterator[object], request_data: dict
|
||||
) -> tuple[object, ...]:
|
||||
|
|
@ -1595,6 +1598,8 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
|
|||
chunks: Final = (first_chunk, *rest_chunks)
|
||||
assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True, through_upstream_error=True)
|
||||
if assembled is None:
|
||||
if has_non_object_json_event(chunks):
|
||||
raise self._unreadable_stream_refusal()
|
||||
verbose_proxy_logger.warning(
|
||||
"Presidio apply_to_output: raw SSE stream could not be assembled into a response. "
|
||||
"Output PII masking was skipped for this response."
|
||||
|
|
|
|||
|
|
@ -2658,6 +2658,60 @@ async def test_apply_to_output_streaming_non_object_line_with_no_anthropic_frame
|
|||
await anext(stream)
|
||||
|
||||
|
||||
def _anthropic_text_frames_without_a_message_start(text: str) -> tuple[bytes, ...]:
|
||||
return (
|
||||
_anthropic_sse(
|
||||
"content_block_start",
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
),
|
||||
_anthropic_sse(
|
||||
"content_block_delta",
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}},
|
||||
),
|
||||
_anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}),
|
||||
)
|
||||
|
||||
|
||||
_PING_FRAME: Final = _anthropic_sse("ping", {"type": "ping"})
|
||||
_TEXT_FRAMES: Final = _anthropic_text_frames_without_a_message_start("4111 1111 1111 1111")
|
||||
_MESSAGE_START_BESIDE_A_NON_OBJECT_LINE: Final = _anthropic_sse(
|
||||
"message_start", {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": []}}
|
||||
).replace(b"data: {", b"data: 5\ndata: {", 1)
|
||||
STREAMS_ASSEMBLY_CANNOT_READ_WITH_A_NON_OBJECT_LINE: Final = {
|
||||
"beside-the-message-start": (b"".join((_MESSAGE_START_BESIDE_A_NON_OBJECT_LINE, *_TEXT_FRAMES)),),
|
||||
"after-a-ping": (_PING_FRAME, b"data: 5\n\n", *_TEXT_FRAMES),
|
||||
"after-a-ping-in-carriage-return-frames": (_PING_FRAME, b'data: {"foo": 1}\r\rdata: 5\r\r', *_TEXT_FRAMES),
|
||||
"beside-undecodable-bytes": (b"data: 5\n\n" + b'data: {"text": "\xff"}\n\n' + b"".join(_TEXT_FRAMES),),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"byte_chunks",
|
||||
STREAMS_ASSEMBLY_CANNOT_READ_WITH_A_NON_OBJECT_LINE.values(),
|
||||
ids=STREAMS_ASSEMBLY_CANNOT_READ_WITH_A_NON_OBJECT_LINE,
|
||||
)
|
||||
async def test_apply_to_output_streaming_unreadable_stream_with_a_non_object_line_is_refused(
|
||||
byte_chunks: tuple[bytes, ...],
|
||||
):
|
||||
guardrail: Final = OPTIONAL_PresidioPIIMasking(
|
||||
mock_testing=True,
|
||||
apply_to_output=True,
|
||||
mock_redacted_text={"text": "<CREDIT_CARD>"},
|
||||
)
|
||||
|
||||
async def mock_stream() -> AsyncIterator[bytes]:
|
||||
for chunk in byte_chunks:
|
||||
yield chunk
|
||||
|
||||
stream: Final = guardrail.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), response=mock_stream(), request_data={}
|
||||
)
|
||||
|
||||
with pytest.raises(GuardrailRaisedException, match="cannot verify PII"):
|
||||
await anext(stream)
|
||||
|
||||
|
||||
STREAMS_OF_UNKNOWN_SHAPE_MAIN_PASSED: Final = {
|
||||
"null-line": (b"data: null\n\n", b'data: {"foo": 1}\n\n'),
|
||||
"line-that-is-not-json": (b"data: not json\n\n", b'data: {"foo": 1}\n\n'),
|
||||
|
|
|
|||
|
|
@ -1552,6 +1552,24 @@ class TestToolPermissionGuardrailAnthropicMessages:
|
|||
|
||||
assert b"".join(out) == b"".join(chunks)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("error_type", [[], {"a": 1}], ids=["list", "dict"])
|
||||
async def test_rewrite_keeps_an_error_frame_whose_type_is_not_a_string(self, error_type: object):
|
||||
error_frame: Final = f"event: error\ndata: {json.dumps({'type': error_type})}\n\n".encode()
|
||||
|
||||
out: Final = await self._drain(self.rewriting, [*self._sse_chunks("Read")[:7], error_frame])
|
||||
|
||||
assert b"".join(out).endswith(error_frame)
|
||||
assert self._tool_use_blocks(self._events(out)) == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crlf_stream_that_opens_with_an_error_is_still_blocked(self):
|
||||
frames: Final = self._ended_by_upstream_error([])
|
||||
chunks: Final = [frame.replace(b"\n", b"\r\n") for frame in (*frames, *self._sse_chunks("Read"))]
|
||||
|
||||
with pytest.raises(GuardrailRaisedException, match="could not be verified"):
|
||||
await self._drain(self.blocking, chunks)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_denied_tool_use_delivered_before_an_upstream_error_is_still_blocked(self):
|
||||
chunks: Final = self._ended_by_upstream_error(self._sse_chunks("Read")[:7])
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue