mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(guardrails): treat a stream as error-only only when every data frame is an error
A tool call split over data: lines beside one error frame read as error-only, so tool_permission passed it, and an SDK joins those lines and reads the call. The check now joins a frame's data lines and refuses anything but an error.
This commit is contained in:
parent
3183e43dc1
commit
23beda8ede
4 changed files with 52 additions and 6 deletions
|
|
@ -706,6 +706,23 @@ def assemble_anthropic_sse_stream(
|
|||
return assembled
|
||||
|
||||
|
||||
def _sse_frame_data(frame: str) -> str:
|
||||
"""The frame's data as a client reads it, every ``data:`` line joined."""
|
||||
data_lines: Final = (line.strip() for line in split_sse_lines(frame))
|
||||
return "\n".join(line[len("data:") :].strip() for line in data_lines if line.startswith("data:")).strip()
|
||||
|
||||
|
||||
def _is_error_payload(payload: str) -> bool:
|
||||
try:
|
||||
value: Final[object] = json.loads(payload)
|
||||
except ValueError:
|
||||
return False
|
||||
event: Final = as_str_mapping(value)
|
||||
if event is None:
|
||||
return False
|
||||
return event.get("type") == "error" or ("type" not in event and isinstance(event.get("error"), Mapping))
|
||||
|
||||
|
||||
def is_sse_error_stream(all_chunks: Sequence[object]) -> bool:
|
||||
"""Whether the buffered stream carries nothing but error frames.
|
||||
|
||||
|
|
@ -721,9 +738,7 @@ def is_sse_error_stream(all_chunks: Sequence[object]) -> bool:
|
|||
sse_stream: Final = _joined_sse_stream(all_chunks)
|
||||
if sse_stream is None:
|
||||
return False
|
||||
events: Final = tuple(
|
||||
event_data for frame in _sse_frames(sse_stream) if (event_data := _sse_event_data(frame)) is not None
|
||||
)
|
||||
return len(events) > 0 and all(
|
||||
event.get("type") == "error" or isinstance(event.get("error"), Mapping) for event in events
|
||||
payloads: Final = tuple(
|
||||
payload for frame in _sse_frames(sse_stream) if (payload := _sse_frame_data(frame)) not in ("", "[DONE]")
|
||||
)
|
||||
return len(payloads) > 0 and all(_is_error_payload(payload) for payload in payloads)
|
||||
|
|
|
|||
|
|
@ -89,6 +89,10 @@ GOOGLE_STREAM: Final = (b'data: {"candidates": [{"content": {"parts": [{"text":
|
|||
ANTHROPIC_ERROR: Final = _frame("error", {"type": "error", "error": {"type": "overloaded_error", "message": "busy"}})
|
||||
CHAT_ERROR: Final = b'data: {"error": {"message": "blocked by guardrail"}}\n\n'
|
||||
INVALID_UTF8: Final = (b"\xff\xfe data",)
|
||||
TOOL_CALL_OVER_TWO_DATA_LINES: Final = (
|
||||
b'event: content_block_start\ndata: {"type": "content_block_start", "index": 0,\n'
|
||||
b'data: "content_block": {"type": "tool_use", "id": "t1", "name": "Bash", "input": {}}}\n\n'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -221,6 +225,13 @@ def test_assemble_anthropic_sse_stream_returns_none_without_a_message_to_build(c
|
|||
(ANTHROPIC_STREAM, False),
|
||||
(INVALID_UTF8, False),
|
||||
((b"\n\n",), False),
|
||||
((TOOL_CALL_OVER_TWO_DATA_LINES, ANTHROPIC_ERROR), False),
|
||||
((b'data:\ndata: {"type": "message_start"}\n\n', ANTHROPIC_ERROR), False),
|
||||
((b"data: some words\n\n", ANTHROPIC_ERROR), False),
|
||||
((b"data: 5\n\n", CHAT_ERROR), False),
|
||||
((b'data: {"type": "message_start", "error": {}}\n\n', ANTHROPIC_ERROR), False),
|
||||
((b'event: error\ndata: {"type": "error",\ndata: "error": {"type": "overloaded_error"}}\n\n',), True),
|
||||
((b": keepalive\n\n", CHAT_ERROR, b"data: [DONE]\n\n"), True),
|
||||
],
|
||||
ids=[
|
||||
"anthropic-error",
|
||||
|
|
@ -231,6 +242,13 @@ def test_assemble_anthropic_sse_stream_returns_none_without_a_message_to_build(c
|
|||
"normal-stream",
|
||||
"undecodable",
|
||||
"no-events",
|
||||
"tool-call-over-two-data-lines",
|
||||
"message-start-behind-an-empty-data-line",
|
||||
"words-beside-an-error",
|
||||
"non-object-beside-an-error",
|
||||
"message-start-with-an-error-field",
|
||||
"error-over-two-data-lines",
|
||||
"keepalive-and-done-marker",
|
||||
],
|
||||
)
|
||||
def test_is_sse_error_stream_is_true_only_for_frames_that_are_all_errors(chunks: tuple[object, ...], expected: bool):
|
||||
|
|
|
|||
|
|
@ -67,6 +67,8 @@ def _make_mock_session_iterator(json_response, status=200, content_type="applica
|
|||
pass
|
||||
|
||||
class MockSession:
|
||||
closed = False
|
||||
|
||||
def post(self, *args, **kwargs):
|
||||
self.last_kwargs = kwargs
|
||||
return MockResponse()
|
||||
|
|
@ -2617,7 +2619,8 @@ async def test_apply_to_output_streaming_anthropic_sse_bytes_led_by_a_non_object
|
|||
user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), response=mock_stream(), request_data={}
|
||||
)
|
||||
|
||||
with patch.object(guardrail, "_get_session_iterator", _make_mock_session_iterator(analyzer_hit)):
|
||||
async with _make_mock_session_iterator(analyzer_hit)() as aiohttp_session:
|
||||
guardrail._http_session = aiohttp_session
|
||||
with pytest.raises(BlockedPiiEntityError):
|
||||
await anext(stream)
|
||||
|
||||
|
|
|
|||
|
|
@ -1570,6 +1570,16 @@ class TestToolPermissionGuardrailAnthropicMessages:
|
|||
with pytest.raises(GuardrailRaisedException, match="could not be verified"):
|
||||
await self._drain(self.blocking, chunks)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_denied_tool_use_split_over_data_lines_beside_an_upstream_error_is_blocked(self):
|
||||
def over_two_data_lines(frame: bytes) -> bytes:
|
||||
return frame.replace(b", ", b",\ndata: ", 1)
|
||||
|
||||
chunks: Final = self._ended_by_upstream_error([over_two_data_lines(frame) for frame in self._sse_chunks("Read")])
|
||||
|
||||
with pytest.raises(GuardrailRaisedException, match="could not be verified"):
|
||||
await self._drain(self.blocking, chunks)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_denied_tool_use_delivered_before_an_upstream_error_is_still_blocked(self):
|
||||
chunks: Final = self._ended_by_upstream_error(self._sse_chunks("Read")[:7])
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue