mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(guardrails): answer ThirdLaw streamed Responses and Messages blocks with their block status
A block decided at the end of a buffered /v1/responses stream went out as a Responses error event with a hardcoded 400, and create_response answered with that code. On /v1/messages the refusal was raw error-frame bytes, which carry no status, so the client got a 200 and a failed stream instead of the 403 ThirdLaw rendered _stream_error_items now takes the resolved block status. The Responses event carries it as its code, and the Messages refusal is an AnthropicErrorSseFrame, the frame create_response already answers as a JSON error with its own status. Its error type now follows the status (permission_error for a 403) rather than the non-standard guardrail_error. The Messages restamper used to re-encode every str chunk to bytes while armed, which dropped the frame's status, so it passes an error frame through untouched. Refusals without a verdict keep 400, as before Part of TL-2934
This commit is contained in:
parent
72232ea88a
commit
93c5cebf9d
4 changed files with 125 additions and 25 deletions
|
|
@ -14,6 +14,7 @@ from typing import Final
|
|||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines_keepends
|
||||
|
||||
_MESSAGE_START_EVENT: Final = "message_start"
|
||||
|
|
@ -108,12 +109,17 @@ class AnthropicStreamModelRestamper:
|
|||
|
||||
def __init__(self, requested_model: str) -> None:
|
||||
self._requested_model: Final = requested_model
|
||||
self._held = b""
|
||||
self._held: bytes = b""
|
||||
self._armed = True
|
||||
|
||||
def process(self, chunk: object) -> object:
|
||||
if not self._armed:
|
||||
return chunk
|
||||
if isinstance(chunk, AnthropicErrorSseFrame) and not self._held:
|
||||
# An error event carries no model, and re-encoding it would drop the status the proxy
|
||||
# answers with when the stream fails before its first byte
|
||||
self._armed = False
|
||||
return chunk
|
||||
if isinstance(chunk, (bytes, bytearray)):
|
||||
return self._process_encoded(bytes(chunk))
|
||||
if isinstance(chunk, str):
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from pydantic import BaseModel, TypeAdapter, ValidationError
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._version import version as litellm_version
|
||||
from litellm.anthropic_interface.exceptions import anthropic_error_sse_frame
|
||||
from litellm.caching import DualCache
|
||||
from litellm.exceptions import GuardrailRaisedException
|
||||
from litellm.exceptions import Timeout as LitellmTimeout
|
||||
|
|
@ -37,7 +38,6 @@ from litellm.proxy._types import (
|
|||
)
|
||||
from litellm.proxy.guardrails.anthropic_sse import (
|
||||
anthropic_sse_chunks_from_body,
|
||||
anthropic_sse_error_frames,
|
||||
assemble_anthropic_sse_body,
|
||||
sse_stream_text,
|
||||
)
|
||||
|
|
@ -1028,23 +1028,25 @@ class ThirdlawGuardrail(CustomGuardrail):
|
|||
|
||||
@staticmethod
|
||||
def _stream_error_items(
|
||||
message: str, surface: StreamSurface, collected: Sequence[object]
|
||||
message: str, surface: StreamSurface, collected: Sequence[object], *, status_code: int
|
||||
) -> Sequence[object] | None:
|
||||
"""Terminal stream items framing a refusal in this surface's wire format.
|
||||
|
||||
``None`` means the surface cannot frame its own error, so the caller raises instead and
|
||||
lets the proxy's data generator serialize it.
|
||||
lets the proxy's data generator serialize it. ``status_code`` travels inside the frame:
|
||||
when the refusal is the stream's first item the proxy answers with it as a JSON error,
|
||||
and after a keepalive ping has flushed a 200 it is only the frame's body.
|
||||
"""
|
||||
match surface:
|
||||
case StreamSurface.ANTHROPIC_MESSAGES:
|
||||
return anthropic_sse_error_frames(message)
|
||||
return (anthropic_error_sse_frame(status_code, message),)
|
||||
case StreamSurface.RESPONSES:
|
||||
return ThirdlawGuardrail._responses_error_events(message, collected)
|
||||
return ThirdlawGuardrail._responses_error_events(message, collected, status_code)
|
||||
case StreamSurface.CHAT_COMPLETIONS | StreamSurface.OPAQUE_SSE:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _responses_error_events(message: str, collected: Sequence[object]) -> Sequence[object]:
|
||||
def _responses_error_events(message: str, collected: Sequence[object], status_code: int) -> Sequence[object]:
|
||||
"""A ``/v1/responses`` refusal as an in-stream error event.
|
||||
|
||||
A raised exception reaches the client as the proxy's ``{"error": ...}`` blob, which carries
|
||||
|
|
@ -1059,7 +1061,7 @@ class ThirdlawGuardrail(CustomGuardrail):
|
|||
|
||||
return (
|
||||
OpenAIResponsesHandler().build_stream_error_items(
|
||||
HTTPException(status_code=400, detail=message),
|
||||
HTTPException(status_code=status_code, detail=message),
|
||||
responses_so_far=collected,
|
||||
)
|
||||
or ()
|
||||
|
|
@ -1102,7 +1104,10 @@ class ThirdlawGuardrail(CustomGuardrail):
|
|||
except Exception as error: # noqa: BLE001 # after keepalive flush a raise cannot reach the client; send a frame
|
||||
flushed_frames: Final = (
|
||||
self._stream_error_items(
|
||||
f"ThirdLaw guardrail request failed: {_service_error_message(error)}", surface, collected
|
||||
f"ThirdLaw guardrail request failed: {_service_error_message(error)}",
|
||||
surface,
|
||||
collected,
|
||||
status_code=self._block_status(None),
|
||||
)
|
||||
if self._sse_headers_flushed(started)
|
||||
else None
|
||||
|
|
@ -1136,9 +1141,10 @@ class ThirdlawGuardrail(CustomGuardrail):
|
|||
|
||||
if decision.action == "block":
|
||||
message: Final = decision.message or "Content violates ThirdLaw policy"
|
||||
block_items: Final = self._stream_error_items(message, surface, collected)
|
||||
block_status: Final = self._block_status(decision.response_status)
|
||||
block_items: Final = self._stream_error_items(message, surface, collected, status_code=block_status)
|
||||
if block_items is None:
|
||||
raise self._streaming_block_error(message, response_status=decision.response_status)
|
||||
raise self._streaming_block_error(message, response_status=block_status)
|
||||
for item in block_items:
|
||||
yield item
|
||||
return
|
||||
|
|
@ -1193,7 +1199,9 @@ class ThirdlawGuardrail(CustomGuardrail):
|
|||
for item in collected:
|
||||
yield item
|
||||
return
|
||||
refusal_items: Final = self._stream_error_items(refusal, surface, collected)
|
||||
refusal_items: Final = self._stream_error_items(
|
||||
refusal, surface, collected, status_code=self._block_status(None)
|
||||
)
|
||||
if refusal_items is None:
|
||||
raise self._streaming_block_error(refusal)
|
||||
for item in refusal_items:
|
||||
|
|
@ -1323,9 +1331,10 @@ class ThirdlawGuardrail(CustomGuardrail):
|
|||
return
|
||||
if final_decision.action == "block":
|
||||
message: Final = final_decision.message or "Content violates ThirdLaw policy"
|
||||
block_items: Final = self._stream_error_items(message, surface, collected)
|
||||
block_status: Final = self._block_status(final_decision.response_status)
|
||||
block_items: Final = self._stream_error_items(message, surface, collected, status_code=block_status)
|
||||
if block_items is None:
|
||||
raise self._streaming_block_error(message, response_status=final_decision.response_status)
|
||||
raise self._streaming_block_error(message, response_status=block_status)
|
||||
for item in block_items:
|
||||
yield item
|
||||
return
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from typing import Final
|
|||
|
||||
import pytest
|
||||
|
||||
from litellm.anthropic_interface.exceptions import anthropic_error_sse_frame
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.anthropic_endpoints.streaming_model_restamp import (
|
||||
|
|
@ -38,9 +39,7 @@ def _model_from_frame(frame: bytes | str) -> str:
|
|||
|
||||
|
||||
def test_restamps_sse_bytes_frame():
|
||||
restamped = restamp_anthropic_stream_chunk_model(
|
||||
_message_start_frame("claude-haiku-4-5-20251001"), "claude-auto-1"
|
||||
)
|
||||
restamped = restamp_anthropic_stream_chunk_model(_message_start_frame("claude-haiku-4-5-20251001"), "claude-auto-1")
|
||||
|
||||
assert isinstance(restamped, bytes)
|
||||
assert _model_from_frame(restamped) == "claude-auto-1"
|
||||
|
|
@ -171,6 +170,14 @@ def test_first_non_ping_event_disarms_the_restamper():
|
|||
assert restamper.process(late_message_start) == late_message_start
|
||||
|
||||
|
||||
def test_an_error_frame_passes_through_with_its_status():
|
||||
"""A stream that fails before its first byte is answered with the frame's status, which only
|
||||
the frame object carries; re-encoding it to bytes answered a ThirdLaw 403 block as a 200."""
|
||||
frame = anthropic_error_sse_frame(status_code=403, raw_message="blocked")
|
||||
|
||||
assert AnthropicStreamModelRestamper("claude-auto-1").process(frame) is frame
|
||||
|
||||
|
||||
def test_oversized_unterminated_chunk_flushes_unmodified():
|
||||
blob = b"data: " + b"x" * 70000
|
||||
restamper = AnthropicStreamModelRestamper("claude-auto-1")
|
||||
|
|
|
|||
|
|
@ -1243,6 +1243,11 @@ async def _collect(agen: AsyncIterator[object]) -> list[object]:
|
|||
return [item async for item in agen]
|
||||
|
||||
|
||||
def _sse_text(out: Sequence[object]) -> str:
|
||||
"""Emitted SSE frames as text. Provider frames are bytes and a refusal frame is a str."""
|
||||
return "".join(item.decode() if isinstance(item, bytes) else item for item in out if isinstance(item, (bytes, str)))
|
||||
|
||||
|
||||
class _StreamWithHiddenParams:
|
||||
"""Mimics CustomStreamWrapper: an async-iterable whose chunks are plain ModelResponseStream
|
||||
objects, but which itself carries _hidden_params (the provider response headers, snapshotted
|
||||
|
|
@ -1476,6 +1481,73 @@ async def test_a_buffered_streamed_block_reaches_the_client_as_its_block_status(
|
|||
assert json.loads(bytes(response.body))["error"]["message"] == "leaked secret"
|
||||
|
||||
|
||||
async def test_a_buffered_responses_stream_block_reaches_the_client_as_its_block_status(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""The Responses error event carried a fixed 400, which the proxy answered with (TL-2934)."""
|
||||
import litellm
|
||||
from litellm.proxy.common_request_processing import create_response
|
||||
from litellm.proxy.proxy_server import (
|
||||
async_data_generator, # pyright: ignore[reportUnknownVariableType] # its response parameter is untyped
|
||||
)
|
||||
|
||||
g = _make_guardrail(
|
||||
event_hook="post_call",
|
||||
decisions=[_decision_response({"action": "block", "message": "leaked secret", "response_status": 403})],
|
||||
)
|
||||
monkeypatch.setattr(litellm, "callbacks", [g])
|
||||
|
||||
response = await create_response(
|
||||
async_data_generator( # pyright: ignore[reportArgumentType] # yields str | bytes; the proxy hands create_response the same generator
|
||||
response=_aiter(_responses_stream_chunks()),
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
request_data=_request_data(),
|
||||
responses_stream_errors=True,
|
||||
),
|
||||
media_type="text/event-stream",
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
assert json.loads(bytes(response.body))["error"]["message"] == "leaked secret"
|
||||
|
||||
|
||||
async def test_a_buffered_messages_stream_block_reaches_the_client_as_its_block_status(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""A /v1/messages block was a raw error frame under a 200, so an Anthropic client saw a
|
||||
successful call that failed mid-stream (TL-2934). ``restamp_model`` is set the way the proxy
|
||||
sets it for every /v1/messages stream."""
|
||||
import litellm
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing, create_response
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
g = _make_guardrail(
|
||||
event_hook="post_call",
|
||||
decisions=[_decision_response({"action": "block", "message": "leaked secret", "response_status": 403})],
|
||||
)
|
||||
monkeypatch.setattr(litellm, "callbacks", [g])
|
||||
|
||||
response = await create_response(
|
||||
ProxyBaseLLMRequestProcessing.async_sse_data_generator( # pyright: ignore[reportUnknownMemberType] # its request_data parameter is an untyped dict
|
||||
response=_aiter(_anthropic_sse_frames()),
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
request_data=_request_data(),
|
||||
proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()),
|
||||
restamp_model="claude-alias",
|
||||
),
|
||||
media_type="text/event-stream",
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
assert json.loads(bytes(response.body)) == {
|
||||
"type": "error",
|
||||
"error": {"type": "permission_error", "message": "leaked secret"},
|
||||
}
|
||||
|
||||
|
||||
async def test_streaming_buffered_holds_chunks_until_decision():
|
||||
call_order: list[str] = []
|
||||
|
||||
|
|
@ -1591,9 +1663,9 @@ async def test_a_truncated_tool_argument_can_still_be_blocked():
|
|||
request_data=_request_data(),
|
||||
)
|
||||
)
|
||||
blob = b"".join(item for item in out if isinstance(item, bytes))
|
||||
assert b"secret in tool args" in blob
|
||||
assert b"sk-live-TRUNC" not in blob
|
||||
blob = _sse_text(out)
|
||||
assert "secret in tool args" in blob
|
||||
assert "sk-live-TRUNC" not in blob
|
||||
|
||||
|
||||
def _anthropic_sse_frames() -> list[bytes]:
|
||||
|
|
@ -1806,7 +1878,11 @@ async def test_a_chat_shaped_rewrite_of_a_messages_stream_fails_closed():
|
|||
|
||||
|
||||
async def test_streaming_raw_sse_block_emits_anthropic_error_frame():
|
||||
g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "leaked secret"})])
|
||||
from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame
|
||||
|
||||
g = _make_guardrail(
|
||||
decisions=[_decision_response({"action": "block", "message": "leaked secret", "response_status": 403})]
|
||||
)
|
||||
out = await _collect(
|
||||
g.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
|
|
@ -1815,9 +1891,11 @@ async def test_streaming_raw_sse_block_emits_anthropic_error_frame():
|
|||
)
|
||||
)
|
||||
assert len(out) == 1
|
||||
assert isinstance(out[0], bytes)
|
||||
assert b"event: error" in out[0]
|
||||
assert b"leaked secret" in out[0]
|
||||
frame = out[0]
|
||||
assert isinstance(frame, AnthropicErrorSseFrame)
|
||||
assert frame.status_code == 403
|
||||
assert frame.startswith("event: error\n")
|
||||
assert frame.error_response["error"] == {"type": "permission_error", "message": "leaked secret"}
|
||||
|
||||
|
||||
def _unscannable_chunks() -> list[dict]:
|
||||
|
|
@ -2539,7 +2617,7 @@ async def test_a_buffered_messages_stream_gets_an_error_frame_when_the_service_f
|
|||
user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_anthropic_sse_frames()), request_data=_request_data()
|
||||
)
|
||||
)
|
||||
frames = b"".join(item for item in out if isinstance(item, bytes)).decode()
|
||||
frames = _sse_text(out)
|
||||
assert "event: error" in frames
|
||||
assert "ThirdLaw guardrail request failed" in frames
|
||||
assert "hello" not in frames
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue