fix(guardrails): answer ThirdLaw streamed Responses and Messages blocks with their block status

A block decided at the end of a buffered /v1/responses stream went out as a
Responses error event with a hardcoded 400, and create_response answered with
that code. On /v1/messages the refusal was raw error-frame bytes, which carry
no status, so the client got a 200 and a failed stream instead of the 403
ThirdLaw rendered

_stream_error_items now takes the resolved block status. The Responses event
carries it as its code, and the Messages refusal is an AnthropicErrorSseFrame,
the frame create_response already answers as a JSON error with its own
status. Its error type now follows the status (permission_error for a 403)
rather than the non-standard guardrail_error. The Messages restamper used to
re-encode every str chunk to bytes while armed, which dropped the frame's
status, so it passes an error frame through untouched. Refusals without a
verdict keep 400, as before

Part of TL-2934
This commit is contained in:
r3ddyarun 2026-10-30 00:17:38 -05:00
parent 72232ea88a
commit 93c5cebf9d
4 changed files with 125 additions and 25 deletions

View file

@ -14,6 +14,7 @@ from typing import Final
from pydantic import TypeAdapter, ValidationError
from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame
from litellm.llms.custom_httpx.sse_lines import split_sse_lines_keepends
_MESSAGE_START_EVENT: Final = "message_start"
@ -108,12 +109,17 @@ class AnthropicStreamModelRestamper:
def __init__(self, requested_model: str) -> None:
self._requested_model: Final = requested_model
self._held = b""
self._held: bytes = b""
self._armed = True
def process(self, chunk: object) -> object:
if not self._armed:
return chunk
if isinstance(chunk, AnthropicErrorSseFrame) and not self._held:
# An error event carries no model, and re-encoding it would drop the status the proxy
# answers with when the stream fails before its first byte
self._armed = False
return chunk
if isinstance(chunk, (bytes, bytearray)):
return self._process_encoded(bytes(chunk))
if isinstance(chunk, str):

View file

@ -18,6 +18,7 @@ from pydantic import BaseModel, TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._version import version as litellm_version
from litellm.anthropic_interface.exceptions import anthropic_error_sse_frame
from litellm.caching import DualCache
from litellm.exceptions import GuardrailRaisedException
from litellm.exceptions import Timeout as LitellmTimeout
@ -37,7 +38,6 @@ from litellm.proxy._types import (
)
from litellm.proxy.guardrails.anthropic_sse import (
anthropic_sse_chunks_from_body,
anthropic_sse_error_frames,
assemble_anthropic_sse_body,
sse_stream_text,
)
@ -1028,23 +1028,25 @@ class ThirdlawGuardrail(CustomGuardrail):
@staticmethod
def _stream_error_items(
message: str, surface: StreamSurface, collected: Sequence[object]
message: str, surface: StreamSurface, collected: Sequence[object], *, status_code: int
) -> Sequence[object] | None:
"""Terminal stream items framing a refusal in this surface's wire format.
``None`` means the surface cannot frame its own error, so the caller raises instead and
lets the proxy's data generator serialize it.
lets the proxy's data generator serialize it. ``status_code`` travels inside the frame:
when the refusal is the stream's first item the proxy answers with it as a JSON error,
and after a keepalive ping has flushed a 200 it is only the frame's body.
"""
match surface:
case StreamSurface.ANTHROPIC_MESSAGES:
return anthropic_sse_error_frames(message)
return (anthropic_error_sse_frame(status_code, message),)
case StreamSurface.RESPONSES:
return ThirdlawGuardrail._responses_error_events(message, collected)
return ThirdlawGuardrail._responses_error_events(message, collected, status_code)
case StreamSurface.CHAT_COMPLETIONS | StreamSurface.OPAQUE_SSE:
return None
@staticmethod
def _responses_error_events(message: str, collected: Sequence[object]) -> Sequence[object]:
def _responses_error_events(message: str, collected: Sequence[object], status_code: int) -> Sequence[object]:
"""A ``/v1/responses`` refusal as an in-stream error event.
A raised exception reaches the client as the proxy's ``{"error": ...}`` blob, which carries
@ -1059,7 +1061,7 @@ class ThirdlawGuardrail(CustomGuardrail):
return (
OpenAIResponsesHandler().build_stream_error_items(
HTTPException(status_code=400, detail=message),
HTTPException(status_code=status_code, detail=message),
responses_so_far=collected,
)
or ()
@ -1102,7 +1104,10 @@ class ThirdlawGuardrail(CustomGuardrail):
except Exception as error: # noqa: BLE001 # after keepalive flush a raise cannot reach the client; send a frame
flushed_frames: Final = (
self._stream_error_items(
f"ThirdLaw guardrail request failed: {_service_error_message(error)}", surface, collected
f"ThirdLaw guardrail request failed: {_service_error_message(error)}",
surface,
collected,
status_code=self._block_status(None),
)
if self._sse_headers_flushed(started)
else None
@ -1136,9 +1141,10 @@ class ThirdlawGuardrail(CustomGuardrail):
if decision.action == "block":
message: Final = decision.message or "Content violates ThirdLaw policy"
block_items: Final = self._stream_error_items(message, surface, collected)
block_status: Final = self._block_status(decision.response_status)
block_items: Final = self._stream_error_items(message, surface, collected, status_code=block_status)
if block_items is None:
raise self._streaming_block_error(message, response_status=decision.response_status)
raise self._streaming_block_error(message, response_status=block_status)
for item in block_items:
yield item
return
@ -1193,7 +1199,9 @@ class ThirdlawGuardrail(CustomGuardrail):
for item in collected:
yield item
return
refusal_items: Final = self._stream_error_items(refusal, surface, collected)
refusal_items: Final = self._stream_error_items(
refusal, surface, collected, status_code=self._block_status(None)
)
if refusal_items is None:
raise self._streaming_block_error(refusal)
for item in refusal_items:
@ -1323,9 +1331,10 @@ class ThirdlawGuardrail(CustomGuardrail):
return
if final_decision.action == "block":
message: Final = final_decision.message or "Content violates ThirdLaw policy"
block_items: Final = self._stream_error_items(message, surface, collected)
block_status: Final = self._block_status(final_decision.response_status)
block_items: Final = self._stream_error_items(message, surface, collected, status_code=block_status)
if block_items is None:
raise self._streaming_block_error(message, response_status=final_decision.response_status)
raise self._streaming_block_error(message, response_status=block_status)
for item in block_items:
yield item
return

View file

@ -8,6 +8,7 @@ from typing import Final
import pytest
from litellm.anthropic_interface.exceptions import anthropic_error_sse_frame
from litellm.caching.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.anthropic_endpoints.streaming_model_restamp import (
@ -38,9 +39,7 @@ def _model_from_frame(frame: bytes | str) -> str:
def test_restamps_sse_bytes_frame():
restamped = restamp_anthropic_stream_chunk_model(
_message_start_frame("claude-haiku-4-5-20251001"), "claude-auto-1"
)
restamped = restamp_anthropic_stream_chunk_model(_message_start_frame("claude-haiku-4-5-20251001"), "claude-auto-1")
assert isinstance(restamped, bytes)
assert _model_from_frame(restamped) == "claude-auto-1"
@ -171,6 +170,14 @@ def test_first_non_ping_event_disarms_the_restamper():
assert restamper.process(late_message_start) == late_message_start
def test_an_error_frame_passes_through_with_its_status():
"""A stream that fails before its first byte is answered with the frame's status, which only
the frame object carries; re-encoding it to bytes answered a ThirdLaw 403 block as a 200."""
frame = anthropic_error_sse_frame(status_code=403, raw_message="blocked")
assert AnthropicStreamModelRestamper("claude-auto-1").process(frame) is frame
def test_oversized_unterminated_chunk_flushes_unmodified():
blob = b"data: " + b"x" * 70000
restamper = AnthropicStreamModelRestamper("claude-auto-1")

View file

@ -1243,6 +1243,11 @@ async def _collect(agen: AsyncIterator[object]) -> list[object]:
return [item async for item in agen]
def _sse_text(out: Sequence[object]) -> str:
"""Emitted SSE frames as text. Provider frames are bytes and a refusal frame is a str."""
return "".join(item.decode() if isinstance(item, bytes) else item for item in out if isinstance(item, (bytes, str)))
class _StreamWithHiddenParams:
"""Mimics CustomStreamWrapper: an async-iterable whose chunks are plain ModelResponseStream
objects, but which itself carries _hidden_params (the provider response headers, snapshotted
@ -1476,6 +1481,73 @@ async def test_a_buffered_streamed_block_reaches_the_client_as_its_block_status(
assert json.loads(bytes(response.body))["error"]["message"] == "leaked secret"
async def test_a_buffered_responses_stream_block_reaches_the_client_as_its_block_status(
monkeypatch: pytest.MonkeyPatch,
):
"""The Responses error event carried a fixed 400, which the proxy answered with (TL-2934)."""
import litellm
from litellm.proxy.common_request_processing import create_response
from litellm.proxy.proxy_server import (
async_data_generator, # pyright: ignore[reportUnknownVariableType] # its response parameter is untyped
)
g = _make_guardrail(
event_hook="post_call",
decisions=[_decision_response({"action": "block", "message": "leaked secret", "response_status": 403})],
)
monkeypatch.setattr(litellm, "callbacks", [g])
response = await create_response(
async_data_generator( # pyright: ignore[reportArgumentType] # yields str | bytes; the proxy hands create_response the same generator
response=_aiter(_responses_stream_chunks()),
user_api_key_dict=UserAPIKeyAuth(),
request_data=_request_data(),
responses_stream_errors=True,
),
media_type="text/event-stream",
headers={},
)
assert response.status_code == 403
assert json.loads(bytes(response.body))["error"]["message"] == "leaked secret"
async def test_a_buffered_messages_stream_block_reaches_the_client_as_its_block_status(
monkeypatch: pytest.MonkeyPatch,
):
"""A /v1/messages block was a raw error frame under a 200, so an Anthropic client saw a
successful call that failed mid-stream (TL-2934). ``restamp_model`` is set the way the proxy
sets it for every /v1/messages stream."""
import litellm
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing, create_response
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.utils import ProxyLogging
g = _make_guardrail(
event_hook="post_call",
decisions=[_decision_response({"action": "block", "message": "leaked secret", "response_status": 403})],
)
monkeypatch.setattr(litellm, "callbacks", [g])
response = await create_response(
ProxyBaseLLMRequestProcessing.async_sse_data_generator( # pyright: ignore[reportUnknownMemberType] # its request_data parameter is an untyped dict
response=_aiter(_anthropic_sse_frames()),
user_api_key_dict=UserAPIKeyAuth(),
request_data=_request_data(),
proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()),
restamp_model="claude-alias",
),
media_type="text/event-stream",
headers={},
)
assert response.status_code == 403
assert json.loads(bytes(response.body)) == {
"type": "error",
"error": {"type": "permission_error", "message": "leaked secret"},
}
async def test_streaming_buffered_holds_chunks_until_decision():
call_order: list[str] = []
@ -1591,9 +1663,9 @@ async def test_a_truncated_tool_argument_can_still_be_blocked():
request_data=_request_data(),
)
)
blob = b"".join(item for item in out if isinstance(item, bytes))
assert b"secret in tool args" in blob
assert b"sk-live-TRUNC" not in blob
blob = _sse_text(out)
assert "secret in tool args" in blob
assert "sk-live-TRUNC" not in blob
def _anthropic_sse_frames() -> list[bytes]:
@ -1806,7 +1878,11 @@ async def test_a_chat_shaped_rewrite_of_a_messages_stream_fails_closed():
async def test_streaming_raw_sse_block_emits_anthropic_error_frame():
g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "leaked secret"})])
from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame
g = _make_guardrail(
decisions=[_decision_response({"action": "block", "message": "leaked secret", "response_status": 403})]
)
out = await _collect(
g.async_post_call_streaming_iterator_hook(
user_api_key_dict=UserAPIKeyAuth(),
@ -1815,9 +1891,11 @@ async def test_streaming_raw_sse_block_emits_anthropic_error_frame():
)
)
assert len(out) == 1
assert isinstance(out[0], bytes)
assert b"event: error" in out[0]
assert b"leaked secret" in out[0]
frame = out[0]
assert isinstance(frame, AnthropicErrorSseFrame)
assert frame.status_code == 403
assert frame.startswith("event: error\n")
assert frame.error_response["error"] == {"type": "permission_error", "message": "leaked secret"}
def _unscannable_chunks() -> list[dict]:
@ -2539,7 +2617,7 @@ async def test_a_buffered_messages_stream_gets_an_error_frame_when_the_service_f
user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_anthropic_sse_frames()), request_data=_request_data()
)
)
frames = b"".join(item for item in out if isinstance(item, bytes)).decode()
frames = _sse_text(out)
assert "event: error" in frames
assert "ThirdLaw guardrail request failed" in frames
assert "hello" not in frames