fix(guardrails): answer a ThirdLaw streamed chat block with its block status

A block decided at the end of a buffered Chat Completions stream raised a bare
StreamingCallbackError. The proxy serializes an exception without a status as
code 500, and because nothing had reached the client yet, create_response sent
an HTTP 500 instead of the 403 ThirdLaw rendered

StreamingCallbackError now takes an optional status_code, and every streamed
block passes the decision's response_status through _block_status, the same
mapping the unary path uses. Other guardrails raise it without a status and
still answer 500

Part of TL-2934
This commit is contained in:
r3ddyarun 2026-10-29 22:47:18 -05:00
parent 037d4d8adc
commit 83a061d175
3 changed files with 45 additions and 12 deletions

View file

@ -1021,10 +1021,10 @@ class ThirdlawGuardrail(CustomGuardrail):
return None
return assembled if isinstance(assembled, ModelResponse) else None
def _streaming_block_error(self, message: str) -> Exception:
def _streaming_block_error(self, message: str, *, response_status: int | None = None) -> Exception:
from litellm.proxy.proxy_server import StreamingCallbackError
return StreamingCallbackError(message)
return StreamingCallbackError(message, status_code=self._block_status(response_status))
@staticmethod
def _stream_error_items(
@ -1138,7 +1138,7 @@ class ThirdlawGuardrail(CustomGuardrail):
message: Final = decision.message or "Content violates ThirdLaw policy"
block_items: Final = self._stream_error_items(message, surface, collected)
if block_items is None:
raise self._streaming_block_error(message)
raise self._streaming_block_error(message, response_status=decision.response_status)
for item in block_items:
yield item
return
@ -1289,7 +1289,10 @@ class ThirdlawGuardrail(CustomGuardrail):
response_headers=response_headers,
)
if interim_decision is not None and interim_decision.action == "block":
raise self._streaming_block_error(interim_decision.message or "Content violates ThirdLaw policy")
raise self._streaming_block_error(
interim_decision.message or "Content violates ThirdLaw policy",
response_status=interim_decision.response_status,
)
surface: Final = classify_stream(collected)
if surface is StreamSurface.RESPONSES:
@ -1322,7 +1325,7 @@ class ThirdlawGuardrail(CustomGuardrail):
message: Final = final_decision.message or "Content violates ThirdLaw policy"
block_items: Final = self._stream_error_items(message, surface, collected)
if block_items is None:
raise self._streaming_block_error(message)
raise self._streaming_block_error(message, response_status=final_decision.response_status)
for item in block_items:
yield item
return

View file

@ -4854,7 +4854,12 @@ async def _run_background_health_check():
class StreamingCallbackError(Exception):
pass
status_code: int
def __init__(self, message: str, *, status_code: int | None = None) -> None:
super().__init__(message)
if status_code is not None:
self.status_code = status_code
# Fields in ``litellm_settings`` / ``general_settings`` whose values flow

View file

@ -1419,12 +1419,34 @@ async def test_streaming_buffered_block_raises_streaming_callback_error():
from litellm.proxy.proxy_server import StreamingCallbackError
g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "leaked secret"})])
with pytest.raises(StreamingCallbackError, match="leaked secret"):
with pytest.raises(StreamingCallbackError, match="leaked secret") as raised:
await _collect(
g.async_post_call_streaming_iterator_hook(
user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data()
)
)
assert raised.value.status_code == 400
@pytest.mark.parametrize(
("response_status", "expected_status"),
[(200, 400), (422, 422), (503, 503)],
)
async def test_streaming_buffered_block_raises_with_the_block_status(response_status: int, expected_status: int):
from litellm.proxy.proxy_server import StreamingCallbackError
g = _make_guardrail(
decisions=[
_decision_response({"action": "block", "message": "leaked secret", "response_status": response_status})
]
)
with pytest.raises(StreamingCallbackError, match="leaked secret") as raised:
await _collect(
g.async_post_call_streaming_iterator_hook(
user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data()
)
)
assert raised.value.status_code == expected_status
async def test_streaming_buffered_holds_chunks_until_decision():
@ -1458,15 +1480,16 @@ async def test_streaming_sampled_interim_block_terminates_stream():
streaming_buffer_until_moderated=False,
streaming_end_of_stream_only=False,
streaming_sampling_rate=1,
decisions=[_decision_response({"action": "block", "message": "bad interim"})],
decisions=[_decision_response({"action": "block", "message": "bad interim", "response_status": 422})],
)
agen = g.async_post_call_streaming_iterator_hook(
user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data()
)
first = await agen.__anext__()
assert isinstance(first, ModelResponseStream)
with pytest.raises(StreamingCallbackError, match="bad interim"):
with pytest.raises(StreamingCallbackError, match="bad interim") as raised:
await agen.__anext__()
assert raised.value.status_code == 422
def _truncated_tool_use_sse_frames() -> list[bytes]:
@ -1798,7 +1821,7 @@ async def test_unscannable_stream_fails_closed_by_default():
from litellm.proxy.proxy_server import StreamingCallbackError
g = _make_guardrail(decisions=[])
with pytest.raises(StreamingCallbackError, match="could not be assembled for scanning"):
with pytest.raises(StreamingCallbackError, match="could not be assembled for scanning") as raised:
await _collect(
g.async_post_call_streaming_iterator_hook(
user_api_key_dict=UserAPIKeyAuth(),
@ -1806,6 +1829,7 @@ async def test_unscannable_stream_fails_closed_by_default():
request_data=_request_data(),
)
)
assert raised.value.status_code == 400
assert g.async_handler.post.await_count == 0
@ -1905,9 +1929,9 @@ async def test_a_sampled_stream_blocks_on_the_final_decision():
streaming_buffer_until_moderated=False,
streaming_end_of_stream_only=False,
streaming_sampling_rate=100,
decisions=[_decision_response({"action": "block", "message": "final says no"})],
decisions=[_decision_response({"action": "block", "message": "final says no", "response_status": 451})],
)
with pytest.raises(StreamingCallbackError, match="final says no"):
with pytest.raises(StreamingCallbackError, match="final says no") as raised:
await _collect(
g.async_post_call_streaming_iterator_hook(
user_api_key_dict=UserAPIKeyAuth(),
@ -1915,6 +1939,7 @@ async def test_a_sampled_stream_blocks_on_the_final_decision():
request_data=_request_data(),
)
)
assert raised.value.status_code == 451
async def test_a_sampled_stream_cannot_apply_a_late_modify_response():