From 83a061d17506be92a1d08bbc4c490c1ed206c8f5 Mon Sep 17 00:00:00 2001 From: r3ddyarun Date: Thu, 29 Oct 2026 22:47:18 -0500 Subject: [PATCH] fix(guardrails): answer a ThirdLaw streamed chat block with its block status A block decided at the end of a buffered Chat Completions stream raised a bare StreamingCallbackError. The proxy serializes an exception without a status as code 500, and because nothing had reached the client yet, create_response sent an HTTP 500 instead of the 403 ThirdLaw rendered StreamingCallbackError now takes an optional status_code, and every streamed block passes the decision's response_status through _block_status, the same mapping the unary path uses. Other guardrails raise it without a status and still answer 500 Part of TL-2934 --- .../guardrail_hooks/thirdlaw/thirdlaw.py | 13 ++++--- litellm/proxy/proxy_server.py | 7 +++- .../guardrail_hooks/test_thirdlaw.py | 37 ++++++++++++++++--- 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py index 61cefcd03da..1048d0177c1 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py +++ b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py @@ -1021,10 +1021,10 @@ class ThirdlawGuardrail(CustomGuardrail): return None return assembled if isinstance(assembled, ModelResponse) else None - def _streaming_block_error(self, message: str) -> Exception: + def _streaming_block_error(self, message: str, *, response_status: int | None = None) -> Exception: from litellm.proxy.proxy_server import StreamingCallbackError - return StreamingCallbackError(message) + return StreamingCallbackError(message, status_code=self._block_status(response_status)) @staticmethod def _stream_error_items( @@ -1138,7 +1138,7 @@ class ThirdlawGuardrail(CustomGuardrail): message: Final = decision.message or "Content violates ThirdLaw policy" block_items: Final = self._stream_error_items(message, surface, collected) if block_items is None: - raise self._streaming_block_error(message) + raise self._streaming_block_error(message, response_status=decision.response_status) for item in block_items: yield item return @@ -1289,7 +1289,10 @@ class ThirdlawGuardrail(CustomGuardrail): response_headers=response_headers, ) if interim_decision is not None and interim_decision.action == "block": - raise self._streaming_block_error(interim_decision.message or "Content violates ThirdLaw policy") + raise self._streaming_block_error( + interim_decision.message or "Content violates ThirdLaw policy", + response_status=interim_decision.response_status, + ) surface: Final = classify_stream(collected) if surface is StreamSurface.RESPONSES: @@ -1322,7 +1325,7 @@ class ThirdlawGuardrail(CustomGuardrail): message: Final = final_decision.message or "Content violates ThirdLaw policy" block_items: Final = self._stream_error_items(message, surface, collected) if block_items is None: - raise self._streaming_block_error(message) + raise self._streaming_block_error(message, response_status=final_decision.response_status) for item in block_items: yield item return diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 78187a59e5b..eccc7ae539f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4854,7 +4854,12 @@ async def _run_background_health_check(): class StreamingCallbackError(Exception): - pass + status_code: int + + def __init__(self, message: str, *, status_code: int | None = None) -> None: + super().__init__(message) + if status_code is not None: + self.status_code = status_code # Fields in ``litellm_settings`` / ``general_settings`` whose values flow diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py index e0af711384b..6b3d4b7d12e 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py @@ -1419,12 +1419,34 @@ async def test_streaming_buffered_block_raises_streaming_callback_error(): from litellm.proxy.proxy_server import StreamingCallbackError g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "leaked secret"})]) - with pytest.raises(StreamingCallbackError, match="leaked secret"): + with pytest.raises(StreamingCallbackError, match="leaked secret") as raised: await _collect( g.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data() ) ) + assert raised.value.status_code == 400 + + +@pytest.mark.parametrize( + ("response_status", "expected_status"), + [(200, 400), (422, 422), (503, 503)], +) +async def test_streaming_buffered_block_raises_with_the_block_status(response_status: int, expected_status: int): + from litellm.proxy.proxy_server import StreamingCallbackError + + g = _make_guardrail( + decisions=[ + _decision_response({"action": "block", "message": "leaked secret", "response_status": response_status}) + ] + ) + with pytest.raises(StreamingCallbackError, match="leaked secret") as raised: + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data() + ) + ) + assert raised.value.status_code == expected_status async def test_streaming_buffered_holds_chunks_until_decision(): @@ -1458,15 +1480,16 @@ async def test_streaming_sampled_interim_block_terminates_stream(): streaming_buffer_until_moderated=False, streaming_end_of_stream_only=False, streaming_sampling_rate=1, - decisions=[_decision_response({"action": "block", "message": "bad interim"})], + decisions=[_decision_response({"action": "block", "message": "bad interim", "response_status": 422})], ) agen = g.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data() ) first = await agen.__anext__() assert isinstance(first, ModelResponseStream) - with pytest.raises(StreamingCallbackError, match="bad interim"): + with pytest.raises(StreamingCallbackError, match="bad interim") as raised: await agen.__anext__() + assert raised.value.status_code == 422 def _truncated_tool_use_sse_frames() -> list[bytes]: @@ -1798,7 +1821,7 @@ async def test_unscannable_stream_fails_closed_by_default(): from litellm.proxy.proxy_server import StreamingCallbackError g = _make_guardrail(decisions=[]) - with pytest.raises(StreamingCallbackError, match="could not be assembled for scanning"): + with pytest.raises(StreamingCallbackError, match="could not be assembled for scanning") as raised: await _collect( g.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth(), @@ -1806,6 +1829,7 @@ async def test_unscannable_stream_fails_closed_by_default(): request_data=_request_data(), ) ) + assert raised.value.status_code == 400 assert g.async_handler.post.await_count == 0 @@ -1905,9 +1929,9 @@ async def test_a_sampled_stream_blocks_on_the_final_decision(): streaming_buffer_until_moderated=False, streaming_end_of_stream_only=False, streaming_sampling_rate=100, - decisions=[_decision_response({"action": "block", "message": "final says no"})], + decisions=[_decision_response({"action": "block", "message": "final says no", "response_status": 451})], ) - with pytest.raises(StreamingCallbackError, match="final says no"): + with pytest.raises(StreamingCallbackError, match="final says no") as raised: await _collect( g.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth(), @@ -1915,6 +1939,7 @@ async def test_a_sampled_stream_blocks_on_the_final_decision(): request_data=_request_data(), ) ) + assert raised.value.status_code == 451 async def test_a_sampled_stream_cannot_apply_a_late_modify_response():