mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(guardrails): answer a ThirdLaw streamed chat block with its block status
A block decided at the end of a buffered Chat Completions stream raised a bare StreamingCallbackError. The proxy serializes an exception without a status as code 500, and because nothing had reached the client yet, create_response sent an HTTP 500 instead of the 403 ThirdLaw rendered StreamingCallbackError now takes an optional status_code, and every streamed block passes the decision's response_status through _block_status, the same mapping the unary path uses. Other guardrails raise it without a status and still answer 500 Part of TL-2934
This commit is contained in:
parent
037d4d8adc
commit
83a061d175
3 changed files with 45 additions and 12 deletions
|
|
@ -1021,10 +1021,10 @@ class ThirdlawGuardrail(CustomGuardrail):
|
|||
return None
|
||||
return assembled if isinstance(assembled, ModelResponse) else None
|
||||
|
||||
def _streaming_block_error(self, message: str) -> Exception:
|
||||
def _streaming_block_error(self, message: str, *, response_status: int | None = None) -> Exception:
|
||||
from litellm.proxy.proxy_server import StreamingCallbackError
|
||||
|
||||
return StreamingCallbackError(message)
|
||||
return StreamingCallbackError(message, status_code=self._block_status(response_status))
|
||||
|
||||
@staticmethod
|
||||
def _stream_error_items(
|
||||
|
|
@ -1138,7 +1138,7 @@ class ThirdlawGuardrail(CustomGuardrail):
|
|||
message: Final = decision.message or "Content violates ThirdLaw policy"
|
||||
block_items: Final = self._stream_error_items(message, surface, collected)
|
||||
if block_items is None:
|
||||
raise self._streaming_block_error(message)
|
||||
raise self._streaming_block_error(message, response_status=decision.response_status)
|
||||
for item in block_items:
|
||||
yield item
|
||||
return
|
||||
|
|
@ -1289,7 +1289,10 @@ class ThirdlawGuardrail(CustomGuardrail):
|
|||
response_headers=response_headers,
|
||||
)
|
||||
if interim_decision is not None and interim_decision.action == "block":
|
||||
raise self._streaming_block_error(interim_decision.message or "Content violates ThirdLaw policy")
|
||||
raise self._streaming_block_error(
|
||||
interim_decision.message or "Content violates ThirdLaw policy",
|
||||
response_status=interim_decision.response_status,
|
||||
)
|
||||
|
||||
surface: Final = classify_stream(collected)
|
||||
if surface is StreamSurface.RESPONSES:
|
||||
|
|
@ -1322,7 +1325,7 @@ class ThirdlawGuardrail(CustomGuardrail):
|
|||
message: Final = final_decision.message or "Content violates ThirdLaw policy"
|
||||
block_items: Final = self._stream_error_items(message, surface, collected)
|
||||
if block_items is None:
|
||||
raise self._streaming_block_error(message)
|
||||
raise self._streaming_block_error(message, response_status=final_decision.response_status)
|
||||
for item in block_items:
|
||||
yield item
|
||||
return
|
||||
|
|
|
|||
|
|
@ -4854,7 +4854,12 @@ async def _run_background_health_check():
|
|||
|
||||
|
||||
class StreamingCallbackError(Exception):
|
||||
pass
|
||||
status_code: int
|
||||
|
||||
def __init__(self, message: str, *, status_code: int | None = None) -> None:
|
||||
super().__init__(message)
|
||||
if status_code is not None:
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
# Fields in ``litellm_settings`` / ``general_settings`` whose values flow
|
||||
|
|
|
|||
|
|
@ -1419,12 +1419,34 @@ async def test_streaming_buffered_block_raises_streaming_callback_error():
|
|||
from litellm.proxy.proxy_server import StreamingCallbackError
|
||||
|
||||
g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "leaked secret"})])
|
||||
with pytest.raises(StreamingCallbackError, match="leaked secret"):
|
||||
with pytest.raises(StreamingCallbackError, match="leaked secret") as raised:
|
||||
await _collect(
|
||||
g.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data()
|
||||
)
|
||||
)
|
||||
assert raised.value.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("response_status", "expected_status"),
|
||||
[(200, 400), (422, 422), (503, 503)],
|
||||
)
|
||||
async def test_streaming_buffered_block_raises_with_the_block_status(response_status: int, expected_status: int):
|
||||
from litellm.proxy.proxy_server import StreamingCallbackError
|
||||
|
||||
g = _make_guardrail(
|
||||
decisions=[
|
||||
_decision_response({"action": "block", "message": "leaked secret", "response_status": response_status})
|
||||
]
|
||||
)
|
||||
with pytest.raises(StreamingCallbackError, match="leaked secret") as raised:
|
||||
await _collect(
|
||||
g.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data()
|
||||
)
|
||||
)
|
||||
assert raised.value.status_code == expected_status
|
||||
|
||||
|
||||
async def test_streaming_buffered_holds_chunks_until_decision():
|
||||
|
|
@ -1458,15 +1480,16 @@ async def test_streaming_sampled_interim_block_terminates_stream():
|
|||
streaming_buffer_until_moderated=False,
|
||||
streaming_end_of_stream_only=False,
|
||||
streaming_sampling_rate=1,
|
||||
decisions=[_decision_response({"action": "block", "message": "bad interim"})],
|
||||
decisions=[_decision_response({"action": "block", "message": "bad interim", "response_status": 422})],
|
||||
)
|
||||
agen = g.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data()
|
||||
)
|
||||
first = await agen.__anext__()
|
||||
assert isinstance(first, ModelResponseStream)
|
||||
with pytest.raises(StreamingCallbackError, match="bad interim"):
|
||||
with pytest.raises(StreamingCallbackError, match="bad interim") as raised:
|
||||
await agen.__anext__()
|
||||
assert raised.value.status_code == 422
|
||||
|
||||
|
||||
def _truncated_tool_use_sse_frames() -> list[bytes]:
|
||||
|
|
@ -1798,7 +1821,7 @@ async def test_unscannable_stream_fails_closed_by_default():
|
|||
from litellm.proxy.proxy_server import StreamingCallbackError
|
||||
|
||||
g = _make_guardrail(decisions=[])
|
||||
with pytest.raises(StreamingCallbackError, match="could not be assembled for scanning"):
|
||||
with pytest.raises(StreamingCallbackError, match="could not be assembled for scanning") as raised:
|
||||
await _collect(
|
||||
g.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
|
|
@ -1806,6 +1829,7 @@ async def test_unscannable_stream_fails_closed_by_default():
|
|||
request_data=_request_data(),
|
||||
)
|
||||
)
|
||||
assert raised.value.status_code == 400
|
||||
assert g.async_handler.post.await_count == 0
|
||||
|
||||
|
||||
|
|
@ -1905,9 +1929,9 @@ async def test_a_sampled_stream_blocks_on_the_final_decision():
|
|||
streaming_buffer_until_moderated=False,
|
||||
streaming_end_of_stream_only=False,
|
||||
streaming_sampling_rate=100,
|
||||
decisions=[_decision_response({"action": "block", "message": "final says no"})],
|
||||
decisions=[_decision_response({"action": "block", "message": "final says no", "response_status": 451})],
|
||||
)
|
||||
with pytest.raises(StreamingCallbackError, match="final says no"):
|
||||
with pytest.raises(StreamingCallbackError, match="final says no") as raised:
|
||||
await _collect(
|
||||
g.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
|
|
@ -1915,6 +1939,7 @@ async def test_a_sampled_stream_blocks_on_the_final_decision():
|
|||
request_data=_request_data(),
|
||||
)
|
||||
)
|
||||
assert raised.value.status_code == 451
|
||||
|
||||
|
||||
async def test_a_sampled_stream_cannot_apply_a_late_modify_response():
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue