diff --git a/litellm/proxy/anthropic_endpoints/streaming_model_restamp.py b/litellm/proxy/anthropic_endpoints/streaming_model_restamp.py index 32e0f6e8682..688cb4250c0 100644 --- a/litellm/proxy/anthropic_endpoints/streaming_model_restamp.py +++ b/litellm/proxy/anthropic_endpoints/streaming_model_restamp.py @@ -14,6 +14,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError +from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame from litellm.llms.custom_httpx.sse_lines import split_sse_lines_keepends _MESSAGE_START_EVENT: Final = "message_start" @@ -108,12 +109,17 @@ class AnthropicStreamModelRestamper: def __init__(self, requested_model: str) -> None: self._requested_model: Final = requested_model - self._held = b"" + self._held: bytes = b"" self._armed = True def process(self, chunk: object) -> object: if not self._armed: return chunk + if isinstance(chunk, AnthropicErrorSseFrame) and not self._held: + # An error event carries no model, and re-encoding it would drop the status the proxy + # answers with when the stream fails before its first byte + self._armed = False + return chunk if isinstance(chunk, (bytes, bytearray)): return self._process_encoded(bytes(chunk)) if isinstance(chunk, str): diff --git a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py index 1048d0177c1..eba785cc628 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py +++ b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py @@ -18,6 +18,7 @@ from pydantic import BaseModel, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm._version import version as litellm_version +from litellm.anthropic_interface.exceptions import anthropic_error_sse_frame from litellm.caching import DualCache from litellm.exceptions import GuardrailRaisedException from litellm.exceptions import Timeout as LitellmTimeout @@ -37,7 +38,6 @@ from litellm.proxy._types import ( ) from litellm.proxy.guardrails.anthropic_sse import ( anthropic_sse_chunks_from_body, - anthropic_sse_error_frames, assemble_anthropic_sse_body, sse_stream_text, ) @@ -1028,23 +1028,25 @@ class ThirdlawGuardrail(CustomGuardrail): @staticmethod def _stream_error_items( - message: str, surface: StreamSurface, collected: Sequence[object] + message: str, surface: StreamSurface, collected: Sequence[object], *, status_code: int ) -> Sequence[object] | None: """Terminal stream items framing a refusal in this surface's wire format. ``None`` means the surface cannot frame its own error, so the caller raises instead and - lets the proxy's data generator serialize it. + lets the proxy's data generator serialize it. ``status_code`` travels inside the frame: + when the refusal is the stream's first item the proxy answers with it as a JSON error, + and after a keepalive ping has flushed a 200 it is only the frame's body. """ match surface: case StreamSurface.ANTHROPIC_MESSAGES: - return anthropic_sse_error_frames(message) + return (anthropic_error_sse_frame(status_code, message),) case StreamSurface.RESPONSES: - return ThirdlawGuardrail._responses_error_events(message, collected) + return ThirdlawGuardrail._responses_error_events(message, collected, status_code) case StreamSurface.CHAT_COMPLETIONS | StreamSurface.OPAQUE_SSE: return None @staticmethod - def _responses_error_events(message: str, collected: Sequence[object]) -> Sequence[object]: + def _responses_error_events(message: str, collected: Sequence[object], status_code: int) -> Sequence[object]: """A ``/v1/responses`` refusal as an in-stream error event. A raised exception reaches the client as the proxy's ``{"error": ...}`` blob, which carries @@ -1059,7 +1061,7 @@ class ThirdlawGuardrail(CustomGuardrail): return ( OpenAIResponsesHandler().build_stream_error_items( - HTTPException(status_code=400, detail=message), + HTTPException(status_code=status_code, detail=message), responses_so_far=collected, ) or () @@ -1102,7 +1104,10 @@ class ThirdlawGuardrail(CustomGuardrail): except Exception as error: # noqa: BLE001 # after keepalive flush a raise cannot reach the client; send a frame flushed_frames: Final = ( self._stream_error_items( - f"ThirdLaw guardrail request failed: {_service_error_message(error)}", surface, collected + f"ThirdLaw guardrail request failed: {_service_error_message(error)}", + surface, + collected, + status_code=self._block_status(None), ) if self._sse_headers_flushed(started) else None @@ -1136,9 +1141,10 @@ class ThirdlawGuardrail(CustomGuardrail): if decision.action == "block": message: Final = decision.message or "Content violates ThirdLaw policy" - block_items: Final = self._stream_error_items(message, surface, collected) + block_status: Final = self._block_status(decision.response_status) + block_items: Final = self._stream_error_items(message, surface, collected, status_code=block_status) if block_items is None: - raise self._streaming_block_error(message, response_status=decision.response_status) + raise self._streaming_block_error(message, response_status=block_status) for item in block_items: yield item return @@ -1193,7 +1199,9 @@ class ThirdlawGuardrail(CustomGuardrail): for item in collected: yield item return - refusal_items: Final = self._stream_error_items(refusal, surface, collected) + refusal_items: Final = self._stream_error_items( + refusal, surface, collected, status_code=self._block_status(None) + ) if refusal_items is None: raise self._streaming_block_error(refusal) for item in refusal_items: @@ -1323,9 +1331,10 @@ class ThirdlawGuardrail(CustomGuardrail): return if final_decision.action == "block": message: Final = final_decision.message or "Content violates ThirdLaw policy" - block_items: Final = self._stream_error_items(message, surface, collected) + block_status: Final = self._block_status(final_decision.response_status) + block_items: Final = self._stream_error_items(message, surface, collected, status_code=block_status) if block_items is None: - raise self._streaming_block_error(message, response_status=final_decision.response_status) + raise self._streaming_block_error(message, response_status=block_status) for item in block_items: yield item return diff --git a/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py b/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py index 5911fcdc073..e1fe3ce5ee5 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py +++ b/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py @@ -8,6 +8,7 @@ from typing import Final import pytest +from litellm.anthropic_interface.exceptions import anthropic_error_sse_frame from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.streaming_model_restamp import ( @@ -38,9 +39,7 @@ def _model_from_frame(frame: bytes | str) -> str: def test_restamps_sse_bytes_frame(): - restamped = restamp_anthropic_stream_chunk_model( - _message_start_frame("claude-haiku-4-5-20251001"), "claude-auto-1" - ) + restamped = restamp_anthropic_stream_chunk_model(_message_start_frame("claude-haiku-4-5-20251001"), "claude-auto-1") assert isinstance(restamped, bytes) assert _model_from_frame(restamped) == "claude-auto-1" @@ -171,6 +170,14 @@ def test_first_non_ping_event_disarms_the_restamper(): assert restamper.process(late_message_start) == late_message_start +def test_an_error_frame_passes_through_with_its_status(): + """A stream that fails before its first byte is answered with the frame's status, which only + the frame object carries; re-encoding it to bytes answered a ThirdLaw 403 block as a 200.""" + frame = anthropic_error_sse_frame(status_code=403, raw_message="blocked") + + assert AnthropicStreamModelRestamper("claude-auto-1").process(frame) is frame + + def test_oversized_unterminated_chunk_flushes_unmodified(): blob = b"data: " + b"x" * 70000 restamper = AnthropicStreamModelRestamper("claude-auto-1") diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py index 4819b139f70..c7da867c5dc 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py @@ -1243,6 +1243,11 @@ async def _collect(agen: AsyncIterator[object]) -> list[object]: return [item async for item in agen] +def _sse_text(out: Sequence[object]) -> str: + """Emitted SSE frames as text. Provider frames are bytes and a refusal frame is a str.""" + return "".join(item.decode() if isinstance(item, bytes) else item for item in out if isinstance(item, (bytes, str))) + + class _StreamWithHiddenParams: """Mimics CustomStreamWrapper: an async-iterable whose chunks are plain ModelResponseStream objects, but which itself carries _hidden_params (the provider response headers, snapshotted @@ -1476,6 +1481,73 @@ async def test_a_buffered_streamed_block_reaches_the_client_as_its_block_status( assert json.loads(bytes(response.body))["error"]["message"] == "leaked secret" +async def test_a_buffered_responses_stream_block_reaches_the_client_as_its_block_status( + monkeypatch: pytest.MonkeyPatch, +): + """The Responses error event carried a fixed 400, which the proxy answered with (TL-2934).""" + import litellm + from litellm.proxy.common_request_processing import create_response + from litellm.proxy.proxy_server import ( + async_data_generator, # pyright: ignore[reportUnknownVariableType] # its response parameter is untyped + ) + + g = _make_guardrail( + event_hook="post_call", + decisions=[_decision_response({"action": "block", "message": "leaked secret", "response_status": 403})], + ) + monkeypatch.setattr(litellm, "callbacks", [g]) + + response = await create_response( + async_data_generator( # pyright: ignore[reportArgumentType] # yields str | bytes; the proxy hands create_response the same generator + response=_aiter(_responses_stream_chunks()), + user_api_key_dict=UserAPIKeyAuth(), + request_data=_request_data(), + responses_stream_errors=True, + ), + media_type="text/event-stream", + headers={}, + ) + + assert response.status_code == 403 + assert json.loads(bytes(response.body))["error"]["message"] == "leaked secret" + + +async def test_a_buffered_messages_stream_block_reaches_the_client_as_its_block_status( + monkeypatch: pytest.MonkeyPatch, +): + """A /v1/messages block was a raw error frame under a 200, so an Anthropic client saw a + successful call that failed mid-stream (TL-2934). ``restamp_model`` is set the way the proxy + sets it for every /v1/messages stream.""" + import litellm + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing, create_response + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.utils import ProxyLogging + + g = _make_guardrail( + event_hook="post_call", + decisions=[_decision_response({"action": "block", "message": "leaked secret", "response_status": 403})], + ) + monkeypatch.setattr(litellm, "callbacks", [g]) + + response = await create_response( + ProxyBaseLLMRequestProcessing.async_sse_data_generator( # pyright: ignore[reportUnknownMemberType] # its request_data parameter is an untyped dict + response=_aiter(_anthropic_sse_frames()), + user_api_key_dict=UserAPIKeyAuth(), + request_data=_request_data(), + proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()), + restamp_model="claude-alias", + ), + media_type="text/event-stream", + headers={}, + ) + + assert response.status_code == 403 + assert json.loads(bytes(response.body)) == { + "type": "error", + "error": {"type": "permission_error", "message": "leaked secret"}, + } + + async def test_streaming_buffered_holds_chunks_until_decision(): call_order: list[str] = [] @@ -1591,9 +1663,9 @@ async def test_a_truncated_tool_argument_can_still_be_blocked(): request_data=_request_data(), ) ) - blob = b"".join(item for item in out if isinstance(item, bytes)) - assert b"secret in tool args" in blob - assert b"sk-live-TRUNC" not in blob + blob = _sse_text(out) + assert "secret in tool args" in blob + assert "sk-live-TRUNC" not in blob def _anthropic_sse_frames() -> list[bytes]: @@ -1806,7 +1878,11 @@ async def test_a_chat_shaped_rewrite_of_a_messages_stream_fails_closed(): async def test_streaming_raw_sse_block_emits_anthropic_error_frame(): - g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "leaked secret"})]) + from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame + + g = _make_guardrail( + decisions=[_decision_response({"action": "block", "message": "leaked secret", "response_status": 403})] + ) out = await _collect( g.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth(), @@ -1815,9 +1891,11 @@ async def test_streaming_raw_sse_block_emits_anthropic_error_frame(): ) ) assert len(out) == 1 - assert isinstance(out[0], bytes) - assert b"event: error" in out[0] - assert b"leaked secret" in out[0] + frame = out[0] + assert isinstance(frame, AnthropicErrorSseFrame) + assert frame.status_code == 403 + assert frame.startswith("event: error\n") + assert frame.error_response["error"] == {"type": "permission_error", "message": "leaked secret"} def _unscannable_chunks() -> list[dict]: @@ -2539,7 +2617,7 @@ async def test_a_buffered_messages_stream_gets_an_error_frame_when_the_service_f user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_anthropic_sse_frames()), request_data=_request_data() ) ) - frames = b"".join(item for item in out if isinstance(item, bytes)).decode() + frames = _sse_text(out) assert "event: error" in frames assert "ThirdLaw guardrail request failed" in frames assert "hello" not in frames