diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index c72ab907897..093cc46bc10 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -210,6 +210,12 @@ def _post_guardrail_tool_call_shapes( returned_shapes: Final = tuple(_returned_tool_call_shape(tool_call) for tool_call in returned_tool_calls) validated_shapes: Final = tuple(shape for shape in returned_shapes if shape is not None) if len(validated_shapes) != len(returned_shapes): + if deliver_ended_stream_rewrites: + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + raise UndeliverableStreamRewrite( + guardrail_name or "unknown", "the guardrail returned a tool call without a function name and arguments" + ) verbose_proxy_logger.warning( "OpenAI Responses API: guardrail %s returned tool calls without a function name and arguments, " "leaving the tool call output items unchanged", diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index a752464081f..71f7e1841f3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -1328,6 +1328,18 @@ class UnifiedLLMGuardrails(CustomLogger): async for remaining_item in response: yield remaining_item return + if ( + chunk_counter == 1 + and buffer_until_moderated + and guardrail_to_apply.mask_response_content + and not mappings[CallTypes(call_type)]().delivers_ended_stream_rewrites + ): + verbose_proxy_logger.warning( + "UnifiedLLMGuardrails: guardrail %s masks response content, but the %s translation cannot write " + "a rewrite back into a buffered stream, so the scan can only block or release the original chunks", + guardrail_to_apply.guardrail_name, + call_type, + ) # If end_of_stream_only mode, yield chunks without processing. # When buffering, withhold them instead -- they are released (or diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_streaming_buffer_until_moderated.py b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_streaming_buffer_until_moderated.py index 81cb8754529..cb33a0c1c35 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_streaming_buffer_until_moderated.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_streaming_buffer_until_moderated.py @@ -624,6 +624,18 @@ class _ToolClearingGuardrail(_MaskingGuardrail): return {**masked, "tool_calls": tool_calls or []} +class _ToolManglingGuardrail(_MaskingGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Any | None = None, + ) -> GenericGuardrailAPIInputs: + masked = await super().apply_guardrail(inputs, request_data, input_type, logging_obj) + return {**masked, "tool_calls": [{"type": "function"} for _ in inputs.get("tool_calls") or []]} + + async def _anthropic_tool_use_stream(): yield _sse_event( "message_start", @@ -853,8 +865,11 @@ async def test_buffered_responses_stream_delivers_masked_text_to_the_client(): @pytest.mark.asyncio -async def test_buffered_responses_stream_withholds_everything_when_the_guardrail_drops_a_tool_call(): - guardrail = _ToolDroppingGuardrail(guardrail_name="dropper", event_hook="post_call") +@pytest.mark.parametrize( + "guardrail_cls", [_ToolDroppingGuardrail, _ToolManglingGuardrail], ids=["returns fewer", "returns a nameless call"] +) +async def test_buffered_responses_stream_withholds_everything_when_the_guardrail_drops_a_tool_call(guardrail_cls: type): + guardrail = guardrail_cls(guardrail_name="dropper", event_hook="post_call") guardrail.streaming_buffer_until_moderated = True guardrail.streaming_buffer_release_on_scan = False unified = UnifiedLLMGuardrails()