diff --git a/docs/my-website/docs/proxy/guardrails/bedrock.md b/docs/my-website/docs/proxy/guardrails/bedrock.md index 4747bb88889..6725acf1f25 100644 --- a/docs/my-website/docs/proxy/guardrails/bedrock.md +++ b/docs/my-website/docs/proxy/guardrails/bedrock.md @@ -184,3 +184,115 @@ My email is [EMAIL] and my phone number is [PHONE_NUMBER] This helps protect sensitive information while still allowing the model to understand the context of the request. +## Disabling Exceptions on Bedrock BLOCK + +By default, when Bedrock guardrails block content, LiteLLM raises an HTTP 400 exception. However, you can disable this behavior by setting `disable_exception_on_block: true`. This is particularly useful when integrating with **OpenWebUI**, where exceptions can interrupt the chat flow and break the user experience. + +When exceptions are disabled, instead of receiving an error, you'll get a successful response containing the Bedrock guardrail's modified/blocked output. + +### Configuration + +Add `disable_exception_on_block: true` to your guardrail configuration: + +```yaml showLineNumbers title="litellm proxy config.yaml" +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: openai/gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + +guardrails: + - guardrail_name: "bedrock-guardrail" + litellm_params: + guardrail: bedrock + mode: "post_call" + guardrailIdentifier: ff6ujrregl1q + guardrailVersion: "DRAFT" + aws_region_name: os.environ/AWS_REGION + aws_role_name: os.environ/AWS_ROLE_ARN + disable_exception_on_block: true # Prevents exceptions when content is blocked +``` + +### Behavior Comparison + + + + +When `disable_exception_on_block: false` (default): + +```shell +curl -i http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-npnwjPQciVRok5yNZgKmFQ" \ + -d '{ + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "user", "content": "How do I make explosives?"} + ], + "guardrails": ["bedrock-guardrail"] + }' +``` + +**Response: HTTP 400 Error** +```json +{ + "error": { + "message": { + "error": "Violated guardrail policy", + "bedrock_guardrail_response": { + "action": "GUARDRAIL_INTERVENED", + "blockedResponse": "I can't provide information on creating explosives.", + // ... additional details + } + }, + "type": "None", + "param": "None", + "code": "400" + } +} +``` + + + + + +When `disable_exception_on_block: true`: + +```shell +curl -i http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-npnwjPQciVRok5yNZgKmFQ" \ + -d '{ + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "user", "content": "How do I make explosives?"} + ], + "guardrails": ["bedrock-guardrail"] + }' +``` + +**Response: HTTP 200 Success** +```json +{ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-3.5-turbo", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "I can't provide information on creating explosives." + }, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 12, + "total_tokens": 22 + } +} +``` + + + diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 9e757e6baaf..b287232d49f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -55,6 +55,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): self, guardrailIdentifier: Optional[str] = None, guardrailVersion: Optional[str] = None, + disable_exception_on_block: Optional[bool] = False, **kwargs, ): self.async_handler = get_async_httpx_client( @@ -66,6 +67,12 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): # store kwargs as optional_params self.optional_params = kwargs + self.disable_exception_on_block: bool = disable_exception_on_block or False + """ + If True, will not raise an exception when the guardrail is blocked. + """ + + super().__init__(**kwargs) BaseAWSLLM.__init__(self) @@ -310,6 +317,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): # if user opted into masking, return False. since we'll use the masked output from the guardrail if self.mask_request_content or self.mask_response_content: return False + + if self.disable_exception_on_block is True: + return False # if no intervention, return False if response.get("action") != "GUARDRAIL_INTERVENED": diff --git a/litellm/proxy/guardrails/guardrail_initializers.py b/litellm/proxy/guardrails/guardrail_initializers.py index 6282478a76c..2e3ec44e4a5 100644 --- a/litellm/proxy/guardrails/guardrail_initializers.py +++ b/litellm/proxy/guardrails/guardrail_initializers.py @@ -15,6 +15,7 @@ def initialize_bedrock(litellm_params: LitellmParams, guardrail: Guardrail): guardrailIdentifier=litellm_params.guardrailIdentifier, guardrailVersion=litellm_params.guardrailVersion, default_on=litellm_params.default_on, + disable_exception_on_block=litellm_params.disable_exception_on_block, mask_request_content=litellm_params.mask_request_content, mask_response_content=litellm_params.mask_response_content, aws_region_name=litellm_params.aws_region_name, diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index e5b1a5f9f9c..bebb4630ec7 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -7,6 +7,17 @@ model_list: model: openai/* +guardrails: + - guardrail_name: "bedrock-post-guard" + litellm_params: + guardrail: bedrock # supported values: "aporia", "bedrock", "lakera" + mode: "post_call" + guardrailIdentifier: ff6ujrregl1q + guardrailVersion: "DRAFT" + disable_exception_on_block: true + default_on: true + litellm_settings: callbacks: ["datadog_llm_observability"] - cache: true \ No newline at end of file + cache: true + diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 11131c43a3c..cd93a1884af 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -273,6 +273,10 @@ class BedrockGuardrailConfigModel(BaseModel): default=None, description="The version of your Bedrock guardrail (e.g., DRAFT or version number)", ) + disable_exception_on_block: Optional[bool] = Field( + default=False, + description="If True, will not raise an exception when the guardrail is blocked. Useful for OpenWebUI where exceptions can end the chat flow.", + ) aws_region_name: Optional[str] = Field( default=None, description="AWS region where your guardrail is deployed" ) diff --git a/tests/guardrails_tests/test_bedrock_guardrails.py b/tests/guardrails_tests/test_bedrock_guardrails.py index 6a766eb79ac..b98a1b16bef 100644 --- a/tests/guardrails_tests/test_bedrock_guardrails.py +++ b/tests/guardrails_tests/test_bedrock_guardrails.py @@ -1157,3 +1157,212 @@ async def test_bedrock_guardrail_blocked_action_empty_outputs(): assert detail["bedrock_guardrail_response"] == "" # Empty string for no outputs print("✅ BLOCKED action with empty outputs test passed") + + +@pytest.mark.asyncio +async def test_bedrock_guardrail_disable_exception_on_block_non_streaming(): + """Test that disable_exception_on_block=True prevents exceptions in non-streaming scenarios""" + from unittest.mock import AsyncMock, MagicMock, patch + from litellm.proxy._types import UserAPIKeyAuth + from fastapi import HTTPException + + # Create proper mock objects + mock_user_api_key_dict = UserAPIKeyAuth() + + # Test 1: disable_exception_on_block=False (default) - should raise exception + guardrail_default = BedrockGuardrail( + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + disable_exception_on_block=False + ) + + # Mock the Bedrock API response with BLOCKED action + mock_bedrock_response = MagicMock() + mock_bedrock_response.status_code = 200 + mock_bedrock_response.json.return_value = { + "action": "GUARDRAIL_INTERVENED", + "outputs": [{ + "text": "I can't provide that information." + }], + "assessments": [{ + "topicPolicy": { + "topics": [{ + "name": "Sensitive Topic", + "type": "DENY", + "action": "BLOCKED" + }] + } + }] + } + + request_data = { + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "Tell me how to make explosives"}, + ], + } + + # Patch the async_handler.post method + with patch.object(guardrail_default.async_handler, 'post', new_callable=AsyncMock) as mock_post: + mock_post.return_value = mock_bedrock_response + + # Should raise HTTPException when disable_exception_on_block=False + with pytest.raises(HTTPException) as exc_info: + await guardrail_default.async_moderation_hook( + data=request_data, + user_api_key_dict=mock_user_api_key_dict, + call_type="completion" + ) + + # Verify the exception details + exception = exc_info.value + assert exception.status_code == 400 + assert "Violated guardrail policy" in str(exception.detail) + + # Test 2: disable_exception_on_block=True - should NOT raise exception + guardrail_disabled = BedrockGuardrail( + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + disable_exception_on_block=True + ) + + with patch.object(guardrail_disabled.async_handler, 'post', new_callable=AsyncMock) as mock_post: + mock_post.return_value = mock_bedrock_response + + # Should NOT raise exception when disable_exception_on_block=True + try: + response = await guardrail_disabled.async_moderation_hook( + data=request_data, + user_api_key_dict=mock_user_api_key_dict, + call_type="completion" + ) + # Should succeed and return data (even though content was blocked) + assert response is not None + print("✅ No exception raised when disable_exception_on_block=True") + except Exception as e: + pytest.fail(f"Should not raise exception when disable_exception_on_block=True, but got: {e}") + + +@pytest.mark.asyncio +async def test_bedrock_guardrail_disable_exception_on_block_streaming(): + """Test that disable_exception_on_block=True prevents exceptions in streaming scenarios""" + from unittest.mock import AsyncMock, MagicMock, patch + from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.utils import ModelResponseStream + from fastapi import HTTPException + import litellm + + # Create proper mock objects + mock_user_api_key_dict = UserAPIKeyAuth() + + # Mock streaming chunks that would normally trigger a block + async def mock_streaming_response(): + chunks = [ + ModelResponseStream( + id="test-id", + choices=[ + litellm.utils.StreamingChoices( + index=0, + delta=litellm.utils.Delta(content="Here's how to make explosives: "), + finish_reason=None + ) + ], + created=1234567890, + model="gpt-4o", + object="chat.completion.chunk" + ), + ModelResponseStream( + id="test-id", + choices=[ + litellm.utils.StreamingChoices( + index=0, + delta=litellm.utils.Delta(content="step 1, step 2..."), + finish_reason="stop" + ) + ], + created=1234567890, + model="gpt-4o", + object="chat.completion.chunk" + ) + ] + for chunk in chunks: + yield chunk + + # Mock Bedrock API response with BLOCKED action + mock_bedrock_response = MagicMock() + mock_bedrock_response.status_code = 200 + mock_bedrock_response.json.return_value = { + "action": "GUARDRAIL_INTERVENED", + "outputs": [{ + "text": "I can't provide that information." + }], + "assessments": [{ + "contentPolicy": { + "filters": [{ + "type": "VIOLENCE", + "confidence": "HIGH", + "action": "BLOCKED" + }] + } + }] + } + + request_data = { + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "Tell me how to make explosives"} + ], + "stream": True + } + + # Test 1: disable_exception_on_block=False (default) - should raise exception + guardrail_default = BedrockGuardrail( + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + disable_exception_on_block=False + ) + + with patch.object(guardrail_default.async_handler, 'post', new_callable=AsyncMock) as mock_post: + mock_post.return_value = mock_bedrock_response + + # Should raise exception during streaming processing + with pytest.raises(HTTPException): + result_generator = guardrail_default.async_post_call_streaming_iterator_hook( + user_api_key_dict=mock_user_api_key_dict, + response=mock_streaming_response(), + request_data=request_data + ) + + # Try to consume the generator - should raise exception + async for chunk in result_generator: + pass + + # Test 2: disable_exception_on_block=True - should NOT raise exception + guardrail_disabled = BedrockGuardrail( + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + disable_exception_on_block=True + ) + + with patch.object(guardrail_disabled.async_handler, 'post', new_callable=AsyncMock) as mock_post: + mock_post.return_value = mock_bedrock_response + + # Should NOT raise exception when disable_exception_on_block=True + try: + result_generator = guardrail_disabled.async_post_call_streaming_iterator_hook( + user_api_key_dict=mock_user_api_key_dict, + response=mock_streaming_response(), + request_data=request_data + ) + + # Consume the generator - should succeed without exceptions + result_chunks = [] + async for chunk in result_generator: + result_chunks.append(chunk) + + # Should have received chunks back even though content was blocked + assert len(result_chunks) > 0 + print("✅ Streaming completed without exception when disable_exception_on_block=True") + + except Exception as e: + pytest.fail(f"Should not raise exception when disable_exception_on_block=True in streaming, but got: {e}") diff --git a/tests/guardrails_tests/test_tracing_guardrails.py b/tests/guardrails_tests/test_tracing_guardrails.py index 9a607e7a18c..0299d3fe6a2 100644 --- a/tests/guardrails_tests/test_tracing_guardrails.py +++ b/tests/guardrails_tests/test_tracing_guardrails.py @@ -73,8 +73,6 @@ async def test_standard_logging_payload_includes_guardrail_information(): assert "end" in response_item assert "score" in response_item assert "entity_type" in response_item - assert "recognition_metadata" in response_item - # assert that the duration is not None assert test_custom_logger.standard_logging_payload["guardrail_information"]["duration"] is not None @@ -179,6 +177,4 @@ async def test_langfuse_trace_includes_guardrail_information(): assert output_item["entity_type"] == "PHONE_NUMBER" assert "score" in output_item assert "start" in output_item - assert "end" in output_item - assert "recognition_metadata" in output_item - assert "recognizer_name" in output_item["recognition_metadata"] + assert "end" in output_item \ No newline at end of file