From 91c6fa975db9533601f4443b81f699d5ca625c66 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 16 Jun 2026 18:56:14 -0700 Subject: [PATCH] fix(guardrails): return 400 not 500 when AIM blocks a request (#30573) * fix(guardrails): return 400 not 500 when AIM blocks a request AIM guardrail blocks raised a bare HTTPException whose type and param serialized as the literal string "None", which broke OpenAI-SDK error parsing for downstream consumers. Switching AIM to raise a ProxyException surfaced a second bug: the shared error funnel re-derived the HTTP status from a nonexistent status_code attribute and downgraded the 400 to a 500. The funnel now honors an already-normalized ProxyException rather than rebuilding it, and ProxyException is excluded from llm_exceptions alerting so a content-policy block no longer pages on-call as an LLM API failure Resolves LIT-3751 * fix(guardrails): route all AIM rejection paths through ProxyException The block-action fix left two AIM rejection paths raising a bare HTTPException: the multimodal anonymize rejection and the output-side block. Both serialized type and param as the literal string "None", the same malformed shape the block fix removed. Funnel all three through a shared _rejection helper so they return a conformant OpenAI error body. The output block carries content_policy_violation; the multimodal rejection stays a plain invalid_request_error because it is a usage error, not a policy violation Resolves LIT-3751 * fix(guardrails): record AIM ProxyException blocks in failure logs Switching AIM blocks from HTTPException to ProxyException made _is_proxy_only_llm_api_error return False for them, so _handle_logging_proxy_only_error was skipped and the blocked prompt was dropped from the configured failure loggers. Classify ProxyException as a proxy-only error alongside HTTPException so guardrail blocks are recorded again, matching the prior behavior. The llm_exceptions alert suppression is a separate check and stays in place Resolves LIT-3751 * style(guardrails): use str | None over Optional[str] in AIM _rejection * style(guardrails): collapse AIM _rejection signature per black (cherry picked from commit b5fcd859bec1388267f6f1f9affc125190555525) --- litellm/proxy/_types.py | 1 + litellm/proxy/common_request_processing.py | 7 + .../guardrails/guardrail_hooks/aim/aim.py | 34 +++-- litellm/proxy/utils.py | 5 +- tests/local_testing/test_aim_guardrails.py | 135 +++++++++++++++++- .../proxy/test_common_request_processing.py | 35 +++++ tests/test_litellm/proxy/test_proxy_utils.py | 114 +++++++++++++++ 7 files changed, 313 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9a34e6359c5..09cf9656b93 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3683,6 +3683,7 @@ class ProxyException(Exception): provider_specific_fields: Optional[dict] = None, ): self.message = str(message) + super().__init__(self.message) self.type = type self.param = param self.openai_code = openai_code or code diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index b8915d6662b..4e26af7fe03 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1799,6 +1799,13 @@ class ProxyBaseLLMRequestProcessing: except Exception: pass + if isinstance(e, ProxyException): + e.headers = { + **e.headers, + **{k: v if isinstance(v, str) else str(v) for k, v in headers.items()}, + } + raise e + if isinstance(e, HTTPException): raw_detail = getattr(e, "detail", str(e)) message, structured_fields = _serialize_http_exception_detail(raw_detail) diff --git a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py index 5b5f91195e7..d70c8e4f310 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py +++ b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py @@ -9,7 +9,6 @@ import json import os from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional, Type, Union -from fastapi import HTTPException from pydantic import BaseModel from websockets.asyncio.client import ClientConnection, connect @@ -21,7 +20,7 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.guardrails._content_utils import ( apply_redacted_messages_back, build_inspection_messages, @@ -129,6 +128,16 @@ class AimGuardrail(CustomGuardrail): verbose_proxy_logger.error(f"Aim: {action_type} action") return data + @staticmethod + def _rejection(message: str, *, openai_code: str | None = None) -> ProxyException: + return ProxyException( + message=message, + type="invalid_request_error", + param=None, + code=400, + openai_code=openai_code, + ) + def _handle_block_action(self, analysis_result: Any, required_action: Any) -> None: detection_message = required_action.get("detection_message", None) verbose_proxy_logger.info( @@ -136,7 +145,7 @@ class AimGuardrail(CustomGuardrail): policies=list(analysis_result["policy_drill_down"].keys()), ), ) - raise HTTPException(status_code=400, detail=detection_message) + raise self._rejection(detection_message, openai_code="content_policy_violation") def _anonymize_request(self, res: Any, data: dict) -> dict: verbose_proxy_logger.info("Aim: anonymize action") @@ -148,14 +157,11 @@ class AimGuardrail(CustomGuardrail): # parts from a multimodal request — degrade to block so the # multimodal payload is never silently rewritten. if has_non_string_content(data): - raise HTTPException( - status_code=400, - detail=( - "Aim: anonymize action requested for multimodal input " - "but mask-in-place would drop non-text parts. Send the " - "request with plain string content to use anonymize, " - "or rely on block-mode policies." - ), + raise self._rejection( + "Aim: anonymize action requested for multimodal input " + "but mask-in-place would drop non-text parts. Send the " + "request with plain string content to use anonymize, " + "or rely on block-mode policies." ) redacted_messages = [ { @@ -287,9 +293,9 @@ class AimGuardrail(CustomGuardrail): if aim_output_guardrail_result and aim_output_guardrail_result.get( "detection_message" ): - raise HTTPException( - status_code=400, - detail=aim_output_guardrail_result.get("detection_message"), + raise self._rejection( + aim_output_guardrail_result.get("detection_message"), + openai_code="content_policy_violation", ) if aim_output_guardrail_result and aim_output_guardrail_result.get( "redacted_output" diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8d5fdb10a51..effd62da717 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1967,7 +1967,7 @@ class ProxyLogging: litellm_call_id=request_data.get("litellm_call_id", ""), status="fail" ) if AlertType.llm_exceptions in self.alert_types and not isinstance( - original_exception, HTTPException + original_exception, (HTTPException, ProxyException) ): """ Just alert on LLM API exceptions. Do not alert on user errors @@ -2071,6 +2071,7 @@ class ProxyLogging: e.g should only return True for: - Authentication Errors from user_api_key_auth - HTTP HTTPException (rate limit errors) + - ProxyException (guardrail blocks, budget / rate-limit errors) """ ######################################################### @@ -2087,7 +2088,7 @@ class ProxyLogging: ): return False - return isinstance(original_exception, HTTPException) or ( + return isinstance(original_exception, (HTTPException, ProxyException)) or ( error_type == ProxyErrorTypes.auth_error ) diff --git a/tests/local_testing/test_aim_guardrails.py b/tests/local_testing/test_aim_guardrails.py index 31416c565c1..2cb7f9cd357 100644 --- a/tests/local_testing/test_aim_guardrails.py +++ b/tests/local_testing/test_aim_guardrails.py @@ -6,10 +6,10 @@ import sys from unittest.mock import AsyncMock, patch, call import pytest -from fastapi.exceptions import HTTPException from httpx import Request, Response from litellm import DualCache +from litellm.proxy._types import ProxyException from litellm.proxy.guardrails.guardrail_hooks.aim.aim import ( AimGuardrail, AimGuardrailMissingSecrets, @@ -101,7 +101,7 @@ async def test_block_callback(mode: str): ], } - with pytest.raises(HTTPException, match="Jailbreak detected"): + with pytest.raises(ProxyException, match="Jailbreak detected") as exc_info: with patch( "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", return_value=Response( @@ -135,6 +135,137 @@ async def test_block_callback(mode: str): call_type="completion", ) + exc = exc_info.value + assert exc.code == "400" + assert exc.type == "invalid_request_error" + assert exc.param is None + assert exc.openai_code == "content_policy_violation" + + +@pytest.mark.asyncio +async def test_output_block_raises_proxy_exception(): + """An output-side block is a content-policy violation, like the input block: + it must surface a conformant ProxyException, not a bare HTTPException whose + type/param serialize as the literal string "None". Regression for LIT-3751.""" + init_guardrails_v2( + all_guardrails=[ + { + "guardrail_name": "gibberish-guard", + "litellm_params": { + "guardrail": "aim", + "mode": "post_call", + "api_key": "hs-aim-key", + }, + }, + ], + config_file_path="", + ) + aim_guardrails = [ + callback for callback in litellm.callbacks if isinstance(callback, AimGuardrail) + ] + assert len(aim_guardrails) == 1 + aim_guardrail = aim_guardrails[0] + + block_on_output = Response( + json={ + "analysis_result": {"policy_drill_down": {"PII": {}}}, + "required_action": { + "action_type": "block_action", + "detection_message": "Output blocked: leaked secret", + "policy_name": "blocking policy", + }, + }, + status_code=200, + request=Request(method="POST", url="http://aim"), + ) + response = ModelResponse( + choices=[ + { + "finish_reason": "stop", + "index": 0, + "message": {"content": "here is the secret", "role": "assistant"}, + } + ] + ) + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=block_on_output, + ): + with pytest.raises(ProxyException, match="Output blocked") as exc_info: + await aim_guardrail.async_post_call_success_hook( + data={"messages": [{"role": "user", "content": "tell me a secret"}]}, + response=response, + user_api_key_dict=UserAPIKeyAuth(), + ) + + exc = exc_info.value + assert exc.code == "400" + assert exc.type == "invalid_request_error" + assert exc.param is None + assert exc.openai_code == "content_policy_violation" + + +@pytest.mark.asyncio +async def test_anonymize_multimodal_rejection_raises_proxy_exception(): + """Anonymize on multimodal input degrades to a 400 because mask-in-place would + drop non-text parts. That is a usage error, not a content-policy violation, so + it must raise a conformant ProxyException WITHOUT the content_policy_violation + code. Regression for LIT-3751.""" + init_guardrails_v2( + all_guardrails=[ + { + "guardrail_name": "gibberish-guard", + "litellm_params": { + "guardrail": "aim", + "mode": "pre_call", + "api_key": "hs-aim-key", + }, + }, + ], + config_file_path="", + ) + aim_guardrails = [ + callback for callback in litellm.callbacks if isinstance(callback, AimGuardrail) + ] + assert len(aim_guardrails) == 1 + aim_guardrail = aim_guardrails[0] + + data = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hi my name is Brian"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, + }, + ], + }, + ], + } + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=response_with_detections, + ): + with pytest.raises( + ProxyException, match="anonymize action requested for multimodal" + ) as exc_info: + await aim_guardrail.async_pre_call_hook( + data=data, + cache=DualCache(), + user_api_key_dict=UserAPIKeyAuth(), + call_type="completion", + ) + + exc = exc_info.value + assert exc.code == "400" + assert exc.type == "invalid_request_error" + assert exc.param is None + assert exc.openai_code != "content_policy_violation" + @pytest.mark.asyncio @pytest.mark.parametrize("mode", ["pre_call", "during_call"]) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 265a82d4a44..657529c85f1 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -2244,6 +2244,41 @@ class TestHandleLLMApiExceptionDictDetail: assert proxy_exc.message == "Content blocked by guardrail" assert proxy_exc.provider_specific_fields is None + async def test_already_normalized_proxy_exception_is_honored(self): + """A ProxyException raised mid-request (e.g. a guardrail block) is already + the OpenAI wire format. The funnel must re-raise it untouched instead of + re-deriving the status from a (nonexistent) status_code attribute and + defaulting to 500. Regression for LIT-3751.""" + from litellm.proxy._types import ProxyException + + exc = ProxyException( + message='"Leroy Jenkins" detected as name', + type="invalid_request_error", + param=None, + code=400, + openai_code="content_policy_violation", + ) + proxy_exc = await self._invoke(exc) + assert proxy_exc is exc + assert proxy_exc.code == "400" + assert proxy_exc.type == "invalid_request_error" + assert proxy_exc.param is None + assert proxy_exc.openai_code == "content_policy_violation" + assert proxy_exc.message == '"Leroy Jenkins" detected as name' + + # The body the OpenAI-SDK client actually receives. The HTTP status line + # comes from int(exc.code) == 400; the wire ``code`` stays the status + # string. ``openai_code`` ("content_policy_violation") is intentionally + # NOT serialized here - to_dict() emits only ``code`` - so this asserts + # the real contract rather than the write-only attribute. + assert int(proxy_exc.code) == 400 + assert proxy_exc.to_dict() == { + "message": '"Leroy Jenkins" detected as name', + "type": "invalid_request_error", + "param": None, + "code": "400", + } + class TestAsyncStreamingDataGeneratorFastPath: """Fast/slow path branching in async_streaming_data_generator.""" diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 7a2b20bd8fb..539a32db57d 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -321,3 +321,117 @@ class TestPostCallFailureHookLiftsFirstApiCallStartTime: await self._run(request_data) assert "first_api_call_start_time" not in request_data assert "litellm_logging_obj" not in request_data + + +class TestPostCallFailureHookLLMExceptionAlerting: + """The llm_exceptions alert is for infra / LLM-API failures, not user + errors (https://github.com/BerriAI/litellm/issues/3395). Already-normalized + client errors must be excluded so a guardrail content-policy block never + pages on-call. ProxyException is such an error; before LIT-3751 only + HTTPException was excluded, so AIM blocks paged as if the LLM API failed.""" + + async def _alerted(self, exc) -> bool: + import asyncio + from unittest.mock import AsyncMock, patch + + from litellm.proxy._types import AlertType, UserAPIKeyAuth + + proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.alert_types = [AlertType.llm_exceptions] + alerting_handler = AsyncMock() + with ( + patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()), + patch.object(proxy_logging_obj, "alerting_handler", new=alerting_handler), + ): + await proxy_logging_obj.post_call_failure_hook( + request_data={}, + original_exception=exc, + user_api_key_dict=UserAPIKeyAuth(), + ) + await asyncio.sleep(0) # let the fire-and-forget alert task run + return alerting_handler.called + + @pytest.mark.asyncio + async def test_proxy_exception_does_not_alert(self): + from litellm.proxy._types import ProxyException + + exc = ProxyException( + message="content blocked", + type="invalid_request_error", + param=None, + code=400, + openai_code="content_policy_violation", + ) + assert await self._alerted(exc) is False + + @pytest.mark.asyncio + async def test_http_exception_does_not_alert(self): + assert ( + await self._alerted(HTTPException(status_code=400, detail="blocked")) + is False + ) + + @pytest.mark.asyncio + async def test_genuine_llm_api_error_still_alerts(self): + assert await self._alerted(Exception("upstream 503")) is True + + +class TestPostCallFailureHookProxyExceptionLogging: + """A guardrail block raises a ProxyException; on an LLM route it must still + drive proxy-only failure logging (_handle_logging_proxy_only_error) so the + blocked request is recorded, exactly as the old HTTPException did. Before + LIT-3751 the classifier only matched HTTPException, so switching AIM to + ProxyException silently dropped the rejected prompt from failure logs.""" + + async def _logged(self, exc, *, request_route) -> bool: + from unittest.mock import AsyncMock, patch + + from litellm.proxy._types import UserAPIKeyAuth + + proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.alert_types = [] + handle_mock = AsyncMock() + with ( + patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()), + patch.object( + proxy_logging_obj, + "_handle_logging_proxy_only_error", + new=handle_mock, + ), + ): + await proxy_logging_obj.post_call_failure_hook( + request_data={}, + original_exception=exc, + user_api_key_dict=UserAPIKeyAuth( + api_key="sk-test", request_route=request_route + ), + ) + return handle_mock.await_count > 0 + + def _block(self): + from litellm.proxy._types import ProxyException + + return ProxyException( + message="content blocked", + type="invalid_request_error", + param=None, + code=400, + openai_code="content_policy_violation", + ) + + @pytest.mark.asyncio + async def test_proxy_exception_on_llm_route_is_logged(self): + assert ( + await self._logged(self._block(), request_route="/v1/chat/completions") + is True + ) + + @pytest.mark.asyncio + async def test_generic_exception_on_llm_route_is_not_logged(self): + # A raw provider/unknown exception is logged by the LLM call path, not here. + assert ( + await self._logged( + Exception("upstream 503"), request_route="/v1/chat/completions" + ) + is False + )