diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index e812b03385d..371a63d2e40 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -52,7 +52,7 @@ from litellm.constants import ( STREAM_SSE_KEEPALIVE_PING_BYTES, UNSAFE_PROXY_RESPONSE_HEADERS, ) -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention from litellm.integrations.otel.runtime import phase_event from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure @@ -286,6 +286,7 @@ from litellm.proxy.litellm_pre_call_utils import ( LiteLLMProxyRequestSetup, add_litellm_data_to_request, get_metadata_variable_name, + redact_blocked_request_body_snapshot, refresh_proxy_server_request_body_snapshot, reject_url_valued_destination, sanitize_for_log, @@ -2670,12 +2671,17 @@ class ProxyBaseLLMRequestProcessing: prefetch_model = self.data.get("model") if llm_router is not None and isinstance(prefetch_model, str): RoutingContext(llm_router).arm_read_prefetch(prefetch_model, self.data) - self.data = await proxy_logging_obj.pre_call_hook( - user_api_key_dict=user_api_key_dict, - data=self.data, - call_type=route_type, - skip_guardrails=skip_guardrails, - ) + try: + self.data = await proxy_logging_obj.pre_call_hook( + user_api_key_dict=user_api_key_dict, + data=self.data, + call_type=route_type, + skip_guardrails=skip_guardrails, + ) + except Exception as e: + if is_guardrail_intervention(e): + redact_blocked_request_body_snapshot(self.data) + raise await _enforce_guardrail_added_tag_budgets( data=self.data, tags_before_guardrails=self._tags_before_guardrails, @@ -2846,7 +2852,9 @@ class ProxyBaseLLMRequestProcessing: if is_hard_per_model_limit(fallback_exc): raise continue - except BaseException: + except BaseException as e: + if is_guardrail_intervention(e): + redact_blocked_request_body_snapshot(rate_limited_data) self.data = rate_limited_data raise diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 4444d6d25f9..57c80892b37 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2608,6 +2608,25 @@ def refresh_proxy_server_request_body_snapshot( ) +def redact_blocked_request_body_snapshot(data: MutableMapping[str, object]) -> None: + """ + Replace the stored ``proxy_server_request`` body with a redacted placeholder. + + A guardrail that blocks in pre_call_hook raises before the post-guardrail + refresh runs, so the snapshot taken by add_litellm_data_to_request still + holds the raw pre-guardrail body, including whatever got the request blocked. + """ + proxy_server_request: Final = data.get("proxy_server_request") + if not isinstance(proxy_server_request, dict): + return + model: Final = data.get("model") + proxy_server_request["body"] = ( + {"model": model, "[redacted]": "blocked by guardrail"} + if isinstance(model, str) + else {"[redacted]": "blocked by guardrail"} + ) + + async def add_litellm_data_to_request( data: dict, request: Request, diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 5c27ad8ce77..e143a233030 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -22,7 +22,7 @@ from fastapi.responses import JSONResponse, StreamingResponse import litellm from litellm._uuid import uuid -from litellm.exceptions import APIError, InternalServerError, MidStreamFallbackError +from litellm.exceptions import APIError, GuardrailRaisedException, InternalServerError, MidStreamFallbackError from litellm.litellm_core_utils.litellm_logging import Logging from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame, anthropic_error_sse_frame from litellm.caching.caching import DualCache @@ -481,6 +481,105 @@ class TestProxyBaseLLMRequestProcessing: pytest.fail("litellm_call_id is not a valid UUID") assert data_passed["litellm_call_id"] == returned_data["litellm_call_id"] + @pytest.mark.asyncio + @pytest.mark.parametrize( + "block_exc", + [ + GuardrailRaisedException( + guardrail_name="block-secrets", + message="Content blocked: secret detected", + blocked_content=True, + ), + HTTPException(status_code=400, detail="Content blocked: secret detected"), + ], + ids=["guardrail-raised-exception", "content-filter-http-exception"], + ) + async def test_common_processing_pre_call_logic_redacts_blocked_request_body_on_guardrail_block( + self, monkeypatch, block_exc + ): + secret = "glpat-FAKE-NOT-A-REAL-TOKEN-1111" + raw_body = { + "model": "small", + "messages": [{"role": "user", "content": f"my token is {secret}"}], + "max_tokens": 20, + } + data = {"model": "small", "messages": raw_body["messages"]} + + async def mock_add_litellm_data_to_request(*args, **kwargs): + data["proxy_server_request"] = { + "url": "http://localhost/v1/chat/completions", + "method": "POST", + "body": dict(raw_body), + } + return data + + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "add_litellm_data_to_request", + mock_add_litellm_data_to_request, + ) + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=block_exc) + mock_request = MagicMock(spec=Request) + mock_request.headers = {} + processing_obj = ProxyBaseLLMRequestProcessing(data=data) + + with pytest.raises(type(block_exc)): + await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(), + proxy_logging_obj=mock_proxy_logging_obj, + proxy_config=MagicMock(spec=ProxyConfig), + route_type="acompletion", + ) + + assert processing_obj.data["proxy_server_request"]["body"] == { + "model": "small", + "[redacted]": "blocked by guardrail", + }, "blocked request body still holds the raw prompt in the spend-log snapshot" + + @pytest.mark.asyncio + async def test_common_processing_pre_call_logic_keeps_body_on_non_guardrail_pre_call_failure(self, monkeypatch): + raw_body = { + "model": "small", + "messages": [{"role": "user", "content": "hello"}], + } + data = {"model": "small", "messages": raw_body["messages"]} + + async def mock_add_litellm_data_to_request(*args, **kwargs): + data["proxy_server_request"] = { + "url": "http://localhost/v1/chat/completions", + "method": "POST", + "body": dict(raw_body), + } + return data + + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "add_litellm_data_to_request", + mock_add_litellm_data_to_request, + ) + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=ValueError("boom")) + mock_request = MagicMock(spec=Request) + mock_request.headers = {} + processing_obj = ProxyBaseLLMRequestProcessing(data=data) + + with pytest.raises(ValueError, match="boom"): + await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(), + proxy_logging_obj=mock_proxy_logging_obj, + proxy_config=MagicMock(spec=ProxyConfig), + route_type="acompletion", + ) + + assert processing_obj.data["proxy_server_request"]["body"] == raw_body, ( + "non-guardrail pre_call failure must not rewrite the stored body" + ) + @pytest.mark.asyncio @pytest.mark.parametrize("requested_model", [{"bad": "value"}, ["gpt-5.2"], 1]) async def test_common_processing_pre_call_logic_rejects_a_non_string_model_with_400( @@ -7876,6 +7975,77 @@ class TestPreCallWithFallbacksOnLocalRateLimit: assert processor.data["model"] == fallback_model assert call_count == 2 + @pytest.mark.asyncio + async def test_fallback_guardrail_block_redacts_restored_request_body(self): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + from litellm.proxy.litellm_pre_call_utils import redact_blocked_request_body_snapshot + + primary_model = "gpt-4" + fallback_model = "gpt-3.5-turbo" + secret = "glpat-FAKE-NOT-A-REAL-TOKEN-2222" + raw_body = { + "model": primary_model, + "messages": [{"role": "user", "content": f"my token is {secret}"}], + } + processor = ProxyBaseLLMRequestProcessing( + data={ + "model": primary_model, + "messages": raw_body["messages"], + "proxy_server_request": { + "url": "http://localhost/v1/chat/completions", + "method": "POST", + "body": dict(raw_body), + }, + } + ) + + async def mock_pre_call_logic(**kwargs): + if processor.data.get("model") == primary_model: + raise ProxyRateLimitError( + detail="TPM limit exceeded for gpt-4", + headers={"retry-after": "30"}, + ) + redact_blocked_request_body_snapshot(processor.data) + raise GuardrailRaisedException( + guardrail_name="block-secrets", + message="Content blocked: secret detected", + blocked_content=True, + ) + + mock_router = route_entry_points_to_stubs(MagicMock()) + mock_router.fallbacks = [{primary_model: [fallback_model]}] + + with ( + patch.object( + processor, + "common_processing_pre_call_logic", + side_effect=mock_pre_call_logic, + ), + pytest.raises(GuardrailRaisedException), + ): + await processor._pre_call_with_fallbacks( + request=MagicMock(), + general_settings={}, + proxy_logging_obj=MagicMock(), + user_api_key_dict=MagicMock(router_settings=None), + version=None, + proxy_config=MagicMock(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model=primary_model, + route_type="acompletion", + llm_router=mock_router, + ) + + assert processor.data["proxy_server_request"]["body"] == { + "model": primary_model, + "[redacted]": "blocked by guardrail", + }, "fallback restoration put the unredacted blocked prompt back in the spend-log snapshot" + @pytest.mark.asyncio async def test_local_rate_limit_fallback_skips_an_aliased_embedding_only_model(self): from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 97c21fcc777..f8c0b8ef11b 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -1392,6 +1392,40 @@ def test_refresh_proxy_server_request_body_snapshot_picks_up_guardrail_masking() assert "123-45-6789" not in json.dumps(refreshed_body) +def test_redact_blocked_request_body_snapshot_replaces_body_with_placeholder(): + from litellm.proxy.litellm_pre_call_utils import redact_blocked_request_body_snapshot + + data = { + "model": "small", + "messages": [{"role": "user", "content": "my token is glpat-FAKE-NOT-A-REAL-TOKEN-1111"}], + "proxy_server_request": { + "url": "http://localhost/v1/chat/completions", + "method": "POST", + "body": { + "model": "small", + "messages": [{"role": "user", "content": "my token is glpat-FAKE-NOT-A-REAL-TOKEN-1111"}], + }, + }, + } + + redact_blocked_request_body_snapshot(data) + + assert data["proxy_server_request"]["body"] == { + "model": "small", + "[redacted]": "blocked by guardrail", + }, "blocked request body still holds the raw prompt in the spend-log snapshot" + + +def test_redact_blocked_request_body_snapshot_is_noop_without_snapshot(): + from litellm.proxy.litellm_pre_call_utils import redact_blocked_request_body_snapshot + + data = {"model": "small", "messages": [{"role": "user", "content": "hello"}]} + + redact_blocked_request_body_snapshot(data) + + assert data == {"model": "small", "messages": [{"role": "user", "content": "hello"}]} + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_strips_string_encoded_admin_injection(): """Regression: metadata arriving as a JSON string (multipart/form-data or