From e5e45520d1d57e549290c87738759d340fe3c29e Mon Sep 17 00:00:00 2001 From: Mrunmayee Daware <165297479+Mrun25@users.noreply.github.com> Date: Wed, 14 Oct 2026 22:09:57 +0530 Subject: [PATCH] Log not_run when a Bedrock guardrail scan never ran apply_guardrail can return without calling Bedrock: an empty text set, a role-scoped skip (no user message, or a latest user message with no text), or an incremental scan whose session has nothing new. The log_guardrail_information decorator still wrote a success entry for those calls, so monitoring could not tell a passed check apart from a check that never ran (#46455). Record a not_run entry with the skip reason on those paths, matching the translation handler's existing not_run convention. Self-recording suppresses the decorator's automatic success entry, so exactly one entry is logged. The empty-texts path keeps the tail's behavior of normalizing inputs["texts"] to the empty list. Fixes #46455 --- .../guardrail_hooks/bedrock_guardrails.py | 43 ++++++++- .../test_bedrock_guardrails.py | 96 +++++++++++++++++++ 2 files changed, 135 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 86ce8cb5c09..209feb6366f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -200,6 +200,8 @@ class ApplyGuardrailMessageSelection(NamedTuple): scanned_role_subset: bool # True when there is nothing to scan (e.g. no user-role message). skip_scan: bool = False + # Why the scan was skipped; recorded as the `not_run` reason (#46455). + skip_reason: str | None = None def _redact_pii_matches(response_json: dict) -> dict: @@ -704,14 +706,18 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): "user message, so scan_only_tool_results leaves nothing to scan for this request" ) verbose_proxy_logger.debug("Bedrock Guardrail: no user-role message in request, skipping INPUT scan") - return ApplyGuardrailMessageSelection(None, None, True, skip_scan=True) + return ApplyGuardrailMessageSelection( + None, None, True, skip_scan=True, skip_reason="no user-role message in request" + ) selected_message: Final = structured_messages[latest_user_index] if self._count_message_texts(selected_message) == 0: verbose_proxy_logger.debug( "Bedrock Guardrail: latest user message has no text content, skipping INPUT scan" ) - return ApplyGuardrailMessageSelection(None, None, True, skip_scan=True) + return ApplyGuardrailMessageSelection( + None, None, True, skip_scan=True, skip_reason="latest user message has no text content" + ) return ApplyGuardrailMessageSelection( filtered_messages=[selected_message], @@ -3290,6 +3296,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if not new_texts: verbose_proxy_logger.debug("Bedrock Guardrail: no new messages to scan for this session, skipping API call") + self._record_guardrail_not_run( + request_data=request_data, # pyright: ignore[reportUnknownArgumentType] # enclosing method's request_data param is a bare dict + reason="no new content to scan for this session", + ) return inputs bedrock_response: Final = await self.make_bedrock_api_request( @@ -3314,6 +3324,23 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) return inputs + def _record_guardrail_not_run( + self, + request_data: dict[str, object], # mutable-ok: request metadata bucket is appended by the logging helper + reason: str, + ) -> None: + """Record a ``not_run`` entry for a call that returned without calling Bedrock. + + Monitoring cannot tell "checked and clean" apart from "not checked" when a + skipped scan is logged as ``success`` (#46455). Self-recording also suppresses + the ``log_guardrail_information`` decorator's automatic ``success`` entry. + """ + self.add_standard_logging_guardrail_information_to_request_data( # pyright: ignore[reportUnknownMemberType] # logging helper's own request_data param predates strict typing + guardrail_json_response=reason, + request_data=request_data, + guardrail_status="not_run", + ) + async def apply_guardrail( self, inputs: "GenericGuardrailAPIInputs", @@ -3362,9 +3389,17 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): request_data=request_data, input_type=input_type, ) - if selection.skip_scan: - return inputs filtered_messages: Final = selection.filtered_messages + if selection.skip_scan or not filtered_messages: + # These paths return without calling Bedrock (it rejects an empty + # content list). Record `not_run` so the entry does not read as a + # passed check (#46455). + self._record_guardrail_not_run( + request_data=request_data, # pyright: ignore[reportUnknownArgumentType] # enclosing method's request_data param is a bare dict + reason=selection.skip_reason or "no scannable content", + ) + inputs["texts"] = texts # rebind-ok: mirrors the scanned path's normalized texts write-back + return inputs scanned_slice: Final = selection.scanned_slice scanned_role_subset: Final = selection.scanned_role_subset diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 3b33ba75eb3..da067acb38a 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -8063,3 +8063,99 @@ async def test_during_call_masking_rewrites_only_pii_block_in_content_list( ] assert messages[1]["content"] == "Hello, how can I help you today?" assert messages[2]["content"] == "who is the president of the united states?" + + +@pytest.mark.asyncio +async def test_apply_guardrail_records_not_run_when_nothing_scanned(): + """A call that returns without calling Bedrock must log `not_run`, not `success` (#46455). + + Monitoring cannot tell "checked and clean" apart from "not checked" when a + skipped scan is logged as `success`. Exactly one entry must be recorded: the + self-recorded `not_run` suppresses the decorator's automatic `success` entry. + """ + guardrail = BedrockGuardrail(guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT") + + with patch.object(guardrail, "make_bedrock_api_request", new_callable=AsyncMock) as mock_api_request: + inputs = { + "texts": [], + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"location":"Paris"}'}, + } + ], + } + request_data: dict = {} + + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="response", + logging_obj=None, + ) + + mock_api_request.assert_not_called() + assert result is inputs + entries = request_data["metadata"]["standard_logging_guardrail_information"] + assert len(entries) == 1 + assert entries[0]["guardrail_status"] == "not_run" + assert entries[0]["guardrail_response"] == "no scannable content" + + +@pytest.mark.asyncio +async def test_apply_guardrail_records_not_run_reason_for_role_scoped_skip(): + """A role-scoped skip carries its reason into the `not_run` entry (#46455).""" + guardrail = BedrockGuardrail( + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + experimental_use_latest_role_message_only=True, + ) + + with patch.object(guardrail, "make_bedrock_api_request", new_callable=AsyncMock) as mock_api_request: + inputs = { + "texts": ["Sure, checking that for you."], + "structured_messages": [{"role": "assistant", "content": "Sure, checking that for you."}], + } + request_data: dict = {} + + await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + logging_obj=None, + ) + + mock_api_request.assert_not_called() + entries = request_data["metadata"]["standard_logging_guardrail_information"] + assert len(entries) == 1 + assert entries[0]["guardrail_status"] == "not_run" + assert entries[0]["guardrail_response"] == "no user-role message in request" + + +@pytest.mark.asyncio +async def test_apply_guardrail_records_not_run_for_incremental_skip(): + """A session with nothing new to scan logs `not_run` with its reason (#46455).""" + guardrail = BedrockGuardrail(guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT") + + with ( + patch.object(guardrail, "_incremental_scan_cache", return_value=object()), + patch.object(guardrail, "filter_new_texts_for_session", new_callable=AsyncMock, return_value=[]), + patch.object(guardrail, "make_bedrock_api_request", new_callable=AsyncMock) as mock_api_request, + ): + inputs = {"texts": ["hello"]} + request_data: dict = {} + + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + logging_obj=None, + ) + + mock_api_request.assert_not_called() + assert result is inputs + entries = request_data["metadata"]["standard_logging_guardrail_information"] + assert len(entries) == 1 + assert entries[0]["guardrail_status"] == "not_run" + assert entries[0]["guardrail_response"] == "no new content to scan for this session"