From fddc14ce147d409ca29c3ed5a0d7ae2df7f83e9d Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 26 Sep 2026 08:52:33 +0000 Subject: [PATCH] fix(guardrails): keep the base hook call shape for non-attachment guardrails The extraction hooks only receive scan_attachments and files_to_check when the applied guardrail sets scans_attachments, so subclasses written against the pre-attachment signatures keep working for every existing guardrail. Adds compatibility tests for base- and head-signature subclasses, a parametrized Responses during_call regression guard showing text-only function_call_output strings scan as text, and the unit and integration coverage for the skip_tool_message and sync-signature fixes. Co-authored-by: feng.tsai Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../chat/guardrail_translation/handler.py | 16 +- .../chat/guardrail_translation/handler.py | 10 +- .../test_bedrock_guardrails.py | 4 +- .../observability/test_guardrail_effects.py | 47 +++ .../test_openai_chat_guardrail_handler.py | 125 ++++++ .../test_bedrock_guardrails.py | 357 ++++++++++++++++-- .../guardrails/test_guardrail_endpoints.py | 4 +- .../proxy/guardrails/test_init_guardrails.py | 5 +- .../test_anthropic_guardrail_handler.py | 101 ++++- 9 files changed, 616 insertions(+), 53 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 297349e224c..7e8b486c828 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -972,7 +972,7 @@ class AnthropicMessagesHandler(BaseTranslation): content_idx=content_idx, skip_tool_message=skip_tool_message, scan_only_tool_results=scan_only_tool_results, - scan_attachments=scan_attachments, + **({"scan_attachments": True} if scan_attachments else {}), ) for content_idx, content_item in enumerate(content) if isinstance(content_item, dict) @@ -1016,7 +1016,7 @@ class AnthropicMessagesHandler(BaseTranslation): content_item=content_item, msg_idx=msg_idx, content_idx=content_idx, - scan_attachments=scan_attachments, + **({"scan_attachments": True} if scan_attachments else {}), ) if scan_only_tool_results: @@ -1027,7 +1027,11 @@ class AnthropicMessagesHandler(BaseTranslation): scanned=( () if text_str is None else (ScannedText(text_str, ContentBlockTextTarget(msg_idx, content_idx)),) ), - images=(cls._image_sources(content_item, scan_attachments) if content_item.get("type") == "image" else ()), + images=( + cls._image_sources(content_item, **({"scan_attachments": True} if scan_attachments else {})) + if content_item.get("type") == "image" + else () + ), files=( cls._document_sources(content_item) if scan_attachments and content_item.get("type") == "document" @@ -1064,7 +1068,9 @@ class AnthropicMessagesHandler(BaseTranslation): ), images=tuple( chain.from_iterable( - cls._image_sources(block, scan_attachments) for _, block in blocks if block.get("type") == "image" + cls._image_sources(block, **({"scan_attachments": True} if scan_attachments else {})) + for _, block in blocks + if block.get("type") == "image" ) ), files=tuple( @@ -1077,7 +1083,7 @@ class AnthropicMessagesHandler(BaseTranslation): ) @staticmethod - def _image_sources(block: Mapping[str, object], scan_attachments: bool) -> tuple[str, ...]: + def _image_sources(block: Mapping[str, object], scan_attachments: bool = False) -> tuple[str, ...]: """Normalize an Anthropic image block into strings a guardrail can read. base64 becomes a data URI so the format travels with the payload, which is what diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 1ffd5426307..c77fe9e22e1 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -184,14 +184,13 @@ class OpenAIChatCompletionsHandler(BaseTranslation): msg_idx=msg_idx, texts_to_check=texts_to_check, images_to_check=images_to_check, - files_to_check=files_to_check, tool_calls_to_check=tool_calls_to_check, text_task_mappings=text_task_mappings, tool_call_task_mappings=tool_call_task_mappings, skip_system_message=skip_system, skip_tool_message=skip_tool, scan_only_tool_results=scan_only_tool_results, - scan_attachments=scan_attachments, + **({"scan_attachments": True, "files_to_check": files_to_check} if scan_attachments else {}), ) # Step 2: Apply guardrail to all texts and tool calls in batch @@ -335,11 +334,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): msg_idx=msg_idx, texts_to_check=texts, images_to_check=images, - files_to_check=files, tool_calls_to_check=tool_calls, text_task_mappings=[], # mutable-ok: required by _extract_inputs, unused here tool_call_task_mappings=[], # mutable-ok: required by _extract_inputs, unused here - scan_attachments=scan_attachments, + **({"scan_attachments": True, "files_to_check": files} if scan_attachments else {}), ) if texts or tool_calls: return "no scannable content after message scoping" @@ -364,7 +362,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation): msg_idx: int, texts_to_check: list[str], images_to_check: list[str], - files_to_check: list[str], tool_calls_to_check: list[ChatCompletionToolParam], text_task_mappings: list[tuple[int, int | None]], tool_call_task_mappings: list[tuple[int, int]], @@ -372,6 +369,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): skip_tool_message: bool = False, scan_only_tool_results: bool = False, scan_attachments: bool = False, + files_to_check: list[str] | None = None, ) -> None: """ Extract text content, images, and tool calls from a message. @@ -406,7 +404,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): image_ref, file_ref = _attachment_part_refs(content_item, scan_attachments) if image_ref is not None: images_to_check.append(image_ref) - if file_ref is not None: + if file_ref is not None and files_to_check is not None: files_to_check.append(file_ref) # Extract tool calls (typically in assistant messages) diff --git a/tests/guardrails_tests/test_bedrock_guardrails.py b/tests/guardrails_tests/test_bedrock_guardrails.py index bb958bcc304..43d088268eb 100644 --- a/tests/guardrails_tests/test_bedrock_guardrails.py +++ b/tests/guardrails_tests/test_bedrock_guardrails.py @@ -945,7 +945,7 @@ async def test_convert_to_bedrock_format_input_source(): ] # Call the method - result = await guardrail.convert_to_bedrock_format(source="INPUT", messages=mock_messages) + result = guardrail.convert_to_bedrock_format(source="INPUT", messages=mock_messages) # Verify the result structure assert isinstance(result, dict) @@ -1007,7 +1007,7 @@ async def test_convert_to_bedrock_format_output_source(): ) # Call the method - result = await guardrail.convert_to_bedrock_format( + result = guardrail.convert_to_bedrock_format( source="OUTPUT", response=mock_response ) diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index b4452cd0fef..6c62a6f26e8 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -1436,3 +1436,50 @@ def test_bedrock_during_call_responses_function_call_output_string_scans_as_text assert response.status_code == 200, response.text eventually(lambda: policy.drain(), lambda values: len(values) == 1, seconds=10) assert len(upstream.drain()) == 1 + + +def test_bedrock_during_call_skip_tool_message_flag_drops_tool_result_text(gateway: Gateway, tmp_path: Path) -> None: + tool_text: Final = "BLOCKME from tool" + + def guardrail(request: Request) -> Reply: + body: Final = json.loads(request.body) + texts: Final = [item["text"]["text"] for item in body["content"] if "text" in item] + assert texts == ["hello"], body + return Reply(body=b'{"action":"NONE","outputs":[],"assessments":[]}') + + with wire_server(guardrail) as policy, wire_server(_anthropic_reply) as upstream: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + bedrock_params: Final = _bedrock_policy(policy.url) + bedrock_params["mode"] = "during_call" + bedrock_params["skip_tool_message_in_guardrail"] = True + config["guardrails"] = [ + {"guardrail_name": "bedrock-skip-tool-" + uuid.uuid4().hex, "litellm_params": bedrock_params} + ] + path: Final = tmp_path / "bedrock-skip-tool.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(gateway, tmp_path, {}, config=path) as candidate, candidate.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-4-5-20250929", api_base=upstream.url, api_key="synthetic-anthropic-key" + ) + response: Final = candidate.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 16, + "messages": [ + {"role": "user", "content": "hello"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01A", "name": "lookup", "input": {}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_01A", "content": tool_text}], + }, + ], + }, + ) + assert response.status_code == 200, response.text + eventually(lambda: policy.drain(), lambda values: len(values) == 1, seconds=10) + assert len(upstream.drain()) == 1 diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_chat_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_chat_guardrail_handler.py index 16781c11297..45d15aa6dc2 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_chat_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_chat_guardrail_handler.py @@ -312,3 +312,128 @@ class TestOpenAIChatHandlerToolsAndModelForwarding: assert guardrail.calls == 1 assert guardrail.inputs is not None assert guardrail.inputs["model"] == ["gpt-4o"] + + +class _BaseSignatureExtractInputsHandler(OpenAIChatCompletionsHandler): + """A subclass written against the pre-attachment-scanning _extract_inputs signature.""" + + def _extract_inputs( + self, + message, + msg_idx, + texts_to_check, + images_to_check, + tool_calls_to_check, + text_task_mappings, + tool_call_task_mappings, + skip_system_message=False, + skip_tool_message=False, + scan_only_tool_results=False, + ) -> None: + return super()._extract_inputs( + message=message, + msg_idx=msg_idx, + texts_to_check=texts_to_check, + images_to_check=images_to_check, + tool_calls_to_check=tool_calls_to_check, + text_task_mappings=text_task_mappings, + tool_call_task_mappings=tool_call_task_mappings, + skip_system_message=skip_system_message, + skip_tool_message=skip_tool_message, + scan_only_tool_results=scan_only_tool_results, + ) + + +class _HeadSignatureRecordingExtractInputsHandler(OpenAIChatCompletionsHandler): + seen_scan_attachments: bool | None = None + seen_files_to_check_is_not_none: bool | None = None + + def _extract_inputs( + self, + message, + msg_idx, + texts_to_check, + images_to_check, + tool_calls_to_check, + text_task_mappings, + tool_call_task_mappings, + skip_system_message=False, + skip_tool_message=False, + scan_only_tool_results=False, + scan_attachments=False, + files_to_check=None, + ) -> None: + _HeadSignatureRecordingExtractInputsHandler.seen_scan_attachments = scan_attachments + _HeadSignatureRecordingExtractInputsHandler.seen_files_to_check_is_not_none = files_to_check is not None + return super()._extract_inputs( + message=message, + msg_idx=msg_idx, + texts_to_check=texts_to_check, + images_to_check=images_to_check, + tool_calls_to_check=tool_calls_to_check, + text_task_mappings=text_task_mappings, + tool_call_task_mappings=tool_call_task_mappings, + skip_system_message=skip_system_message, + skip_tool_message=skip_tool_message, + scan_only_tool_results=scan_only_tool_results, + scan_attachments=scan_attachments, + files_to_check=files_to_check, + ) + + +class TestExtractInputsBaseSignatureCompatibility: + @pytest.mark.asyncio + async def test_legacy_signature_subclass_runs_for_plain_guardrail(self): + """A non-attachment guardrail must not force files_to_check onto old subclasses.""" + guardrail = RecordingGuardrail() + data = { + "model": "gpt-4o", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarize this"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg=="}, + }, + ], + } + ], + } + + await _BaseSignatureExtractInputsHandler().process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.inputs is not None + assert guardrail.inputs["texts"] == ["summarize this"] + + @pytest.mark.asyncio + async def test_attachment_guardrail_still_passes_the_new_kwargs(self): + """With scans_attachments on, the head signature receives scan_attachments and files.""" + _HeadSignatureRecordingExtractInputsHandler.seen_scan_attachments = None + _HeadSignatureRecordingExtractInputsHandler.seen_files_to_check_is_not_none = None + guardrail = ScanningGuardrail() + data = { + "model": "gpt-4o", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarize this"}, + { + "type": "file", + "file": {"filename": "a.pdf", "file_data": "data:application/pdf;base64,AAAA"}, + }, + ], + } + ], + } + + await _HeadSignatureRecordingExtractInputsHandler().process_input_messages( + data=data, guardrail_to_apply=guardrail + ) + + assert _HeadSignatureRecordingExtractInputsHandler.seen_scan_attachments is True + assert _HeadSignatureRecordingExtractInputsHandler.seen_files_to_check_is_not_none is True + assert guardrail.inputs is not None + assert guardrail.inputs["files"] diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index b9447d1c0ae..f89363f1cb5 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -2562,15 +2562,13 @@ _GUARD_BLOCK = {"text": {"text": _GROUNDING_RESPONSE_TEXT, "qualifiers": ["guard def _input_request(messages: list) -> dict: """Arrange a guardrail and act: build the Bedrock INPUT payload.""" - return asyncio.run(_grounding_guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)) + return _grounding_guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) def _output_request(messages: list, response=None, from_messages: bool = False) -> dict: """Arrange a guardrail and act: build the Bedrock OUTPUT payload.""" - return asyncio.run( - _grounding_guardrail(from_messages).convert_to_bedrock_format( - source="OUTPUT", response=response, messages=messages - ) + return _grounding_guardrail(from_messages).convert_to_bedrock_format( + source="OUTPUT", response=response, messages=messages ) @@ -2743,8 +2741,8 @@ def test_grounding_input_ignores_plain_message_derivation(): "content": [{"text": {"text": _GROUNDING_SOURCE_TEXT}}, {"text": {"text": _GROUNDING_QUERY_TEXT}}], } - actual_request = asyncio.run( - _grounding_guardrail(from_messages=True).convert_to_bedrock_format(source="INPUT", messages=messages) + actual_request = _grounding_guardrail(from_messages=True).convert_to_bedrock_format( + source="INPUT", messages=messages ) assert actual_request == expected_request @@ -5667,7 +5665,7 @@ class TestBedrockGuardrailImageInput: } ] - request = await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) + request = self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) assert request["content"] == [ {"text": {"text": "what does this say?"}}, @@ -5694,7 +5692,7 @@ class TestBedrockGuardrailImageInput: } ] - request = await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) + request = self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) assert request["content"] == [ {"image": {"format": "png", "source": {"bytes": self._PNG_DATA_URI.split(",")[1]}}} @@ -5711,7 +5709,7 @@ class TestBedrockGuardrailImageInput: ] with pytest.raises(HTTPException) as exc_info: - await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) + self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) assert exc_info.value.status_code == 400 @@ -5733,7 +5731,7 @@ class TestBedrockGuardrailImageInput: ] with pytest.raises(HTTPException) as exc_info: - await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) + self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) assert exc_info.value.status_code == 400 assert exc_info.value.detail["error"] == "Violated guardrail policy" @@ -5748,7 +5746,7 @@ class TestBedrockGuardrailImageInput: ] with pytest.raises(HTTPException) as exc_info: - await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) + self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) assert exc_info.value.status_code == 400 @@ -5765,7 +5763,7 @@ class TestBedrockGuardrailImageInput: sent: list = [] async def spy(**kwargs): - sent.append(await g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"])) + sent.append(g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"])) return {"action": "NONE", "outputs": []} # Every text is already in the session cache: the exact state that made the @@ -5795,7 +5793,7 @@ class TestBedrockGuardrailImageInput: sent: list = [] async def spy(**kwargs): - sent.append(await g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"])) + sent.append(g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"])) return {"action": "NONE", "outputs": []} with patch.object(g, "make_bedrock_api_request", new=spy): @@ -5824,7 +5822,7 @@ class TestBedrockGuardrailImageInput: } ] - request = await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) + request = self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) assert len(request["content"]) == 20 @@ -5845,7 +5843,7 @@ class TestBedrockGuardrailImageInput: ) as decode ): with pytest.raises(HTTPException) as exc_info: - await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) + self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) decode.assert_not_awaited() assert "at most 20 images" in str(exc_info.value.detail) @@ -5867,7 +5865,7 @@ class TestBedrockGuardrailImageInput: into a content item: an unrecognised part that fell through to the text branch would be reported to the operator as scanned when it was not. """ - request = await self._guardrail().convert_to_bedrock_format( + request = self._guardrail().convert_to_bedrock_format( source="INPUT", messages=[{"role": "user", "content": content}] ) @@ -5876,7 +5874,7 @@ class TestBedrockGuardrailImageInput: @pytest.mark.asyncio async def test_a_bare_string_part_is_scanned_as_text(self): """A content list may hold plain strings, not only typed parts.""" - request = await self._guardrail().convert_to_bedrock_format( + request = self._guardrail().convert_to_bedrock_format( source="INPUT", messages=[{"role": "user", "content": ["just text"]}] ) @@ -5888,7 +5886,7 @@ class TestBedrockGuardrailImageInput: Both reach the model as an image, so both have to reach the scan. """ - request = await self._guardrail().convert_to_bedrock_format( + request = self._guardrail().convert_to_bedrock_format( source="INPUT", messages=[{"role": "user", "content": [{"type": "image_url", "image_url": self._PNG_DATA_URI}]}], ) @@ -5956,7 +5954,7 @@ class TestBedrockGuardrailImageInput: new_callable=AsyncMock, ) as decode: with pytest.raises(HTTPException) as exc_info: - await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) + self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) decode.assert_not_awaited() assert "remote image URLs are not supported" in str(exc_info.value.detail) @@ -6074,7 +6072,7 @@ class TestBedrockGuardrailImageInput: ] with pytest.raises(HTTPException) as exc_info: - await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) + self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) assert "Violated guardrail policy" in str(exc_info.value.detail) @@ -6095,7 +6093,7 @@ class TestBedrockGuardrailImageInput: ] with pytest.raises(HTTPException) as exc_info: - await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) + self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) assert "Violated guardrail policy" in str(exc_info.value.detail) @@ -6119,7 +6117,7 @@ class TestBedrockGuardrailImageInput: } ] - request = await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) + request = self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) assert request["content"] == [ {"text": {"text": "what does this say?"}}, @@ -6140,7 +6138,7 @@ class TestBedrockGuardrailImageInput: ] with pytest.raises(HTTPException) as exc_info: - await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) + self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) assert "Violated guardrail policy" in str(exc_info.value.detail) @@ -6189,7 +6187,7 @@ class TestBedrockGuardrailImageInput: sent: list = [] async def spy(**kwargs): - sent.append(await g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"])) + sent.append(g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"])) return {"action": "NONE", "outputs": []} with patch.object(g, "make_bedrock_api_request", new=spy): @@ -6226,7 +6224,7 @@ class TestBedrockGuardrailImageInput: sent: list = [] async def spy(**kwargs): - sent.append(await g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"])) + sent.append(g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"])) return {"action": "NONE", "outputs": []} with patch.object(g, "make_bedrock_api_request", new=spy): @@ -6304,7 +6302,7 @@ class TestBedrockGuardrailImageInput: ] with pytest.raises(HTTPException) as exc_info: - await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) + self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) assert "Violated guardrail policy" in str(exc_info.value.detail) @@ -6334,7 +6332,7 @@ class TestBedrockGuardrailImageInput: } ] - request = await g.convert_to_bedrock_format(source="INPUT", messages=messages) + request = g.convert_to_bedrock_format(source="INPUT", messages=messages) kinds = [key for item in request["content"] for key in item] assert "image" in kinds @@ -6347,7 +6345,7 @@ class TestBedrockGuardrailImageInput: messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"file_id": "file_abc"}}]}] with pytest.raises(HTTPException) as exc_info: - await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) + self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) assert "Violated guardrail policy" in str(exc_info.value.detail) @@ -6367,7 +6365,7 @@ class TestBedrockGuardrailImageInput: messages = [{"role": "user", "content": [part]}] with pytest.raises(HTTPException) as exc_info: - await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) + self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages) assert "image part carries no inline url" in str(exc_info.value.detail) @@ -6394,7 +6392,7 @@ class TestBedrockGuardrailImageInput: sent: list = [] async def spy(**kwargs): - sent.append(await g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"])) + sent.append(g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"])) return {"action": "NONE", "outputs": []} with patch.object(g, "make_bedrock_api_request", new=spy): @@ -6419,7 +6417,7 @@ class TestBedrockGuardrailImageInput: sent: list = [] async def spy(**kwargs): - sent.append(await g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"])) + sent.append(g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"])) return {"action": "NONE", "outputs": []} bare_base64 = self._PNG_DATA_URI.split(",")[1] @@ -6463,7 +6461,7 @@ class TestBedrockGuardrailImageInput: # cast-ok: pins the runtime fallback the Literal forbids at type-check time source = cast(Literal["INPUT", "OUTPUT"], "SOMETHING_ELSE") - request = await self._guardrail().convert_to_bedrock_format( + request = self._guardrail().convert_to_bedrock_format( source=source, messages=[{"role": "user", "content": "hello"}], ) @@ -6480,7 +6478,7 @@ class TestBedrockGuardrailImageInput: oversized_png = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"\x00" * (5 * 1024 * 1024)).decode() with pytest.raises(HTTPException) as exc_info: - await self._guardrail().convert_to_bedrock_format( + self._guardrail().convert_to_bedrock_format( source="INPUT", messages=[ { @@ -7768,3 +7766,294 @@ async def test_during_call_hook_responses_tool_output_input_image_member_refused assert exc_info.value.status_code == 400 mock_post.assert_not_called() + + +def _skip_flag_tool_result_messages() -> list[dict[str, object]]: + return [ + {"role": "user", "content": "hello"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01A", "name": "get_weather", "input": {}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_01A", "content": "BLOCKME from tool"}], + }, + ] + + +def _sent_apply_guardrail_texts(mock_post) -> list[str]: + sent_body = json.loads(mock_post.call_args.kwargs["data"]) + return [item["text"]["text"] for item in sent_body["content"] if "text" in item] + + +async def _run_anthropic_during_call(guardrail, messages, response_json=None): + """Drive async_moderation_hook on an anthropic_messages call; return (data, post mock).""" + mock_credentials = MagicMock() + mock_credentials.access_key = "test-access-key" + mock_credentials.secret_key = "test-secret-key" + mock_credentials.token = None + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = response_json or {"action": "NONE", "assessments": []} + data = {"model": "claude-sonnet-4-5", "messages": messages} + with ( + patch.object(guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1")), + patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post, + ): + mock_post.return_value = mock_response + await guardrail.async_moderation_hook( + data=data, + user_api_key_dict=UserAPIKeyAuth(), + call_type=CallTypes.anthropic_messages.value, + ) + return data, mock_post + + +@pytest.mark.asyncio +async def test_during_call_hook_scans_tool_result_text_when_skip_flag_off(): + """Without skip_tool_message_in_guardrail a tool_result string is scanned like base.""" + guardrail = BedrockGuardrail( + guardrail_name="bedrock-tool-result-default", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + event_hook=GuardrailEventHooks.during_call, + default_on=True, + ) + + _, mock_post = await _run_anthropic_during_call(guardrail, _skip_flag_tool_result_messages()) + + mock_post.assert_called_once() + assert _sent_apply_guardrail_texts(mock_post) == ["hello", "BLOCKME from tool"] + + +@pytest.mark.asyncio +async def test_during_call_hook_skip_tool_message_flag_drops_tool_result_text(): + """skip_tool_message_in_guardrail removes the tool_result string from the INPUT scan.""" + guardrail = BedrockGuardrail( + guardrail_name="bedrock-tool-result-skip", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + event_hook=GuardrailEventHooks.during_call, + default_on=True, + skip_tool_message_in_guardrail=True, + ) + + _, mock_post = await _run_anthropic_during_call(guardrail, _skip_flag_tool_result_messages()) + + mock_post.assert_called_once() + assert _sent_apply_guardrail_texts(mock_post) == ["hello"] + + +@pytest.mark.asyncio +async def test_during_call_hook_global_skip_tool_message_flag_drops_tool_result_text(monkeypatch): + """The litellm-level flag applies when the per-guardrail attr is unset.""" + monkeypatch.setattr(litellm, "skip_tool_message_in_guardrail", True) + guardrail = BedrockGuardrail( + guardrail_name="bedrock-tool-result-skip-global", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + event_hook=GuardrailEventHooks.during_call, + default_on=True, + ) + + _, mock_post = await _run_anthropic_during_call(guardrail, _skip_flag_tool_result_messages()) + + mock_post.assert_called_once() + assert _sent_apply_guardrail_texts(mock_post) == ["hello"] + + +@pytest.mark.asyncio +async def test_during_call_hook_skip_tool_message_flag_keeps_tool_result_out_of_masked_writeback(): + """Masked outputs align to scanned texts; a skipped tool_result consumes no slot.""" + guardrail = BedrockGuardrail( + guardrail_name="bedrock-tool-result-skip-mask", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + event_hook=GuardrailEventHooks.during_call, + default_on=True, + skip_tool_message_in_guardrail=True, + ) + messages = [ + {"role": "user", "content": "hello"}, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_01A", "content": "BLOCKME from tool"}], + }, + {"role": "user", "content": "and bye"}, + ] + + data, mock_post = await _run_anthropic_during_call( + guardrail, + messages, + response_json={"action": "GUARDRAIL_INTERVENED", "outputs": [{"text": "M1"}, {"text": "M2"}]}, + ) + + assert _sent_apply_guardrail_texts(mock_post) == ["hello", "and bye"] + assert data["messages"][0]["content"] == "M1" + assert data["messages"][1]["content"] == [ + {"type": "tool_result", "tool_use_id": "toolu_01A", "content": "BLOCKME from tool"} + ] + assert data["messages"][2]["content"] == "M2" + + +@pytest.mark.asyncio +async def test_during_call_hook_skip_tool_message_flag_tool_result_only_turn_posts_nothing(): + """Flag on + latest-role scoping over a tool_result-only turn scans nothing.""" + guardrail = BedrockGuardrail( + guardrail_name="bedrock-tool-result-skip-empty", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + event_hook=GuardrailEventHooks.during_call, + default_on=True, + experimental_use_latest_role_message_only=True, + skip_tool_message_in_guardrail=True, + ) + + _, mock_post = await _run_anthropic_during_call(guardrail, _anthropic_tool_result_conversation()) + + mock_post.assert_not_called() + + +@pytest.mark.asyncio +async def test_during_call_hook_skip_tool_message_flag_still_refuses_tool_result_document(): + """The skip flag narrows text scanning only; attachments inside it stay refused.""" + guardrail = BedrockGuardrail( + guardrail_name="bedrock-tool-result-skip-doc", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + event_hook=GuardrailEventHooks.during_call, + default_on=True, + skip_tool_message_in_guardrail=True, + ) + messages = [ + {"role": "user", "content": "hello"}, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01A", + "content": [ + { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": "AAAA"}, + } + ], + } + ], + }, + ] + + with ( + patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post, + pytest.raises(HTTPException) as exc_info, + ): + await guardrail.async_moderation_hook( + data={"model": "claude-sonnet-4-5", "messages": messages}, + user_api_key_dict=UserAPIKeyAuth(), + call_type=CallTypes.anthropic_messages.value, + ) + + assert exc_info.value.status_code == 400 + mock_post.assert_not_called() + + +def test_convert_to_bedrock_format_returns_bedrock_request_synchronously(): + """The public INPUT converter is sync again; callers never await it.""" + guardrail = BedrockGuardrail(guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT") + + request = guardrail.convert_to_bedrock_format(source="INPUT", messages=[{"role": "user", "content": "hi"}]) + + import inspect + + assert inspect.isawaitable(request) is False + assert request["content"][0]["text"]["text"] == "hi" + + +def test_convert_to_bedrock_format_inline_png_builds_an_image_item(): + """An inline png data URI decodes into an ApplyGuardrail image item without IO.""" + guardrail = BedrockGuardrail(guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT") + messages = [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg=="}} + ], + } + ] + + request = guardrail.convert_to_bedrock_format(source="INPUT", messages=messages) + + assert request["content"][0]["image"]["format"] == "png" + + +@pytest.mark.asyncio +async def test_during_call_hook_refuses_remote_image_without_calling_a_fetcher(): + """A remote image is refused by the substring test before any sync/async fetch.""" + guardrail = BedrockGuardrail( + guardrail_name="bedrock-remote-image", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + event_hook=GuardrailEventHooks.during_call, + default_on=True, + ) + data = { + "model": "gpt-4o-mini", + "messages": [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}], + } + ], + } + + with ( + patch( + "litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails.BedrockImageProcessor.get_image_details", + side_effect=AssertionError("sync fetch reached"), + ) as sync_fetch, + patch( + "litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails.BedrockImageProcessor.get_image_details_async", + new_callable=AsyncMock, + side_effect=AssertionError("async fetch reached"), + ) as async_fetch, + patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post, + pytest.raises(HTTPException) as exc_info, + ): + await guardrail.async_moderation_hook( + data=data, + user_api_key_dict=UserAPIKeyAuth(), + call_type=CallTypes.acompletion.value, + ) + + assert exc_info.value.status_code == 400 + sync_fetch.assert_not_called() + async_fetch.assert_not_called() + mock_post.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "output", + [ + json.dumps([{"type": "file", "name": "README.md", "url": "https://api.github.com/x"}]), + json.dumps([{"type": "file", "name": "q3.pdf", "file_id": "file-abc"}]), + json.dumps([{"type": "image_url", "image_url": "https://example.com/cat.jpg"}]), + json.dumps([{"type": "document", "source": {"title": "wiki", "page": 3}, "text": "hello"}]), + [{"type": "input_text", "text": json.dumps([{"type": "file", "file_id": "file-abc"}])}], + ], +) +async def test_during_call_hook_responses_function_call_output_json_string_scans_as_text(output): + """Client string outputs are never re-parsed; JSON-shaped tool output scans as text.""" + guardrail = BedrockGuardrail( + guardrail_name="bedrock-resp-tool-json-text", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + event_hook=GuardrailEventHooks.during_call, + default_on=True, + ) + + mock_post = await _run_responses_during_call(guardrail, output) + + mock_post.assert_called_once() diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py index 6dfe9e30d22..d962e920a71 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py @@ -1,7 +1,7 @@ import json from datetime import datetime from typing import Dict, List, Optional -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock import pytest @@ -945,7 +945,7 @@ async def test_bedrock_guardrail_make_api_request_passes_api_key(): ), patch.object(guardrail_hook, "_load_credentials") as mock_load_creds, patch.object( - guardrail_hook, "convert_to_bedrock_format", new_callable=AsyncMock + guardrail_hook, "convert_to_bedrock_format", new_callable=MagicMock ) as mock_convert, patch.object( guardrail_hook, "get_guardrail_dynamic_request_body_params" diff --git a/tests/test_litellm/proxy/guardrails/test_init_guardrails.py b/tests/test_litellm/proxy/guardrails/test_init_guardrails.py index 37a75c77dae..31494e6f3ed 100644 --- a/tests/test_litellm/proxy/guardrails/test_init_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/test_init_guardrails.py @@ -1,4 +1,3 @@ -import asyncio import json from unittest.mock import MagicMock, patch @@ -114,8 +113,8 @@ def test_initialize_bedrock_forwards_contextual_grounding_from_messages(): if isinstance(callback, BedrockGuardrail) and callback.guardrail_name == "test_bedrock_grounding_from_messages" ] assert initialized, "bedrock guardrail was not registered as a callback" - actual_request = asyncio.run( - initialized[-1].convert_to_bedrock_format(source="OUTPUT", response=response, messages=messages) + actual_request = initialized[-1].convert_to_bedrock_format( + source="OUTPUT", response=response, messages=messages ) assert json.loads(json.dumps(actual_request)) == expected_request diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index a460478d2e5..13482174762 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -2904,7 +2904,7 @@ class _BaseSignatureExtractHandler(AnthropicMessagesHandler): class _HeadSignatureRecordingHandler(AnthropicMessagesHandler): - seen_scan_attachments: Optional[bool] = None + seen_scan_attachments: bool | None = None @classmethod def _extract_input_text_and_images( @@ -2966,3 +2966,102 @@ class TestExtractHookBaseSignatureCompatibility: assert _HeadSignatureRecordingHandler.seen_scan_attachments is True assert guardrail.inputs is not None assert guardrail.inputs["files"] + + +class _LegacyNestedHooksHandler(AnthropicMessagesHandler): + """Base-shape overrides of every nested extraction hook (merge-base signatures).""" + + seen_tool_result: Optional[bool] = None + seen_image_sources: Optional[bool] = None + + @classmethod + def _extract_content_block( + cls, content_item, msg_idx, content_idx, skip_tool_message, scan_only_tool_results=False + ): + return super()._extract_content_block( + content_item=content_item, + msg_idx=msg_idx, + content_idx=content_idx, + skip_tool_message=skip_tool_message, + scan_only_tool_results=scan_only_tool_results, + ) + + @classmethod + def _extract_tool_result(cls, content_item, msg_idx, content_idx): + cls.seen_tool_result = True + return super()._extract_tool_result( + content_item=content_item, msg_idx=msg_idx, content_idx=content_idx + ) + + @staticmethod + def _image_sources(block): + _LegacyNestedHooksHandler.seen_image_sources = True + return AnthropicMessagesHandler._image_sources(block, False) + + +class _HeadNestedRecordingHandler(AnthropicMessagesHandler): + seen_scan_attachments: bool | None = None + + @classmethod + def _extract_tool_result(cls, content_item, msg_idx, content_idx, scan_attachments=False): + cls.seen_scan_attachments = scan_attachments + return super()._extract_tool_result( + content_item=content_item, + msg_idx=msg_idx, + content_idx=content_idx, + scan_attachments=scan_attachments, + ) + + +class TestNestedHookBaseSignatureCompatibility: + @pytest.mark.asyncio + async def test_legacy_signature_subclasses_run_for_plain_guardrail(self): + """A plain guardrail must not force the attachment kwarg onto old nested hooks.""" + _LegacyNestedHooksHandler.seen_tool_result = None + _LegacyNestedHooksHandler.seen_image_sources = None + guardrail = MockMaskingGuardrail() + data = { + "model": "claude-3", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "hello"}, + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": "iVBOR"}, + }, + {"type": "tool_result", "tool_use_id": "t1", "content": "from tool"}, + ], + } + ], + } + + await _LegacyNestedHooksHandler().process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert _LegacyNestedHooksHandler.seen_tool_result is True + assert _LegacyNestedHooksHandler.seen_image_sources is True + assert guardrail.inputs is not None + assert "from tool" in guardrail.inputs["texts"] + + @pytest.mark.asyncio + async def test_attachment_guardrail_still_passes_scan_attachments_to_nested_hooks(self): + """With scans_attachments on, the new kwarg reaches nested extraction hooks.""" + _HeadNestedRecordingHandler.seen_scan_attachments = None + guardrail = MockMaskingGuardrail() + guardrail.scans_attachments = True + data = { + "model": "claude-3", + "messages": [ + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "from tool"}, + ], + } + ], + } + + await _HeadNestedRecordingHandler().process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert _HeadNestedRecordingHandler.seen_scan_attachments is True