fix(guardrails): keep the base hook call shape for non-attachment guardrails

The extraction hooks only receive scan_attachments and files_to_check when
the applied guardrail sets scans_attachments, so subclasses written
against the pre-attachment signatures keep working for every existing
guardrail. Adds compatibility tests for base- and head-signature
subclasses, a parametrized Responses during_call regression guard showing
text-only function_call_output strings scan as text, and the unit and
integration coverage for the skip_tool_message and sync-signature fixes.

Co-authored-by: feng.tsai <fengtsai@berri.ai>
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-26 08:52:33 +00:00
parent 0884f42c51
commit fddc14ce14
9 changed files with 616 additions and 53 deletions

View file

@ -972,7 +972,7 @@ class AnthropicMessagesHandler(BaseTranslation):
content_idx=content_idx,
skip_tool_message=skip_tool_message,
scan_only_tool_results=scan_only_tool_results,
scan_attachments=scan_attachments,
**({"scan_attachments": True} if scan_attachments else {}),
)
for content_idx, content_item in enumerate(content)
if isinstance(content_item, dict)
@ -1016,7 +1016,7 @@ class AnthropicMessagesHandler(BaseTranslation):
content_item=content_item,
msg_idx=msg_idx,
content_idx=content_idx,
scan_attachments=scan_attachments,
**({"scan_attachments": True} if scan_attachments else {}),
)
if scan_only_tool_results:
@ -1027,7 +1027,11 @@ class AnthropicMessagesHandler(BaseTranslation):
scanned=(
() if text_str is None else (ScannedText(text_str, ContentBlockTextTarget(msg_idx, content_idx)),)
),
images=(cls._image_sources(content_item, scan_attachments) if content_item.get("type") == "image" else ()),
images=(
cls._image_sources(content_item, **({"scan_attachments": True} if scan_attachments else {}))
if content_item.get("type") == "image"
else ()
),
files=(
cls._document_sources(content_item)
if scan_attachments and content_item.get("type") == "document"
@ -1064,7 +1068,9 @@ class AnthropicMessagesHandler(BaseTranslation):
),
images=tuple(
chain.from_iterable(
cls._image_sources(block, scan_attachments) for _, block in blocks if block.get("type") == "image"
cls._image_sources(block, **({"scan_attachments": True} if scan_attachments else {}))
for _, block in blocks
if block.get("type") == "image"
)
),
files=tuple(
@ -1077,7 +1083,7 @@ class AnthropicMessagesHandler(BaseTranslation):
)
@staticmethod
def _image_sources(block: Mapping[str, object], scan_attachments: bool) -> tuple[str, ...]:
def _image_sources(block: Mapping[str, object], scan_attachments: bool = False) -> tuple[str, ...]:
"""Normalize an Anthropic image block into strings a guardrail can read.
base64 becomes a data URI so the format travels with the payload, which is what

View file

@ -184,14 +184,13 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
msg_idx=msg_idx,
texts_to_check=texts_to_check,
images_to_check=images_to_check,
files_to_check=files_to_check,
tool_calls_to_check=tool_calls_to_check,
text_task_mappings=text_task_mappings,
tool_call_task_mappings=tool_call_task_mappings,
skip_system_message=skip_system,
skip_tool_message=skip_tool,
scan_only_tool_results=scan_only_tool_results,
scan_attachments=scan_attachments,
**({"scan_attachments": True, "files_to_check": files_to_check} if scan_attachments else {}),
)
# Step 2: Apply guardrail to all texts and tool calls in batch
@ -335,11 +334,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
msg_idx=msg_idx,
texts_to_check=texts,
images_to_check=images,
files_to_check=files,
tool_calls_to_check=tool_calls,
text_task_mappings=[], # mutable-ok: required by _extract_inputs, unused here
tool_call_task_mappings=[], # mutable-ok: required by _extract_inputs, unused here
scan_attachments=scan_attachments,
**({"scan_attachments": True, "files_to_check": files} if scan_attachments else {}),
)
if texts or tool_calls:
return "no scannable content after message scoping"
@ -364,7 +362,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
msg_idx: int,
texts_to_check: list[str],
images_to_check: list[str],
files_to_check: list[str],
tool_calls_to_check: list[ChatCompletionToolParam],
text_task_mappings: list[tuple[int, int | None]],
tool_call_task_mappings: list[tuple[int, int]],
@ -372,6 +369,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
skip_tool_message: bool = False,
scan_only_tool_results: bool = False,
scan_attachments: bool = False,
files_to_check: list[str] | None = None,
) -> None:
"""
Extract text content, images, and tool calls from a message.
@ -406,7 +404,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
image_ref, file_ref = _attachment_part_refs(content_item, scan_attachments)
if image_ref is not None:
images_to_check.append(image_ref)
if file_ref is not None:
if file_ref is not None and files_to_check is not None:
files_to_check.append(file_ref)
# Extract tool calls (typically in assistant messages)

View file

@ -945,7 +945,7 @@ async def test_convert_to_bedrock_format_input_source():
]
# Call the method
result = await guardrail.convert_to_bedrock_format(source="INPUT", messages=mock_messages)
result = guardrail.convert_to_bedrock_format(source="INPUT", messages=mock_messages)
# Verify the result structure
assert isinstance(result, dict)
@ -1007,7 +1007,7 @@ async def test_convert_to_bedrock_format_output_source():
)
# Call the method
result = await guardrail.convert_to_bedrock_format(
result = guardrail.convert_to_bedrock_format(
source="OUTPUT", response=mock_response
)

View file

@ -1436,3 +1436,50 @@ def test_bedrock_during_call_responses_function_call_output_string_scans_as_text
assert response.status_code == 200, response.text
eventually(lambda: policy.drain(), lambda values: len(values) == 1, seconds=10)
assert len(upstream.drain()) == 1
def test_bedrock_during_call_skip_tool_message_flag_drops_tool_result_text(gateway: Gateway, tmp_path: Path) -> None:
tool_text: Final = "BLOCKME from tool"
def guardrail(request: Request) -> Reply:
body: Final = json.loads(request.body)
texts: Final = [item["text"]["text"] for item in body["content"] if "text" in item]
assert texts == ["hello"], body
return Reply(body=b'{"action":"NONE","outputs":[],"assessments":[]}')
with wire_server(guardrail) as policy, wire_server(_anthropic_reply) as upstream:
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
bedrock_params: Final = _bedrock_policy(policy.url)
bedrock_params["mode"] = "during_call"
bedrock_params["skip_tool_message_in_guardrail"] = True
config["guardrails"] = [
{"guardrail_name": "bedrock-skip-tool-" + uuid.uuid4().hex, "litellm_params": bedrock_params}
]
path: Final = tmp_path / "bedrock-skip-tool.yaml"
path.write_text(yaml.safe_dump(config))
with owned_proxy(gateway, tmp_path, {}, config=path) as candidate, candidate.scenario() as scenario:
model: Final = scenario.model(
model="anthropic/claude-sonnet-4-5-20250929", api_base=upstream.url, api_key="synthetic-anthropic-key"
)
response: Final = candidate.request(
"POST",
"/v1/messages",
{
"model": model,
"max_tokens": 16,
"messages": [
{"role": "user", "content": "hello"},
{
"role": "assistant",
"content": [{"type": "tool_use", "id": "toolu_01A", "name": "lookup", "input": {}}],
},
{
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": "toolu_01A", "content": tool_text}],
},
],
},
)
assert response.status_code == 200, response.text
eventually(lambda: policy.drain(), lambda values: len(values) == 1, seconds=10)
assert len(upstream.drain()) == 1

View file

@ -312,3 +312,128 @@ class TestOpenAIChatHandlerToolsAndModelForwarding:
assert guardrail.calls == 1
assert guardrail.inputs is not None
assert guardrail.inputs["model"] == ["gpt-4o"]
class _BaseSignatureExtractInputsHandler(OpenAIChatCompletionsHandler):
"""A subclass written against the pre-attachment-scanning _extract_inputs signature."""
def _extract_inputs(
self,
message,
msg_idx,
texts_to_check,
images_to_check,
tool_calls_to_check,
text_task_mappings,
tool_call_task_mappings,
skip_system_message=False,
skip_tool_message=False,
scan_only_tool_results=False,
) -> None:
return super()._extract_inputs(
message=message,
msg_idx=msg_idx,
texts_to_check=texts_to_check,
images_to_check=images_to_check,
tool_calls_to_check=tool_calls_to_check,
text_task_mappings=text_task_mappings,
tool_call_task_mappings=tool_call_task_mappings,
skip_system_message=skip_system_message,
skip_tool_message=skip_tool_message,
scan_only_tool_results=scan_only_tool_results,
)
class _HeadSignatureRecordingExtractInputsHandler(OpenAIChatCompletionsHandler):
seen_scan_attachments: bool | None = None
seen_files_to_check_is_not_none: bool | None = None
def _extract_inputs(
self,
message,
msg_idx,
texts_to_check,
images_to_check,
tool_calls_to_check,
text_task_mappings,
tool_call_task_mappings,
skip_system_message=False,
skip_tool_message=False,
scan_only_tool_results=False,
scan_attachments=False,
files_to_check=None,
) -> None:
_HeadSignatureRecordingExtractInputsHandler.seen_scan_attachments = scan_attachments
_HeadSignatureRecordingExtractInputsHandler.seen_files_to_check_is_not_none = files_to_check is not None
return super()._extract_inputs(
message=message,
msg_idx=msg_idx,
texts_to_check=texts_to_check,
images_to_check=images_to_check,
tool_calls_to_check=tool_calls_to_check,
text_task_mappings=text_task_mappings,
tool_call_task_mappings=tool_call_task_mappings,
skip_system_message=skip_system_message,
skip_tool_message=skip_tool_message,
scan_only_tool_results=scan_only_tool_results,
scan_attachments=scan_attachments,
files_to_check=files_to_check,
)
class TestExtractInputsBaseSignatureCompatibility:
@pytest.mark.asyncio
async def test_legacy_signature_subclass_runs_for_plain_guardrail(self):
"""A non-attachment guardrail must not force files_to_check onto old subclasses."""
guardrail = RecordingGuardrail()
data = {
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "summarize this"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg=="},
},
],
}
],
}
await _BaseSignatureExtractInputsHandler().process_input_messages(data=data, guardrail_to_apply=guardrail)
assert guardrail.inputs is not None
assert guardrail.inputs["texts"] == ["summarize this"]
@pytest.mark.asyncio
async def test_attachment_guardrail_still_passes_the_new_kwargs(self):
"""With scans_attachments on, the head signature receives scan_attachments and files."""
_HeadSignatureRecordingExtractInputsHandler.seen_scan_attachments = None
_HeadSignatureRecordingExtractInputsHandler.seen_files_to_check_is_not_none = None
guardrail = ScanningGuardrail()
data = {
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "summarize this"},
{
"type": "file",
"file": {"filename": "a.pdf", "file_data": "data:application/pdf;base64,AAAA"},
},
],
}
],
}
await _HeadSignatureRecordingExtractInputsHandler().process_input_messages(
data=data, guardrail_to_apply=guardrail
)
assert _HeadSignatureRecordingExtractInputsHandler.seen_scan_attachments is True
assert _HeadSignatureRecordingExtractInputsHandler.seen_files_to_check_is_not_none is True
assert guardrail.inputs is not None
assert guardrail.inputs["files"]

View file

@ -2562,15 +2562,13 @@ _GUARD_BLOCK = {"text": {"text": _GROUNDING_RESPONSE_TEXT, "qualifiers": ["guard
def _input_request(messages: list) -> dict:
"""Arrange a guardrail and act: build the Bedrock INPUT payload."""
return asyncio.run(_grounding_guardrail().convert_to_bedrock_format(source="INPUT", messages=messages))
return _grounding_guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
def _output_request(messages: list, response=None, from_messages: bool = False) -> dict:
"""Arrange a guardrail and act: build the Bedrock OUTPUT payload."""
return asyncio.run(
_grounding_guardrail(from_messages).convert_to_bedrock_format(
source="OUTPUT", response=response, messages=messages
)
return _grounding_guardrail(from_messages).convert_to_bedrock_format(
source="OUTPUT", response=response, messages=messages
)
@ -2743,8 +2741,8 @@ def test_grounding_input_ignores_plain_message_derivation():
"content": [{"text": {"text": _GROUNDING_SOURCE_TEXT}}, {"text": {"text": _GROUNDING_QUERY_TEXT}}],
}
actual_request = asyncio.run(
_grounding_guardrail(from_messages=True).convert_to_bedrock_format(source="INPUT", messages=messages)
actual_request = _grounding_guardrail(from_messages=True).convert_to_bedrock_format(
source="INPUT", messages=messages
)
assert actual_request == expected_request
@ -5667,7 +5665,7 @@ class TestBedrockGuardrailImageInput:
}
]
request = await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
request = self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
assert request["content"] == [
{"text": {"text": "what does this say?"}},
@ -5694,7 +5692,7 @@ class TestBedrockGuardrailImageInput:
}
]
request = await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
request = self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
assert request["content"] == [
{"image": {"format": "png", "source": {"bytes": self._PNG_DATA_URI.split(",")[1]}}}
@ -5711,7 +5709,7 @@ class TestBedrockGuardrailImageInput:
]
with pytest.raises(HTTPException) as exc_info:
await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
assert exc_info.value.status_code == 400
@ -5733,7 +5731,7 @@ class TestBedrockGuardrailImageInput:
]
with pytest.raises(HTTPException) as exc_info:
await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
assert exc_info.value.status_code == 400
assert exc_info.value.detail["error"] == "Violated guardrail policy"
@ -5748,7 +5746,7 @@ class TestBedrockGuardrailImageInput:
]
with pytest.raises(HTTPException) as exc_info:
await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
assert exc_info.value.status_code == 400
@ -5765,7 +5763,7 @@ class TestBedrockGuardrailImageInput:
sent: list = []
async def spy(**kwargs):
sent.append(await g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"]))
sent.append(g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"]))
return {"action": "NONE", "outputs": []}
# Every text is already in the session cache: the exact state that made the
@ -5795,7 +5793,7 @@ class TestBedrockGuardrailImageInput:
sent: list = []
async def spy(**kwargs):
sent.append(await g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"]))
sent.append(g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"]))
return {"action": "NONE", "outputs": []}
with patch.object(g, "make_bedrock_api_request", new=spy):
@ -5824,7 +5822,7 @@ class TestBedrockGuardrailImageInput:
}
]
request = await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
request = self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
assert len(request["content"]) == 20
@ -5845,7 +5843,7 @@ class TestBedrockGuardrailImageInput:
) as decode
):
with pytest.raises(HTTPException) as exc_info:
await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
decode.assert_not_awaited()
assert "at most 20 images" in str(exc_info.value.detail)
@ -5867,7 +5865,7 @@ class TestBedrockGuardrailImageInput:
into a content item: an unrecognised part that fell through to the text
branch would be reported to the operator as scanned when it was not.
"""
request = await self._guardrail().convert_to_bedrock_format(
request = self._guardrail().convert_to_bedrock_format(
source="INPUT", messages=[{"role": "user", "content": content}]
)
@ -5876,7 +5874,7 @@ class TestBedrockGuardrailImageInput:
@pytest.mark.asyncio
async def test_a_bare_string_part_is_scanned_as_text(self):
"""A content list may hold plain strings, not only typed parts."""
request = await self._guardrail().convert_to_bedrock_format(
request = self._guardrail().convert_to_bedrock_format(
source="INPUT", messages=[{"role": "user", "content": ["just text"]}]
)
@ -5888,7 +5886,7 @@ class TestBedrockGuardrailImageInput:
Both reach the model as an image, so both have to reach the scan.
"""
request = await self._guardrail().convert_to_bedrock_format(
request = self._guardrail().convert_to_bedrock_format(
source="INPUT",
messages=[{"role": "user", "content": [{"type": "image_url", "image_url": self._PNG_DATA_URI}]}],
)
@ -5956,7 +5954,7 @@ class TestBedrockGuardrailImageInput:
new_callable=AsyncMock,
) as decode:
with pytest.raises(HTTPException) as exc_info:
await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
decode.assert_not_awaited()
assert "remote image URLs are not supported" in str(exc_info.value.detail)
@ -6074,7 +6072,7 @@ class TestBedrockGuardrailImageInput:
]
with pytest.raises(HTTPException) as exc_info:
await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
assert "Violated guardrail policy" in str(exc_info.value.detail)
@ -6095,7 +6093,7 @@ class TestBedrockGuardrailImageInput:
]
with pytest.raises(HTTPException) as exc_info:
await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
assert "Violated guardrail policy" in str(exc_info.value.detail)
@ -6119,7 +6117,7 @@ class TestBedrockGuardrailImageInput:
}
]
request = await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
request = self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
assert request["content"] == [
{"text": {"text": "what does this say?"}},
@ -6140,7 +6138,7 @@ class TestBedrockGuardrailImageInput:
]
with pytest.raises(HTTPException) as exc_info:
await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
assert "Violated guardrail policy" in str(exc_info.value.detail)
@ -6189,7 +6187,7 @@ class TestBedrockGuardrailImageInput:
sent: list = []
async def spy(**kwargs):
sent.append(await g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"]))
sent.append(g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"]))
return {"action": "NONE", "outputs": []}
with patch.object(g, "make_bedrock_api_request", new=spy):
@ -6226,7 +6224,7 @@ class TestBedrockGuardrailImageInput:
sent: list = []
async def spy(**kwargs):
sent.append(await g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"]))
sent.append(g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"]))
return {"action": "NONE", "outputs": []}
with patch.object(g, "make_bedrock_api_request", new=spy):
@ -6304,7 +6302,7 @@ class TestBedrockGuardrailImageInput:
]
with pytest.raises(HTTPException) as exc_info:
await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
assert "Violated guardrail policy" in str(exc_info.value.detail)
@ -6334,7 +6332,7 @@ class TestBedrockGuardrailImageInput:
}
]
request = await g.convert_to_bedrock_format(source="INPUT", messages=messages)
request = g.convert_to_bedrock_format(source="INPUT", messages=messages)
kinds = [key for item in request["content"] for key in item]
assert "image" in kinds
@ -6347,7 +6345,7 @@ class TestBedrockGuardrailImageInput:
messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"file_id": "file_abc"}}]}]
with pytest.raises(HTTPException) as exc_info:
await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
assert "Violated guardrail policy" in str(exc_info.value.detail)
@ -6367,7 +6365,7 @@ class TestBedrockGuardrailImageInput:
messages = [{"role": "user", "content": [part]}]
with pytest.raises(HTTPException) as exc_info:
await self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
self._guardrail().convert_to_bedrock_format(source="INPUT", messages=messages)
assert "image part carries no inline url" in str(exc_info.value.detail)
@ -6394,7 +6392,7 @@ class TestBedrockGuardrailImageInput:
sent: list = []
async def spy(**kwargs):
sent.append(await g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"]))
sent.append(g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"]))
return {"action": "NONE", "outputs": []}
with patch.object(g, "make_bedrock_api_request", new=spy):
@ -6419,7 +6417,7 @@ class TestBedrockGuardrailImageInput:
sent: list = []
async def spy(**kwargs):
sent.append(await g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"]))
sent.append(g.convert_to_bedrock_format(source="INPUT", messages=kwargs["messages"]))
return {"action": "NONE", "outputs": []}
bare_base64 = self._PNG_DATA_URI.split(",")[1]
@ -6463,7 +6461,7 @@ class TestBedrockGuardrailImageInput:
# cast-ok: pins the runtime fallback the Literal forbids at type-check time
source = cast(Literal["INPUT", "OUTPUT"], "SOMETHING_ELSE")
request = await self._guardrail().convert_to_bedrock_format(
request = self._guardrail().convert_to_bedrock_format(
source=source,
messages=[{"role": "user", "content": "hello"}],
)
@ -6480,7 +6478,7 @@ class TestBedrockGuardrailImageInput:
oversized_png = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"\x00" * (5 * 1024 * 1024)).decode()
with pytest.raises(HTTPException) as exc_info:
await self._guardrail().convert_to_bedrock_format(
self._guardrail().convert_to_bedrock_format(
source="INPUT",
messages=[
{
@ -7768,3 +7766,294 @@ async def test_during_call_hook_responses_tool_output_input_image_member_refused
assert exc_info.value.status_code == 400
mock_post.assert_not_called()
def _skip_flag_tool_result_messages() -> list[dict[str, object]]:
return [
{"role": "user", "content": "hello"},
{
"role": "assistant",
"content": [{"type": "tool_use", "id": "toolu_01A", "name": "get_weather", "input": {}}],
},
{
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": "toolu_01A", "content": "BLOCKME from tool"}],
},
]
def _sent_apply_guardrail_texts(mock_post) -> list[str]:
sent_body = json.loads(mock_post.call_args.kwargs["data"])
return [item["text"]["text"] for item in sent_body["content"] if "text" in item]
async def _run_anthropic_during_call(guardrail, messages, response_json=None):
"""Drive async_moderation_hook on an anthropic_messages call; return (data, post mock)."""
mock_credentials = MagicMock()
mock_credentials.access_key = "test-access-key"
mock_credentials.secret_key = "test-secret-key"
mock_credentials.token = None
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = response_json or {"action": "NONE", "assessments": []}
data = {"model": "claude-sonnet-4-5", "messages": messages}
with (
patch.object(guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1")),
patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post,
):
mock_post.return_value = mock_response
await guardrail.async_moderation_hook(
data=data,
user_api_key_dict=UserAPIKeyAuth(),
call_type=CallTypes.anthropic_messages.value,
)
return data, mock_post
@pytest.mark.asyncio
async def test_during_call_hook_scans_tool_result_text_when_skip_flag_off():
"""Without skip_tool_message_in_guardrail a tool_result string is scanned like base."""
guardrail = BedrockGuardrail(
guardrail_name="bedrock-tool-result-default",
guardrailIdentifier="test-guardrail",
guardrailVersion="DRAFT",
event_hook=GuardrailEventHooks.during_call,
default_on=True,
)
_, mock_post = await _run_anthropic_during_call(guardrail, _skip_flag_tool_result_messages())
mock_post.assert_called_once()
assert _sent_apply_guardrail_texts(mock_post) == ["hello", "BLOCKME from tool"]
@pytest.mark.asyncio
async def test_during_call_hook_skip_tool_message_flag_drops_tool_result_text():
"""skip_tool_message_in_guardrail removes the tool_result string from the INPUT scan."""
guardrail = BedrockGuardrail(
guardrail_name="bedrock-tool-result-skip",
guardrailIdentifier="test-guardrail",
guardrailVersion="DRAFT",
event_hook=GuardrailEventHooks.during_call,
default_on=True,
skip_tool_message_in_guardrail=True,
)
_, mock_post = await _run_anthropic_during_call(guardrail, _skip_flag_tool_result_messages())
mock_post.assert_called_once()
assert _sent_apply_guardrail_texts(mock_post) == ["hello"]
@pytest.mark.asyncio
async def test_during_call_hook_global_skip_tool_message_flag_drops_tool_result_text(monkeypatch):
"""The litellm-level flag applies when the per-guardrail attr is unset."""
monkeypatch.setattr(litellm, "skip_tool_message_in_guardrail", True)
guardrail = BedrockGuardrail(
guardrail_name="bedrock-tool-result-skip-global",
guardrailIdentifier="test-guardrail",
guardrailVersion="DRAFT",
event_hook=GuardrailEventHooks.during_call,
default_on=True,
)
_, mock_post = await _run_anthropic_during_call(guardrail, _skip_flag_tool_result_messages())
mock_post.assert_called_once()
assert _sent_apply_guardrail_texts(mock_post) == ["hello"]
@pytest.mark.asyncio
async def test_during_call_hook_skip_tool_message_flag_keeps_tool_result_out_of_masked_writeback():
"""Masked outputs align to scanned texts; a skipped tool_result consumes no slot."""
guardrail = BedrockGuardrail(
guardrail_name="bedrock-tool-result-skip-mask",
guardrailIdentifier="test-guardrail",
guardrailVersion="DRAFT",
event_hook=GuardrailEventHooks.during_call,
default_on=True,
skip_tool_message_in_guardrail=True,
)
messages = [
{"role": "user", "content": "hello"},
{
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": "toolu_01A", "content": "BLOCKME from tool"}],
},
{"role": "user", "content": "and bye"},
]
data, mock_post = await _run_anthropic_during_call(
guardrail,
messages,
response_json={"action": "GUARDRAIL_INTERVENED", "outputs": [{"text": "M1"}, {"text": "M2"}]},
)
assert _sent_apply_guardrail_texts(mock_post) == ["hello", "and bye"]
assert data["messages"][0]["content"] == "M1"
assert data["messages"][1]["content"] == [
{"type": "tool_result", "tool_use_id": "toolu_01A", "content": "BLOCKME from tool"}
]
assert data["messages"][2]["content"] == "M2"
@pytest.mark.asyncio
async def test_during_call_hook_skip_tool_message_flag_tool_result_only_turn_posts_nothing():
"""Flag on + latest-role scoping over a tool_result-only turn scans nothing."""
guardrail = BedrockGuardrail(
guardrail_name="bedrock-tool-result-skip-empty",
guardrailIdentifier="test-guardrail",
guardrailVersion="DRAFT",
event_hook=GuardrailEventHooks.during_call,
default_on=True,
experimental_use_latest_role_message_only=True,
skip_tool_message_in_guardrail=True,
)
_, mock_post = await _run_anthropic_during_call(guardrail, _anthropic_tool_result_conversation())
mock_post.assert_not_called()
@pytest.mark.asyncio
async def test_during_call_hook_skip_tool_message_flag_still_refuses_tool_result_document():
"""The skip flag narrows text scanning only; attachments inside it stay refused."""
guardrail = BedrockGuardrail(
guardrail_name="bedrock-tool-result-skip-doc",
guardrailIdentifier="test-guardrail",
guardrailVersion="DRAFT",
event_hook=GuardrailEventHooks.during_call,
default_on=True,
skip_tool_message_in_guardrail=True,
)
messages = [
{"role": "user", "content": "hello"},
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "toolu_01A",
"content": [
{
"type": "document",
"source": {"type": "base64", "media_type": "application/pdf", "data": "AAAA"},
}
],
}
],
},
]
with (
patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post,
pytest.raises(HTTPException) as exc_info,
):
await guardrail.async_moderation_hook(
data={"model": "claude-sonnet-4-5", "messages": messages},
user_api_key_dict=UserAPIKeyAuth(),
call_type=CallTypes.anthropic_messages.value,
)
assert exc_info.value.status_code == 400
mock_post.assert_not_called()
def test_convert_to_bedrock_format_returns_bedrock_request_synchronously():
"""The public INPUT converter is sync again; callers never await it."""
guardrail = BedrockGuardrail(guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT")
request = guardrail.convert_to_bedrock_format(source="INPUT", messages=[{"role": "user", "content": "hi"}])
import inspect
assert inspect.isawaitable(request) is False
assert request["content"][0]["text"]["text"] == "hi"
def test_convert_to_bedrock_format_inline_png_builds_an_image_item():
"""An inline png data URI decodes into an ApplyGuardrail image item without IO."""
guardrail = BedrockGuardrail(guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT")
messages = [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg=="}}
],
}
]
request = guardrail.convert_to_bedrock_format(source="INPUT", messages=messages)
assert request["content"][0]["image"]["format"] == "png"
@pytest.mark.asyncio
async def test_during_call_hook_refuses_remote_image_without_calling_a_fetcher():
"""A remote image is refused by the substring test before any sync/async fetch."""
guardrail = BedrockGuardrail(
guardrail_name="bedrock-remote-image",
guardrailIdentifier="test-guardrail",
guardrailVersion="DRAFT",
event_hook=GuardrailEventHooks.during_call,
default_on=True,
)
data = {
"model": "gpt-4o-mini",
"messages": [
{
"role": "user",
"content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}],
}
],
}
with (
patch(
"litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails.BedrockImageProcessor.get_image_details",
side_effect=AssertionError("sync fetch reached"),
) as sync_fetch,
patch(
"litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails.BedrockImageProcessor.get_image_details_async",
new_callable=AsyncMock,
side_effect=AssertionError("async fetch reached"),
) as async_fetch,
patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post,
pytest.raises(HTTPException) as exc_info,
):
await guardrail.async_moderation_hook(
data=data,
user_api_key_dict=UserAPIKeyAuth(),
call_type=CallTypes.acompletion.value,
)
assert exc_info.value.status_code == 400
sync_fetch.assert_not_called()
async_fetch.assert_not_called()
mock_post.assert_not_called()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"output",
[
json.dumps([{"type": "file", "name": "README.md", "url": "https://api.github.com/x"}]),
json.dumps([{"type": "file", "name": "q3.pdf", "file_id": "file-abc"}]),
json.dumps([{"type": "image_url", "image_url": "https://example.com/cat.jpg"}]),
json.dumps([{"type": "document", "source": {"title": "wiki", "page": 3}, "text": "hello"}]),
[{"type": "input_text", "text": json.dumps([{"type": "file", "file_id": "file-abc"}])}],
],
)
async def test_during_call_hook_responses_function_call_output_json_string_scans_as_text(output):
"""Client string outputs are never re-parsed; JSON-shaped tool output scans as text."""
guardrail = BedrockGuardrail(
guardrail_name="bedrock-resp-tool-json-text",
guardrailIdentifier="test-guardrail",
guardrailVersion="DRAFT",
event_hook=GuardrailEventHooks.during_call,
default_on=True,
)
mock_post = await _run_responses_during_call(guardrail, output)
mock_post.assert_called_once()

View file

@ -1,7 +1,7 @@
import json
from datetime import datetime
from typing import Dict, List, Optional
from unittest.mock import AsyncMock
from unittest.mock import AsyncMock, MagicMock
import pytest
@ -945,7 +945,7 @@ async def test_bedrock_guardrail_make_api_request_passes_api_key():
),
patch.object(guardrail_hook, "_load_credentials") as mock_load_creds,
patch.object(
guardrail_hook, "convert_to_bedrock_format", new_callable=AsyncMock
guardrail_hook, "convert_to_bedrock_format", new_callable=MagicMock
) as mock_convert,
patch.object(
guardrail_hook, "get_guardrail_dynamic_request_body_params"

View file

@ -1,4 +1,3 @@
import asyncio
import json
from unittest.mock import MagicMock, patch
@ -114,8 +113,8 @@ def test_initialize_bedrock_forwards_contextual_grounding_from_messages():
if isinstance(callback, BedrockGuardrail) and callback.guardrail_name == "test_bedrock_grounding_from_messages"
]
assert initialized, "bedrock guardrail was not registered as a callback"
actual_request = asyncio.run(
initialized[-1].convert_to_bedrock_format(source="OUTPUT", response=response, messages=messages)
actual_request = initialized[-1].convert_to_bedrock_format(
source="OUTPUT", response=response, messages=messages
)
assert json.loads(json.dumps(actual_request)) == expected_request

View file

@ -2904,7 +2904,7 @@ class _BaseSignatureExtractHandler(AnthropicMessagesHandler):
class _HeadSignatureRecordingHandler(AnthropicMessagesHandler):
seen_scan_attachments: Optional[bool] = None
seen_scan_attachments: bool | None = None
@classmethod
def _extract_input_text_and_images(
@ -2966,3 +2966,102 @@ class TestExtractHookBaseSignatureCompatibility:
assert _HeadSignatureRecordingHandler.seen_scan_attachments is True
assert guardrail.inputs is not None
assert guardrail.inputs["files"]
class _LegacyNestedHooksHandler(AnthropicMessagesHandler):
"""Base-shape overrides of every nested extraction hook (merge-base signatures)."""
seen_tool_result: Optional[bool] = None
seen_image_sources: Optional[bool] = None
@classmethod
def _extract_content_block(
cls, content_item, msg_idx, content_idx, skip_tool_message, scan_only_tool_results=False
):
return super()._extract_content_block(
content_item=content_item,
msg_idx=msg_idx,
content_idx=content_idx,
skip_tool_message=skip_tool_message,
scan_only_tool_results=scan_only_tool_results,
)
@classmethod
def _extract_tool_result(cls, content_item, msg_idx, content_idx):
cls.seen_tool_result = True
return super()._extract_tool_result(
content_item=content_item, msg_idx=msg_idx, content_idx=content_idx
)
@staticmethod
def _image_sources(block):
_LegacyNestedHooksHandler.seen_image_sources = True
return AnthropicMessagesHandler._image_sources(block, False)
class _HeadNestedRecordingHandler(AnthropicMessagesHandler):
seen_scan_attachments: bool | None = None
@classmethod
def _extract_tool_result(cls, content_item, msg_idx, content_idx, scan_attachments=False):
cls.seen_scan_attachments = scan_attachments
return super()._extract_tool_result(
content_item=content_item,
msg_idx=msg_idx,
content_idx=content_idx,
scan_attachments=scan_attachments,
)
class TestNestedHookBaseSignatureCompatibility:
@pytest.mark.asyncio
async def test_legacy_signature_subclasses_run_for_plain_guardrail(self):
"""A plain guardrail must not force the attachment kwarg onto old nested hooks."""
_LegacyNestedHooksHandler.seen_tool_result = None
_LegacyNestedHooksHandler.seen_image_sources = None
guardrail = MockMaskingGuardrail()
data = {
"model": "claude-3",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "hello"},
{
"type": "image",
"source": {"type": "base64", "media_type": "image/png", "data": "iVBOR"},
},
{"type": "tool_result", "tool_use_id": "t1", "content": "from tool"},
],
}
],
}
await _LegacyNestedHooksHandler().process_input_messages(data=data, guardrail_to_apply=guardrail)
assert _LegacyNestedHooksHandler.seen_tool_result is True
assert _LegacyNestedHooksHandler.seen_image_sources is True
assert guardrail.inputs is not None
assert "from tool" in guardrail.inputs["texts"]
@pytest.mark.asyncio
async def test_attachment_guardrail_still_passes_scan_attachments_to_nested_hooks(self):
"""With scans_attachments on, the new kwarg reaches nested extraction hooks."""
_HeadNestedRecordingHandler.seen_scan_attachments = None
guardrail = MockMaskingGuardrail()
guardrail.scans_attachments = True
data = {
"model": "claude-3",
"messages": [
{
"role": "user",
"content": [
{"type": "tool_result", "tool_use_id": "t1", "content": "from tool"},
],
}
],
}
await _HeadNestedRecordingHandler().process_input_messages(data=data, guardrail_to_apply=guardrail)
assert _HeadNestedRecordingHandler.seen_scan_attachments is True