fix(guardrails): refuse serialized tool output attachments only on the responses bridge

A chat tool string reaches the model as text and is scanned as text, so
the serialized refusal only applies where the Responses to chat bridge
packs attachment parts into a string. Structural payloads (inline bytes,
data urls, file references) still refuse; metadata strings like a source
name or an api url no longer do.

Co-authored-by: feng.tsai <fengtsai@berri.ai>
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-24 08:42:14 +00:00
parent ff8aa061d0
commit c63bae89db
3 changed files with 303 additions and 27 deletions

View file

@ -260,6 +260,21 @@ def _redact_assessment_match_fields(assessments: list[dict]) -> list[dict]:
_RESPONSES_API_CALL_TYPES: Final = frozenset({CallTypes.responses, CallTypes.aresponses})
_RESPONSES_API_CALL_TYPE_VALUES: Final[frozenset[str]] = frozenset(
call_type.value for call_type in _RESPONSES_API_CALL_TYPES
)
_TOOL_OUTPUT_REFERENCE_KEYS: Final[frozenset[str]] = frozenset(("file_id", "file_url", "file_data", "image_url"))
def _is_attachment_payload(key: str, value: object) -> bool:
"""Inline bytes or an attachment reference, never a plain metadata string."""
if isinstance(value, Mapping):
return bool(value)
if not isinstance(value, str) or not value:
return False
if key in _TOOL_OUTPUT_REFERENCE_KEYS:
return True
return value.startswith("data:")
def _decoded_base64_length(encoded: str) -> int:
@ -715,7 +730,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
)
self._refuse_unscannable_image_ref(image_url=image_ref)
def _refuse_unscannable_leaf_parts(self, messages: "Sequence[AllMessageValues]") -> None:
def _refuse_unscannable_leaf_parts(
self, messages: "Sequence[AllMessageValues]", *, bridged_tool_output: bool
) -> None:
"""Raise the attachment refusal for any leaf part the guardrail cannot scan.
Runs on the unscoped message list so an attachment hiding in a message role
@ -725,10 +742,15 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
for message in messages:
for leaf in _content_leaf_parts(message.get("content")):
self._refuse_unscannable_part(part=leaf)
self._refuse_serialized_tool_output(message=message)
if bridged_tool_output:
self._refuse_serialized_tool_output(message=message)
def _refuse_serialized_tool_output(self, message: Mapping[str, object]) -> None:
"""Refuse an attachment hiding in a ``tool`` message's serialized output."""
"""Refuse an attachment hiding in a ``tool`` message's serialized output.
A chat tool string reaches the model as text and is scanned as text; only the
Responses to chat bridge serializes attachment parts into a string.
"""
if message.get("role") != "tool":
return
content: Final = message.get("content")
@ -736,7 +758,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
return
for part in _serialized_tool_output_parts(content):
if part.get("type") in _TOOL_OUTPUT_ATTACHMENT_TYPES and any(
part.get(key) for key in _TOOL_OUTPUT_PAYLOAD_KEYS
_is_attachment_payload(key, part.get(key)) for key in _TOOL_OUTPUT_PAYLOAD_KEYS
):
self._handle_unscannable_attachment(reason="a tool output attachment cannot be scanned")
@ -2970,7 +2992,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
# A file or unscannable image is refused even when role scoping below drops
# its message from the scan set; the ApplyGuardrail call itself stays scoped
self._refuse_unscannable_leaf_parts(messages=new_messages)
self._refuse_unscannable_leaf_parts(
messages=new_messages, bridged_tool_output=call_type in _RESPONSES_API_CALL_TYPE_VALUES
)
filter_result: Final = self._prepare_guardrail_messages_for_role(messages=new_messages)
filtered_messages: Final = filter_result.payload_messages

View file

@ -1249,9 +1249,7 @@ def test_non_bedrock_policy_forwards_non_string_image_url(gateway: Gateway, tmp_
@pytest.mark.covers("other.observability.guardrails.bedrock_during_call_tool_output_metadata_passes_as_text")
def test_bedrock_during_call_tool_output_without_payload_keys_scans_as_text(
gateway: Gateway, tmp_path: Path
) -> None:
def test_bedrock_during_call_tool_output_without_payload_keys_scans_as_text(gateway: Gateway, tmp_path: Path) -> None:
tool_output: Final = json.dumps(
[
{"type": "file", "name": "README.md", "path": "README.md", "size": 1200},
@ -1317,3 +1315,78 @@ def test_bedrock_during_call_tool_output_without_payload_keys_scans_as_text(
assert response.status_code == 200, response.text
eventually(lambda: policy.drain(), lambda values: len(values) == 1, seconds=10)
assert len(upstream.drain()) == 1
@pytest.mark.covers("other.observability.guardrails.bedrock_during_call_tool_output_url_members_scan_as_text")
def test_bedrock_during_call_tool_output_url_members_scan_as_text(gateway: Gateway, tmp_path: Path) -> None:
tool_output: Final = json.dumps(
[
{
"type": "file",
"name": "README.md",
"path": "README.md",
"size": 1200,
"url": "https://api.github.com/repos/o/r/contents/README.md",
},
{"type": "dir", "name": "src", "path": "src", "url": "https://api.github.com/repos/o/r/contents/src"},
]
)
def guardrail(request: Request) -> Reply:
body: Final = json.loads(request.body)
texts: Final = [item["text"]["text"] for item in body["content"] if "text" in item]
assert tool_output in texts, body
return Reply(body=b'{"action":"NONE","outputs":[],"assessments":[]}')
def chat_reply(request: Request) -> Reply:
return Reply(
body=json.dumps(
{
"id": "chatcmpl-wire",
"object": "chat.completion",
"created": 0,
"model": "gpt-4o-mini",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "done"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
).encode()
)
with wire_server(guardrail) as policy, wire_server(chat_reply) as upstream:
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
bedrock_params: Final = _bedrock_policy(policy.url)
bedrock_params["mode"] = "during_call"
config["guardrails"] = [
{"guardrail_name": "bedrock-during-" + uuid.uuid4().hex, "litellm_params": bedrock_params}
]
path: Final = tmp_path / "bedrock-tool-url.yaml"
path.write_text(yaml.safe_dump(config))
with owned_proxy(gateway, tmp_path, {}, config=path) as candidate, candidate.scenario() as scenario:
model: Final = scenario.model(api_base=upstream.url)
response: Final = candidate.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [
{"role": "user", "content": "list files"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "t1", "type": "function", "function": {"name": "ls", "arguments": "{}"}}
],
},
{"role": "tool", "tool_call_id": "t1", "content": tool_output},
],
},
)
assert response.status_code == 200, response.text
eventually(lambda: policy.drain(), lambda values: len(values) == 1, seconds=10)
assert len(upstream.drain()) == 1

View file

@ -7167,7 +7167,7 @@ def test_apply_masking_to_messages_masks_tool_result_inner_list_items() -> None:
@pytest.mark.asyncio
async def test_during_call_hook_refuses_serialized_tool_output_attachment():
"""A tool message's JSON-serialized output list can hide an attachment part that
never becomes a leaf; the unscoped pass has to refuse it before any scan."""
never becomes a leaf; on the Responses bridge the unscoped pass refuses it."""
guardrail = BedrockGuardrail(
guardrail_name="bedrock-tool-output-file",
guardrailIdentifier="test-guardrail",
@ -7177,21 +7177,20 @@ async def test_during_call_hook_refuses_serialized_tool_output_attachment():
)
data = {
"model": "gpt-4o-mini",
"messages": [
"input": [
{"type": "function_call", "call_id": "c1", "name": "read", "arguments": "{}"},
{
"role": "tool",
"tool_call_id": "c1",
"content": json.dumps(
[
{
"type": "input_file",
"filename": "a.pdf",
"file_data": "data:application/pdf;base64,JVBERi0=",
}
]
),
"type": "function_call_output",
"call_id": "c1",
"output": [
{
"type": "input_file",
"filename": "a.pdf",
"file_data": "data:application/pdf;base64,JVBERi0=",
}
],
},
{"role": "user", "content": "hi"},
{"role": "user", "content": [{"type": "input_text", "text": "hi"}]},
],
}
@ -7202,13 +7201,56 @@ async def test_during_call_hook_refuses_serialized_tool_output_attachment():
await guardrail.async_moderation_hook(
data=data,
user_api_key_dict=UserAPIKeyAuth(),
call_type=CallTypes.acompletion.value,
call_type=CallTypes.aresponses.value,
)
assert exc_info.value.status_code == 400
mock_post.assert_not_called()
@pytest.mark.asyncio
async def test_during_call_hook_chat_tool_output_attachment_scans_as_text():
"""On a plain chat call a serialized tool output reaches the model as text, so the
serialized pass must not refuse it even when a member carries attachment bytes."""
guardrail = BedrockGuardrail(
guardrail_name="bedrock-chat-tool-string",
guardrailIdentifier="test-guardrail",
guardrailVersion="DRAFT",
event_hook=GuardrailEventHooks.during_call,
default_on=True,
)
mock_credentials = MagicMock()
mock_credentials.access_key = "test-access-key"
mock_credentials.secret_key = "test-secret-key"
mock_credentials.token = None
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {"action": "NONE", "assessments": []}
tool_output = json.dumps(
[{"type": "input_file", "filename": "a.pdf", "file_data": "data:application/pdf;base64,JVBERi0="}]
)
data = {
"model": "gpt-4o-mini",
"messages": [
{"role": "tool", "tool_call_id": "c1", "content": tool_output},
{"role": "user", "content": "hi"},
],
}
with (
patch.object(guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1")),
patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post,
):
mock_post.return_value = mock_response
await guardrail.async_moderation_hook(
data=data,
user_api_key_dict=UserAPIKeyAuth(),
call_type=CallTypes.acompletion.value,
)
mock_post.assert_called_once()
@pytest.mark.asyncio
async def test_during_call_hook_refuses_serialized_tool_output_attachment_unknown_shape_falls_through():
"""Non-attachment members of the serialized tool output are left to the normal scan."""
@ -7372,16 +7414,17 @@ async def test_during_call_hook_refuses_tool_output_file_id_and_scoped_media():
{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}},
):
data = {
"messages": [
{"role": "tool", "content": json.dumps([member])},
{"role": "user", "content": "summarize"},
"input": [
{"type": "function_call", "call_id": "c1", "name": "read", "arguments": "{}"},
{"type": "function_call_output", "call_id": "c1", "output": [member]},
{"role": "user", "content": [{"type": "input_text", "text": "summarize"}]},
]
}
with pytest.raises(HTTPException) as exc_info:
await guardrail.async_moderation_hook(
data=data,
user_api_key_dict=UserAPIKeyAuth(),
call_type=CallTypes.acompletion.value,
call_type=CallTypes.aresponses.value,
)
assert exc_info.value.status_code == 400
@ -7550,3 +7593,139 @@ async def test_during_call_hook_tool_output_metadata_only_parts_scan_as_text():
sent_body = json.loads(mock_post.call_args.kwargs["data"])
sent_texts = [item["text"]["text"] for item in sent_body["content"] if "text" in item]
assert tool_output in sent_texts
@pytest.mark.asyncio
@pytest.mark.parametrize(
"member",
[
{"type": "file", "name": "README.md", "path": "README.md", "size": 1200},
{"type": "file", "name": "README.md", "url": "https://api.github.com/repos/o/r/contents/README.md"},
{"type": "image", "title": "Golden Gate", "source": "wikimedia.org"},
{"type": "document", "id": "kb-17", "source": "confluence", "text": "Refund policy"},
],
)
async def test_during_call_hook_chat_tool_output_metadata_members_scan_as_text(member):
"""Metadata-shaped members in a chat tool string are plain text to the scan."""
guardrail = BedrockGuardrail(
guardrail_name="bedrock-chat-tool-metadata",
guardrailIdentifier="test-guardrail",
guardrailVersion="DRAFT",
event_hook=GuardrailEventHooks.during_call,
default_on=True,
)
mock_credentials = MagicMock()
mock_credentials.access_key = "test-access-key"
mock_credentials.secret_key = "test-secret-key"
mock_credentials.token = None
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {"action": "NONE", "assessments": []}
data = {
"messages": [
{"role": "tool", "tool_call_id": "t1", "content": json.dumps([member])},
{"role": "user", "content": "summarize"},
]
}
with (
patch.object(guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1")),
patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post,
):
mock_post.return_value = mock_response
await guardrail.async_moderation_hook(
data=data,
user_api_key_dict=UserAPIKeyAuth(),
call_type=CallTypes.acompletion.value,
)
mock_post.assert_called_once()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"member",
[
{"type": "document", "id": "kb-17", "source": "confluence"},
{"type": "file", "name": "README.md", "url": "https://api.github.com/repos/o/r/contents/README.md"},
],
)
async def test_during_call_hook_responses_tool_output_metadata_members_scan_as_text(member):
"""On the Responses bridge only structural payloads refuse; plain metadata strings
in a serialized tool output are still scanned as text."""
guardrail = BedrockGuardrail(
guardrail_name="bedrock-resp-tool-metadata",
guardrailIdentifier="test-guardrail",
guardrailVersion="DRAFT",
event_hook=GuardrailEventHooks.during_call,
default_on=True,
)
mock_credentials = MagicMock()
mock_credentials.access_key = "test-access-key"
mock_credentials.secret_key = "test-secret-key"
mock_credentials.token = None
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {"action": "NONE", "assessments": []}
data = {
"input": [
{"type": "function_call", "call_id": "c1", "name": "read", "arguments": "{}"},
{"type": "function_call_output", "call_id": "c1", "output": [member]},
{"role": "user", "content": [{"type": "input_text", "text": "summarize"}]},
]
}
with (
patch.object(guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1")),
patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post,
):
mock_post.return_value = mock_response
await guardrail.async_moderation_hook(
data=data,
user_api_key_dict=UserAPIKeyAuth(),
call_type=CallTypes.aresponses.value,
)
mock_post.assert_called_once()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"member",
[
{"type": "document", "id": "kb-17", "source": {"type": "base64", "data": "x"}},
{"type": "input_file", "filename": "a.pdf", "file_data": "data:application/pdf;base64,AA=="},
{"type": "input_image", "file_id": "f1"},
{"type": "file", "name": "clip", "url": "data:video/mp4;base64,AA=="},
],
)
async def test_during_call_hook_responses_tool_output_payload_members_refused(member):
"""Structural attachment payloads in a serialized tool output refuse on the
Responses bridge: a mapping source, inline file bytes, or a file reference."""
guardrail = BedrockGuardrail(
guardrail_name="bedrock-resp-tool-payloads",
guardrailIdentifier="test-guardrail",
guardrailVersion="DRAFT",
event_hook=GuardrailEventHooks.during_call,
default_on=True,
)
data = {
"input": [
{"type": "function_call", "call_id": "c1", "name": "read", "arguments": "{}"},
{"type": "function_call_output", "call_id": "c1", "output": [member]},
{"role": "user", "content": [{"type": "input_text", "text": "summarize"}]},
]
}
with (
patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post,
pytest.raises(HTTPException) as exc_info,
):
await guardrail.async_moderation_hook(
data=data,
user_api_key_dict=UserAPIKeyAuth(),
call_type=CallTypes.aresponses.value,
)
assert exc_info.value.status_code == 400
mock_post.assert_not_called()