mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(guardrails): refuse serialized tool output attachments only on the responses bridge
A chat tool string reaches the model as text and is scanned as text, so the serialized refusal only applies where the Responses to chat bridge packs attachment parts into a string. Structural payloads (inline bytes, data urls, file references) still refuse; metadata strings like a source name or an api url no longer do. Co-authored-by: feng.tsai <fengtsai@berri.ai> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
ff8aa061d0
commit
c63bae89db
3 changed files with 303 additions and 27 deletions
|
|
@ -260,6 +260,21 @@ def _redact_assessment_match_fields(assessments: list[dict]) -> list[dict]:
|
|||
|
||||
|
||||
_RESPONSES_API_CALL_TYPES: Final = frozenset({CallTypes.responses, CallTypes.aresponses})
|
||||
_RESPONSES_API_CALL_TYPE_VALUES: Final[frozenset[str]] = frozenset(
|
||||
call_type.value for call_type in _RESPONSES_API_CALL_TYPES
|
||||
)
|
||||
_TOOL_OUTPUT_REFERENCE_KEYS: Final[frozenset[str]] = frozenset(("file_id", "file_url", "file_data", "image_url"))
|
||||
|
||||
|
||||
def _is_attachment_payload(key: str, value: object) -> bool:
|
||||
"""Inline bytes or an attachment reference, never a plain metadata string."""
|
||||
if isinstance(value, Mapping):
|
||||
return bool(value)
|
||||
if not isinstance(value, str) or not value:
|
||||
return False
|
||||
if key in _TOOL_OUTPUT_REFERENCE_KEYS:
|
||||
return True
|
||||
return value.startswith("data:")
|
||||
|
||||
|
||||
def _decoded_base64_length(encoded: str) -> int:
|
||||
|
|
@ -715,7 +730,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
|
|||
)
|
||||
self._refuse_unscannable_image_ref(image_url=image_ref)
|
||||
|
||||
def _refuse_unscannable_leaf_parts(self, messages: "Sequence[AllMessageValues]") -> None:
|
||||
def _refuse_unscannable_leaf_parts(
|
||||
self, messages: "Sequence[AllMessageValues]", *, bridged_tool_output: bool
|
||||
) -> None:
|
||||
"""Raise the attachment refusal for any leaf part the guardrail cannot scan.
|
||||
|
||||
Runs on the unscoped message list so an attachment hiding in a message role
|
||||
|
|
@ -725,10 +742,15 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
|
|||
for message in messages:
|
||||
for leaf in _content_leaf_parts(message.get("content")):
|
||||
self._refuse_unscannable_part(part=leaf)
|
||||
self._refuse_serialized_tool_output(message=message)
|
||||
if bridged_tool_output:
|
||||
self._refuse_serialized_tool_output(message=message)
|
||||
|
||||
def _refuse_serialized_tool_output(self, message: Mapping[str, object]) -> None:
|
||||
"""Refuse an attachment hiding in a ``tool`` message's serialized output."""
|
||||
"""Refuse an attachment hiding in a ``tool`` message's serialized output.
|
||||
|
||||
A chat tool string reaches the model as text and is scanned as text; only the
|
||||
Responses to chat bridge serializes attachment parts into a string.
|
||||
"""
|
||||
if message.get("role") != "tool":
|
||||
return
|
||||
content: Final = message.get("content")
|
||||
|
|
@ -736,7 +758,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
|
|||
return
|
||||
for part in _serialized_tool_output_parts(content):
|
||||
if part.get("type") in _TOOL_OUTPUT_ATTACHMENT_TYPES and any(
|
||||
part.get(key) for key in _TOOL_OUTPUT_PAYLOAD_KEYS
|
||||
_is_attachment_payload(key, part.get(key)) for key in _TOOL_OUTPUT_PAYLOAD_KEYS
|
||||
):
|
||||
self._handle_unscannable_attachment(reason="a tool output attachment cannot be scanned")
|
||||
|
||||
|
|
@ -2970,7 +2992,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
|
|||
|
||||
# A file or unscannable image is refused even when role scoping below drops
|
||||
# its message from the scan set; the ApplyGuardrail call itself stays scoped
|
||||
self._refuse_unscannable_leaf_parts(messages=new_messages)
|
||||
self._refuse_unscannable_leaf_parts(
|
||||
messages=new_messages, bridged_tool_output=call_type in _RESPONSES_API_CALL_TYPE_VALUES
|
||||
)
|
||||
|
||||
filter_result: Final = self._prepare_guardrail_messages_for_role(messages=new_messages)
|
||||
filtered_messages: Final = filter_result.payload_messages
|
||||
|
|
|
|||
|
|
@ -1249,9 +1249,7 @@ def test_non_bedrock_policy_forwards_non_string_image_url(gateway: Gateway, tmp_
|
|||
|
||||
|
||||
@pytest.mark.covers("other.observability.guardrails.bedrock_during_call_tool_output_metadata_passes_as_text")
|
||||
def test_bedrock_during_call_tool_output_without_payload_keys_scans_as_text(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
def test_bedrock_during_call_tool_output_without_payload_keys_scans_as_text(gateway: Gateway, tmp_path: Path) -> None:
|
||||
tool_output: Final = json.dumps(
|
||||
[
|
||||
{"type": "file", "name": "README.md", "path": "README.md", "size": 1200},
|
||||
|
|
@ -1317,3 +1315,78 @@ def test_bedrock_during_call_tool_output_without_payload_keys_scans_as_text(
|
|||
assert response.status_code == 200, response.text
|
||||
eventually(lambda: policy.drain(), lambda values: len(values) == 1, seconds=10)
|
||||
assert len(upstream.drain()) == 1
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.guardrails.bedrock_during_call_tool_output_url_members_scan_as_text")
|
||||
def test_bedrock_during_call_tool_output_url_members_scan_as_text(gateway: Gateway, tmp_path: Path) -> None:
|
||||
tool_output: Final = json.dumps(
|
||||
[
|
||||
{
|
||||
"type": "file",
|
||||
"name": "README.md",
|
||||
"path": "README.md",
|
||||
"size": 1200,
|
||||
"url": "https://api.github.com/repos/o/r/contents/README.md",
|
||||
},
|
||||
{"type": "dir", "name": "src", "path": "src", "url": "https://api.github.com/repos/o/r/contents/src"},
|
||||
]
|
||||
)
|
||||
|
||||
def guardrail(request: Request) -> Reply:
|
||||
body: Final = json.loads(request.body)
|
||||
texts: Final = [item["text"]["text"] for item in body["content"] if "text" in item]
|
||||
assert tool_output in texts, body
|
||||
return Reply(body=b'{"action":"NONE","outputs":[],"assessments":[]}')
|
||||
|
||||
def chat_reply(request: Request) -> Reply:
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": "chatcmpl-wire",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "done"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
with wire_server(guardrail) as policy, wire_server(chat_reply) as upstream:
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
bedrock_params: Final = _bedrock_policy(policy.url)
|
||||
bedrock_params["mode"] = "during_call"
|
||||
config["guardrails"] = [
|
||||
{"guardrail_name": "bedrock-during-" + uuid.uuid4().hex, "litellm_params": bedrock_params}
|
||||
]
|
||||
path: Final = tmp_path / "bedrock-tool-url.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
with owned_proxy(gateway, tmp_path, {}, config=path) as candidate, candidate.scenario() as scenario:
|
||||
model: Final = scenario.model(api_base=upstream.url)
|
||||
response: Final = candidate.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "user", "content": "list files"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": "t1", "type": "function", "function": {"name": "ls", "arguments": "{}"}}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "t1", "content": tool_output},
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
eventually(lambda: policy.drain(), lambda values: len(values) == 1, seconds=10)
|
||||
assert len(upstream.drain()) == 1
|
||||
|
|
|
|||
|
|
@ -7167,7 +7167,7 @@ def test_apply_masking_to_messages_masks_tool_result_inner_list_items() -> None:
|
|||
@pytest.mark.asyncio
|
||||
async def test_during_call_hook_refuses_serialized_tool_output_attachment():
|
||||
"""A tool message's JSON-serialized output list can hide an attachment part that
|
||||
never becomes a leaf; the unscoped pass has to refuse it before any scan."""
|
||||
never becomes a leaf; on the Responses bridge the unscoped pass refuses it."""
|
||||
guardrail = BedrockGuardrail(
|
||||
guardrail_name="bedrock-tool-output-file",
|
||||
guardrailIdentifier="test-guardrail",
|
||||
|
|
@ -7177,21 +7177,20 @@ async def test_during_call_hook_refuses_serialized_tool_output_attachment():
|
|||
)
|
||||
data = {
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [
|
||||
"input": [
|
||||
{"type": "function_call", "call_id": "c1", "name": "read", "arguments": "{}"},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "c1",
|
||||
"content": json.dumps(
|
||||
[
|
||||
{
|
||||
"type": "input_file",
|
||||
"filename": "a.pdf",
|
||||
"file_data": "data:application/pdf;base64,JVBERi0=",
|
||||
}
|
||||
]
|
||||
),
|
||||
"type": "function_call_output",
|
||||
"call_id": "c1",
|
||||
"output": [
|
||||
{
|
||||
"type": "input_file",
|
||||
"filename": "a.pdf",
|
||||
"file_data": "data:application/pdf;base64,JVBERi0=",
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "user", "content": [{"type": "input_text", "text": "hi"}]},
|
||||
],
|
||||
}
|
||||
|
||||
|
|
@ -7202,13 +7201,56 @@ async def test_during_call_hook_refuses_serialized_tool_output_attachment():
|
|||
await guardrail.async_moderation_hook(
|
||||
data=data,
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
call_type=CallTypes.acompletion.value,
|
||||
call_type=CallTypes.aresponses.value,
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
mock_post.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_during_call_hook_chat_tool_output_attachment_scans_as_text():
|
||||
"""On a plain chat call a serialized tool output reaches the model as text, so the
|
||||
serialized pass must not refuse it even when a member carries attachment bytes."""
|
||||
guardrail = BedrockGuardrail(
|
||||
guardrail_name="bedrock-chat-tool-string",
|
||||
guardrailIdentifier="test-guardrail",
|
||||
guardrailVersion="DRAFT",
|
||||
event_hook=GuardrailEventHooks.during_call,
|
||||
default_on=True,
|
||||
)
|
||||
mock_credentials = MagicMock()
|
||||
mock_credentials.access_key = "test-access-key"
|
||||
mock_credentials.secret_key = "test-secret-key"
|
||||
mock_credentials.token = None
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {"action": "NONE", "assessments": []}
|
||||
tool_output = json.dumps(
|
||||
[{"type": "input_file", "filename": "a.pdf", "file_data": "data:application/pdf;base64,JVBERi0="}]
|
||||
)
|
||||
data = {
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [
|
||||
{"role": "tool", "tool_call_id": "c1", "content": tool_output},
|
||||
{"role": "user", "content": "hi"},
|
||||
],
|
||||
}
|
||||
|
||||
with (
|
||||
patch.object(guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1")),
|
||||
patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post,
|
||||
):
|
||||
mock_post.return_value = mock_response
|
||||
await guardrail.async_moderation_hook(
|
||||
data=data,
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
call_type=CallTypes.acompletion.value,
|
||||
)
|
||||
|
||||
mock_post.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_during_call_hook_refuses_serialized_tool_output_attachment_unknown_shape_falls_through():
|
||||
"""Non-attachment members of the serialized tool output are left to the normal scan."""
|
||||
|
|
@ -7372,16 +7414,17 @@ async def test_during_call_hook_refuses_tool_output_file_id_and_scoped_media():
|
|||
{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}},
|
||||
):
|
||||
data = {
|
||||
"messages": [
|
||||
{"role": "tool", "content": json.dumps([member])},
|
||||
{"role": "user", "content": "summarize"},
|
||||
"input": [
|
||||
{"type": "function_call", "call_id": "c1", "name": "read", "arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": "c1", "output": [member]},
|
||||
{"role": "user", "content": [{"type": "input_text", "text": "summarize"}]},
|
||||
]
|
||||
}
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await guardrail.async_moderation_hook(
|
||||
data=data,
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
call_type=CallTypes.acompletion.value,
|
||||
call_type=CallTypes.aresponses.value,
|
||||
)
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
|
@ -7550,3 +7593,139 @@ async def test_during_call_hook_tool_output_metadata_only_parts_scan_as_text():
|
|||
sent_body = json.loads(mock_post.call_args.kwargs["data"])
|
||||
sent_texts = [item["text"]["text"] for item in sent_body["content"] if "text" in item]
|
||||
assert tool_output in sent_texts
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"member",
|
||||
[
|
||||
{"type": "file", "name": "README.md", "path": "README.md", "size": 1200},
|
||||
{"type": "file", "name": "README.md", "url": "https://api.github.com/repos/o/r/contents/README.md"},
|
||||
{"type": "image", "title": "Golden Gate", "source": "wikimedia.org"},
|
||||
{"type": "document", "id": "kb-17", "source": "confluence", "text": "Refund policy"},
|
||||
],
|
||||
)
|
||||
async def test_during_call_hook_chat_tool_output_metadata_members_scan_as_text(member):
|
||||
"""Metadata-shaped members in a chat tool string are plain text to the scan."""
|
||||
guardrail = BedrockGuardrail(
|
||||
guardrail_name="bedrock-chat-tool-metadata",
|
||||
guardrailIdentifier="test-guardrail",
|
||||
guardrailVersion="DRAFT",
|
||||
event_hook=GuardrailEventHooks.during_call,
|
||||
default_on=True,
|
||||
)
|
||||
mock_credentials = MagicMock()
|
||||
mock_credentials.access_key = "test-access-key"
|
||||
mock_credentials.secret_key = "test-secret-key"
|
||||
mock_credentials.token = None
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {"action": "NONE", "assessments": []}
|
||||
data = {
|
||||
"messages": [
|
||||
{"role": "tool", "tool_call_id": "t1", "content": json.dumps([member])},
|
||||
{"role": "user", "content": "summarize"},
|
||||
]
|
||||
}
|
||||
|
||||
with (
|
||||
patch.object(guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1")),
|
||||
patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post,
|
||||
):
|
||||
mock_post.return_value = mock_response
|
||||
await guardrail.async_moderation_hook(
|
||||
data=data,
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
call_type=CallTypes.acompletion.value,
|
||||
)
|
||||
|
||||
mock_post.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"member",
|
||||
[
|
||||
{"type": "document", "id": "kb-17", "source": "confluence"},
|
||||
{"type": "file", "name": "README.md", "url": "https://api.github.com/repos/o/r/contents/README.md"},
|
||||
],
|
||||
)
|
||||
async def test_during_call_hook_responses_tool_output_metadata_members_scan_as_text(member):
|
||||
"""On the Responses bridge only structural payloads refuse; plain metadata strings
|
||||
in a serialized tool output are still scanned as text."""
|
||||
guardrail = BedrockGuardrail(
|
||||
guardrail_name="bedrock-resp-tool-metadata",
|
||||
guardrailIdentifier="test-guardrail",
|
||||
guardrailVersion="DRAFT",
|
||||
event_hook=GuardrailEventHooks.during_call,
|
||||
default_on=True,
|
||||
)
|
||||
mock_credentials = MagicMock()
|
||||
mock_credentials.access_key = "test-access-key"
|
||||
mock_credentials.secret_key = "test-secret-key"
|
||||
mock_credentials.token = None
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {"action": "NONE", "assessments": []}
|
||||
data = {
|
||||
"input": [
|
||||
{"type": "function_call", "call_id": "c1", "name": "read", "arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": "c1", "output": [member]},
|
||||
{"role": "user", "content": [{"type": "input_text", "text": "summarize"}]},
|
||||
]
|
||||
}
|
||||
|
||||
with (
|
||||
patch.object(guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1")),
|
||||
patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post,
|
||||
):
|
||||
mock_post.return_value = mock_response
|
||||
await guardrail.async_moderation_hook(
|
||||
data=data,
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
call_type=CallTypes.aresponses.value,
|
||||
)
|
||||
|
||||
mock_post.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"member",
|
||||
[
|
||||
{"type": "document", "id": "kb-17", "source": {"type": "base64", "data": "x"}},
|
||||
{"type": "input_file", "filename": "a.pdf", "file_data": "data:application/pdf;base64,AA=="},
|
||||
{"type": "input_image", "file_id": "f1"},
|
||||
{"type": "file", "name": "clip", "url": "data:video/mp4;base64,AA=="},
|
||||
],
|
||||
)
|
||||
async def test_during_call_hook_responses_tool_output_payload_members_refused(member):
|
||||
"""Structural attachment payloads in a serialized tool output refuse on the
|
||||
Responses bridge: a mapping source, inline file bytes, or a file reference."""
|
||||
guardrail = BedrockGuardrail(
|
||||
guardrail_name="bedrock-resp-tool-payloads",
|
||||
guardrailIdentifier="test-guardrail",
|
||||
guardrailVersion="DRAFT",
|
||||
event_hook=GuardrailEventHooks.during_call,
|
||||
default_on=True,
|
||||
)
|
||||
data = {
|
||||
"input": [
|
||||
{"type": "function_call", "call_id": "c1", "name": "read", "arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": "c1", "output": [member]},
|
||||
{"role": "user", "content": [{"type": "input_text", "text": "summarize"}]},
|
||||
]
|
||||
}
|
||||
|
||||
with (
|
||||
patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post,
|
||||
pytest.raises(HTTPException) as exc_info,
|
||||
):
|
||||
await guardrail.async_moderation_hook(
|
||||
data=data,
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
call_type=CallTypes.aresponses.value,
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
mock_post.assert_not_called()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue