From 1f80f93750a4c0c4c8717d9d34a69b6d93208caa Mon Sep 17 00:00:00 2001 From: samtsai15 <6171228+samtsai15@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:57:27 +0800 Subject: [PATCH 1/4] fix(guardrails): carry Anthropic url and file image sources through to guardrails _image_sources returned source["data"] only. An Anthropic image block has three shapes (types/llms/anthropic.py:259) and only the base64 one carries "data", so {"type": "url", "url": ...} yielded nothing and the image never reached any guardrail at all. This is not Bedrock-specific. Five guardrails consume GenericGuardrailAPIInputs["images"] (vigil_guard, custom_code, deepkeep, straiker, generic_guardrail_api) and every one of them was blind to url sources on /v1/messages. base64 now returns a data URI rather than the bare payload. A consumer otherwise has no way to recover media_type, and an API like Bedrock's ApplyGuardrail needs the format to build its request. The file shape stays unresolvable here: the bytes live behind the Files API and this extractor has no client to fetch them. Documented rather than silently dropped, so a consumer treating a missing entry as "no image to scan" is a known gap and not a surprise. Co-Authored-By: Claude Opus 5 (1M context) --- .../chat/guardrail_translation/handler.py | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index eb6dbd3e5b3..4b763d11652 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -859,12 +859,40 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _image_sources(block: Mapping[str, object]) -> tuple[str, ...]: + """Normalize an Anthropic image block into strings a guardrail can read. + + `source` is one of three shapes (types/llms/anthropic.py:259): + + {"type": "base64", "media_type": "image/png", "data": ""} + {"type": "url", "url": "https://..."} + {"type": "file", "file_id": "..."} + + base64 is returned as a data URI rather than the bare payload: consumers of + ``GenericGuardrailAPIInputs["images"]`` otherwise have no way to know the + format, and an API like Bedrock's ApplyGuardrail requires it. url is passed + through so the consumer can fetch it under its own SSRF policy. + + file is not resolvable here (the bytes live behind the Files API), so it + yields nothing. That is a silent gap for any consumer that treats a missing + entry as "no image to scan"; scanning a file_id needs a fetch this extractor + has no client for. + """ source: Final = block.get("source") if not isinstance(source, Mapping): return () - # Could be base64 or url + + source_type: Final = source.get("type") + if source_type == "url": + url: Final = source.get("url") + return (url,) if isinstance(url, str) and url else () + data: Final = source.get("data") - return (data,) if data else () + if not isinstance(data, str) or not data: + return () + media_type: Final = source.get("media_type") + if isinstance(media_type, str) and media_type: + return (f"data:{media_type};base64,{data}",) + return (data,) async def _apply_guardrail_responses_to_input( self, From 0e7562dbc692812dfb5cb02ad72434d54a25b715 Mon Sep 17 00:00:00 2001 From: samtsai15 <6171228+samtsai15@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:22:51 +0800 Subject: [PATCH 2/4] test(guardrails): cover every Anthropic image source shape in the extractor's own suite _image_sources had no test asserting what it extracts. The existing image tests live on the Bedrock side and all use base64 without a media_type, which is the one path the fix left unchanged, so both behaviors it does change went unverified: the url shape reaching the guardrail at all, and base64 arriving as a data URI. Against the pre-fix extractor the url case sees [] and the media_type case sees ['AAAA'] instead of ['data:image/png;base64,AAAA']. The remaining three assert behavior the fix deliberately preserves -- bare base64 passed through, a file source yielding nothing, a malformed source dropped rather than handed on for a consumer to choke on. Each message carries a text block because a message with no text never reaches the guardrail, which would make every source shape look equally dropped. Co-Authored-By: Claude Opus 5 (1M context) --- .../test_anthropic_guardrail_handler.py | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index af3ccd65b11..2b606b9639a 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -1490,6 +1490,92 @@ class MockCanaryMaskingGuardrail(CustomGuardrail): return inputs +class TestAnthropicMessagesImageSources: + """An Anthropic image block has three source shapes (types/llms/anthropic.py:259). + + Only the base64 one carries "data", so reading that key alone drops url images + entirely -- for every guardrail consuming GenericGuardrailAPIInputs["images"], + not just Bedrock. + """ + + def _data(self, messages): + return {"model": "claude-sonnet-4-5", "messages": messages} + + async def _images_seen(self, content) -> list[str]: + handler = AnthropicMessagesHandler() + + class ImageRecordingGuardrail(MockCanaryMaskingGuardrail): + def __init__(self): + super().__init__() + self.seen_images: list[str] = [] # mutable-ok: accumulator for the assertion + + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + self.seen_images.extend(inputs.get("images") or []) + return await super().apply_guardrail(inputs, request_data, input_type, logging_obj) + + guardrail = ImageRecordingGuardrail() + # The text block is what gets the guardrail invoked at all: a message with + # no text gives the handler nothing to scan, so it never reaches the + # guardrail and every source shape would look equally "dropped". + await handler.process_input_messages( + data=self._data([{"role": "user", "content": [{"type": "text", "text": "describe it"}, *content]}]), + guardrail_to_apply=guardrail, + ) + return guardrail.seen_images + + @pytest.mark.asyncio + async def test_url_source_reaches_the_guardrail(self): + """A url source has no "data" key, so it used to yield nothing at all.""" + seen = await self._images_seen( + [{"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}}] + ) + + assert seen == ["https://example.com/a.png"] + + @pytest.mark.asyncio + async def test_base64_source_carries_its_media_type(self): + """Bare base64 leaves the consumer no way to recover the format. + + An API like Bedrock's ApplyGuardrail needs it to build the request, so the + media_type travels with the payload as a data URI. + """ + seen = await self._images_seen( + [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}}] + ) + + assert seen == ["data:image/png;base64,AAAA"] + + @pytest.mark.asyncio + async def test_base64_source_without_a_media_type_is_passed_through(self): + """There is no format to attach, so the payload goes through unchanged.""" + seen = await self._images_seen([{"type": "image", "source": {"type": "base64", "data": "AAAA"}}]) + + assert seen == ["AAAA"] + + @pytest.mark.asyncio + async def test_file_source_yields_nothing(self): + """The bytes live behind the Files API and this extractor has no client. + + Documented as a known gap rather than silently handed on as a file_id string, + which a consumer would try to decode as an image. + """ + seen = await self._images_seen([{"type": "image", "source": {"type": "file", "file_id": "file_abc"}}]) + + assert seen == [] + + @pytest.mark.asyncio + async def test_a_malformed_source_is_dropped_rather_than_passed_on(self): + seen = await self._images_seen( + [ + {"type": "image", "source": {"type": "base64"}}, + {"type": "image", "source": {"type": "url"}}, + {"type": "image", "source": {"type": "base64", "data": ""}}, + ] + ) + + assert seen == [] + + class TestAnthropicMessagesToolResultScanning: """LIT-5251: tool_result blocks carry whatever a client's local tool fetched, so they are the request-path payload an indirect prompt injection actually arrives in. From bb51c121cf943b384fd00b0a2b106deca4dd5d53 Mon Sep 17 00:00:00 2001 From: "feng.tsai" Date: Mon, 31 Aug 2026 12:21:24 +0800 Subject: [PATCH 3/4] docs: reference the source union by type instead of a line number The line number went stale when the base moved. --- litellm/llms/anthropic/chat/guardrail_translation/handler.py | 2 +- .../guardrail_translation/test_anthropic_guardrail_handler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 4b763d11652..5aa0dc94b89 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -861,7 +861,7 @@ class AnthropicMessagesHandler(BaseTranslation): def _image_sources(block: Mapping[str, object]) -> tuple[str, ...]: """Normalize an Anthropic image block into strings a guardrail can read. - `source` is one of three shapes (types/llms/anthropic.py:259): + `source` is one of three shapes (`AnthropicMessagesImageParam.source`): {"type": "base64", "media_type": "image/png", "data": ""} {"type": "url", "url": "https://..."} diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 2b606b9639a..0fe7730e91e 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -1491,7 +1491,7 @@ class MockCanaryMaskingGuardrail(CustomGuardrail): class TestAnthropicMessagesImageSources: - """An Anthropic image block has three source shapes (types/llms/anthropic.py:259). + """An Anthropic image block has three source shapes (`AnthropicMessagesImageParam.source`). Only the base64 one carries "data", so reading that key alone drops url images entirely -- for every guardrail consuming GenericGuardrailAPIInputs["images"], From dc034a086a17d349e005fc2b547ed8a6b9c2654e Mon Sep 17 00:00:00 2001 From: "feng.tsai" Date: Mon, 31 Aug 2026 13:26:02 +0800 Subject: [PATCH 4/4] docs: trim the _image_sources docstring It restated the source union that the type definition already carries. --- .../chat/guardrail_translation/handler.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 5aa0dc94b89..9395cc3d33e 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -861,21 +861,9 @@ class AnthropicMessagesHandler(BaseTranslation): def _image_sources(block: Mapping[str, object]) -> tuple[str, ...]: """Normalize an Anthropic image block into strings a guardrail can read. - `source` is one of three shapes (`AnthropicMessagesImageParam.source`): - - {"type": "base64", "media_type": "image/png", "data": ""} - {"type": "url", "url": "https://..."} - {"type": "file", "file_id": "..."} - - base64 is returned as a data URI rather than the bare payload: consumers of - ``GenericGuardrailAPIInputs["images"]`` otherwise have no way to know the - format, and an API like Bedrock's ApplyGuardrail requires it. url is passed - through so the consumer can fetch it under its own SSRF policy. - - file is not resolvable here (the bytes live behind the Files API), so it - yields nothing. That is a silent gap for any consumer that treats a missing - entry as "no image to scan"; scanning a file_id needs a fetch this extractor - has no client for. + base64 becomes a data URI so the format travels with the payload, which is what + the OpenAI path already puts in this field. A file source yields nothing: those + bytes live behind the Files API and this extractor has no client to fetch them. """ source: Final = block.get("source") if not isinstance(source, Mapping):