From 681906b6eb96b3b30da249a0737426d47bcc3672 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 9 Oct 2026 17:24:36 -0700 Subject: [PATCH 1/2] test(e2e): cover images returned in Bedrock tool results on /v1/messages Sends a Claude Code shaped Read round trip whose tool_result is an image, then a follow-up turn that keeps the image in history, to Bedrock models over the proxy's /v1/messages: the GPT family, Qwen3-VL, Kimi K3 and Mistral Large 3, with Claude and Nova as controls. On main every non-control case fails with Bedrock's 400 for the image field (#45576) --- .../test_messages_bedrock_e2e.py | 120 +++++++++++++++++- 1 file changed, 119 insertions(+), 1 deletion(-) diff --git a/tests/e2e/llm_translation/test_messages_bedrock_e2e.py b/tests/e2e/llm_translation/test_messages_bedrock_e2e.py index 31daa9bba32..d5ce0b73c88 100644 --- a/tests/e2e/llm_translation/test_messages_bedrock_e2e.py +++ b/tests/e2e/llm_translation/test_messages_bedrock_e2e.py @@ -3,7 +3,18 @@ from __future__ import annotations from typing import Final import pytest -from anthropic.types import RawContentBlockDeltaEvent, RawMessageDeltaEvent, TextBlock, TextDelta, ToolParam +from _pytest.mark.structures import ParameterSet +from anthropic import Anthropic +from anthropic.types import ( + Message, + MessageParam, + RawContentBlockDeltaEvent, + RawMessageDeltaEvent, + TextBlock, + TextDelta, + ToolParam, + ToolUseBlock, +) from e2e_config import unique_marker from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager @@ -17,6 +28,59 @@ pytestmark = pytest.mark.e2e GPT_6_1_SOL_BACKEND: Final = "bedrock/global.openai.gpt-6.1-sol" CONVERSE_CLAUDE_BACKEND: Final = "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0" NOVA_BACKEND: Final = "bedrock/us.amazon.nova-2-lite-v1:0" +TOOL_RESULT_IMAGE_CLAUDE_BACKEND: Final = "bedrock/converse/us.anthropic.claude-haiku-5-5" +TOOL_RESULT_IMAGE_BACKENDS: Final = ( + GPT_6_1_SOL_BACKEND, + "bedrock/global.openai.gpt-6-sol", + "bedrock/global.openai.gpt-5.6-sol", + "bedrock/global.openai.gpt-5.5", + "bedrock/global.openai.gpt-5.4", + "bedrock/qwen.qwen3-vl-235b-a22b", + "bedrock/global.moonshotai.kimi-k3", + "bedrock/mistral.mistral-large-3-675b-instruct", + TOOL_RESULT_IMAGE_CLAUDE_BACKEND, + NOVA_BACKEND, +) + +RED_PIXEL_PNG_B64: Final = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) +READ_TOOL: Final[ToolParam] = { + "name": "Read", + "description": "Read a file from the local filesystem.", + "input_schema": { + "type": "object", + "properties": {"file_path": {"type": "string"}}, + "required": ["file_path"], + }, +} +READ_TOOL_USE_ID: Final = "toolu_01A09q90qw90lq917835lq9" +READ_IMAGE_CONVERSATION: Final[tuple[MessageParam, ...]] = ( + {"role": "user", "content": "What color is the image in pixel.png? Answer in one word."}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": READ_TOOL_USE_ID, "name": "Read", "input": {"file_path": "pixel.png"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": READ_TOOL_USE_ID, + "content": [ + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": RED_PIXEL_PNG_B64}, + } + ], + } + ], + }, +) +FOLLOW_UP_TURNS: Final[tuple[MessageParam, ...]] = ( + {"role": "assistant", "content": "The file holds a single pixel."}, + {"role": "user", "content": "Thanks. Reply with the word done."}, +) TOOL_SEARCH: Final[ToolParam] = { "name": "ToolSearch", @@ -62,6 +126,39 @@ def _register(proxy: ProxyClient, resources: ResourceManager, backend: str) -> s return model +def _tool_result_image_params() -> list[ParameterSet]: + return [ + pytest.param( + backend, + id=backend.rsplit("/", 1)[-1], + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=(backend,), + capabilities=(Capability.FUNCTION_CALLING, Capability.VISION), + mode=Mode.NONSTREAM, + ) + ), + ) + for backend in TOOL_RESULT_IMAGE_BACKENDS + ] + + +def _reply(client: Anthropic, model: str, messages: tuple[MessageParam, ...]) -> Message: + return client.messages.create( + model=model, max_tokens=2048, tools=[READ_TOOL], messages=messages, extra_body=NO_PROXY_CACHE + ) + + +def _answered(message: Message) -> bool: + return any( + isinstance(block, ToolUseBlock) or (isinstance(block, TextBlock) and block.text.strip()) + for block in message.content + ) + + class TestBedrockMessages: @meta( Subject( @@ -175,3 +272,24 @@ class TestBedrockMessages: assert len(deltas) == 1, f"expected exactly one message_delta: {types}" assert deltas[0].delta.stop_reason is not None, f"message_delta carried no stop_reason: {deltas[0]!r}" assert deltas[0].usage.output_tokens > 0, f"message_delta reported no output tokens: {deltas[0]!r}" + + +class TestBedrockToolResultImages: + @pytest.mark.parametrize("backend", _tool_result_image_params()) + def test_image_returned_by_a_tool_gets_a_reply( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients, backend: str + ) -> None: + """Claude Code's Read tool returns an image file as an image inside tool_result. + Converse documents that nesting for Claude and Nova only and rejects it for other + models (#45576), so the proxy has to reshape it for them. The image stays in + history on every later turn, so the follow-up must be accepted too or the session + is stuck. Some models answer by calling Read again, which is still a well-formed + next turn.""" + model: Final = _register(proxy, resources, backend) + client: Final = sdk.anthropic(resources.key()) + + tool_result_reply: Final = _reply(client, model, READ_IMAGE_CONVERSATION) + assert _answered(tool_result_reply), f"{backend} sent back no text or tool call: {tool_result_reply!r}" + + follow_up_reply: Final = _reply(client, model, (*READ_IMAGE_CONVERSATION, *FOLLOW_UP_TURNS)) + assert _answered(follow_up_reply), f"{backend} sent back no text or tool call: {follow_up_reply!r}" From 729f48cee5a35728692eb750593c9e21ad75510a Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 9 Oct 2026 17:20:43 -0700 Subject: [PATCH 2/2] fix(bedrock): keep tool-result images nested only for Claude and Nova on Converse AWS documents images inside toolResult.content for Claude and Nova only. Every other model now gets the image beside the tool result by default, so Qwen3-VL, Kimi and Mistral no longer need a price map flag. The flag still overrides in either direction. --- litellm/llms/bedrock/common_utils.py | 20 +++--- ...odel_prices_and_context_window_backup.json | 20 ------ model_prices_and_context_window.json | 20 ------ .../bedrock/chat/test_tool_result_images.py | 63 +++++++++++++++---- 4 files changed, 65 insertions(+), 58 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 151f4858fdb..edc01da3801 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -47,6 +47,7 @@ _OPENAI_FAMILY_MODEL_RE: Final = re.compile(r"(^|[./])openai\.") _OPENAI_GPT_VERSION_RE: Final = re.compile(r"(^|[./])openai\.gpt-(\d{1,3})(?!\d)(?:\.(\d{1,3})(?!\d))?") _BEDROCK_RUNTIME_CHAT_COMPLETIONS_DEFAULT_SINCE: Final = (5, 6) _XAI_GROK_MODEL_RE: Final = re.compile(r"(^|[./])xai\.grok-") +_CONVERSE_NESTED_TOOL_RESULT_IMAGE_RE: Final = re.compile(r"(^|[./])(anthropic\.claude|amazon\.nova)-") _BEDROCK_RUNTIME_CHAT_COMPLETIONS_ENDPOINT: Final = "/v1/chat/completions" BedrockRoute = Literal[ "converse", @@ -926,14 +927,19 @@ def bedrock_runtime_chat_completions_enforces_response_format(model: str) -> boo def bedrock_converse_supports_tool_result_images(model: str) -> bool: """Whether Converse accepts an image nested in ``toolResult.content``. - Missing means yes, which is what Claude accepts. A price-map row sets - ``supports_bedrock_converse_tool_result_images`` to false when Bedrock - rejects that image and it has to sit beside the tool result instead. + AWS documents that for Claude and Nova only (ToolResultContentBlock API reference, checked 2026-10-09), + so every other model gets the image beside the tool result. The price-map flag + ``supports_bedrock_converse_tool_result_images`` overrides that in either direction. """ - entries: Final = tuple(entry for entry in _bedrock_price_map_entries(model) if entry is not None) - if not entries: - return True - return all(entry.get("supports_bedrock_converse_tool_result_images") is not False for entry in entries) + flags: Final = tuple( + entry.get("supports_bedrock_converse_tool_result_images") + for entry in _bedrock_price_map_entries(model) + if entry is not None + ) + explicit: Final = tuple(flag for flag in flags if isinstance(flag, bool)) + if explicit: + return all(explicit) + return _CONVERSE_NESTED_TOOL_RESULT_IMAGE_RE.search(model) is not None def bedrock_model_is_openai_gpt(model: str) -> bool: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6d49e7cd7fd..90f6bb11c4a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -56693,7 +56693,6 @@ "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56730,7 +56729,6 @@ "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56767,7 +56765,6 @@ "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56804,7 +56801,6 @@ "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56841,7 +56837,6 @@ "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56875,7 +56870,6 @@ "output_cost_per_token": 1.65e-05, "output_cost_per_token_above_272k_tokens": 2.475e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56907,7 +56901,6 @@ "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56939,7 +56932,6 @@ "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56971,7 +56963,6 @@ "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57006,7 +56997,6 @@ "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57174,7 +57164,6 @@ "output_cost_per_token": 5.5e-05, "output_cost_per_token_above_272k_tokens": 8.25e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57215,7 +57204,6 @@ "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57254,7 +57242,6 @@ "output_cost_per_token": 5.5e-07, "output_cost_per_token_above_272k_tokens": 8.25e-07, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57299,7 +57286,6 @@ "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57339,7 +57325,6 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57374,7 +57359,6 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57412,7 +57396,6 @@ "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57447,7 +57430,6 @@ "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -70523,7 +70505,6 @@ "input_cost_per_token_above_272k_tokens_ultrafast": 2.4e-05, "input_cost_per_token_ultrafast": 1.2e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1000000, "max_output_tokens": 131072, "max_tokens": 131072, @@ -70617,7 +70598,6 @@ "input_cost_per_token_above_272k_tokens_ultrafast": 2.64e-05, "input_cost_per_token_ultrafast": 1.32e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1000000, "max_output_tokens": 131072, "max_tokens": 131072, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6d49e7cd7fd..90f6bb11c4a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -56693,7 +56693,6 @@ "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56730,7 +56729,6 @@ "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56767,7 +56765,6 @@ "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56804,7 +56801,6 @@ "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56841,7 +56837,6 @@ "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56875,7 +56870,6 @@ "output_cost_per_token": 1.65e-05, "output_cost_per_token_above_272k_tokens": 2.475e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56907,7 +56901,6 @@ "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56939,7 +56932,6 @@ "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -56971,7 +56963,6 @@ "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57006,7 +56997,6 @@ "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57174,7 +57164,6 @@ "output_cost_per_token": 5.5e-05, "output_cost_per_token_above_272k_tokens": 8.25e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57215,7 +57204,6 @@ "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57254,7 +57242,6 @@ "output_cost_per_token": 5.5e-07, "output_cost_per_token_above_272k_tokens": 8.25e-07, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57299,7 +57286,6 @@ "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57339,7 +57325,6 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57374,7 +57359,6 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57412,7 +57396,6 @@ "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -57447,7 +57430,6 @@ "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, @@ -70523,7 +70505,6 @@ "input_cost_per_token_above_272k_tokens_ultrafast": 2.4e-05, "input_cost_per_token_ultrafast": 1.2e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1000000, "max_output_tokens": 131072, "max_tokens": 131072, @@ -70617,7 +70598,6 @@ "input_cost_per_token_above_272k_tokens_ultrafast": 2.64e-05, "input_cost_per_token_ultrafast": 1.32e-05, "litellm_provider": "bedrock_converse", - "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1000000, "max_output_tokens": 131072, "max_tokens": 131072, diff --git a/tests/unit/llms/bedrock/chat/test_tool_result_images.py b/tests/unit/llms/bedrock/chat/test_tool_result_images.py index 4e9800c83f6..b1bb715bd2d 100644 --- a/tests/unit/llms/bedrock/chat/test_tool_result_images.py +++ b/tests/unit/llms/bedrock/chat/test_tool_result_images.py @@ -1,5 +1,8 @@ from typing import Final +import pytest + +import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_PLACEHOLDER from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_images from litellm.types.llms.bedrock import ( @@ -14,26 +17,64 @@ _PNG_BYTES: Final = b"\x89PNG\r\n\x1a\n" _TOOL_IMAGE: Final = ImageBlock(format="png", source={"bytes": _PNG_BYTES}) -def test_place_tool_result_images_keeps_nested_images_for_claude(): - message: Final = MessageBlock( +def _nested_image_message() -> MessageBlock: + return MessageBlock( role="user", content=[ ContentBlock( toolResult=ToolResultBlock( toolUseId="tooluse_nested", - content=[ - ToolResultContentBlock(text="nested", image=_TOOL_IMAGE), - ], + content=[ToolResultContentBlock(text="nested"), ToolResultContentBlock(image=_TOOL_IMAGE)], ), ), ], ) - placed: Final = place_tool_result_images( - [message], - "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - ) - assert placed == (message,) - assert placed[0] is message + + +def _keeps_image_nested(placed: tuple[MessageBlock, ...], message: MessageBlock) -> bool: + if placed[0] is message: + return True + tool_result: Final = placed[0]["content"][0]["toolResult"] + assert isinstance(tool_result, dict) + assert tool_result["content"] == [{"text": "nested"}] + assert placed[0]["content"][1] == {"image": _TOOL_IMAGE} + return False + + +@pytest.mark.parametrize( + ("model", "nested"), + [ + ("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", True), + ("bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0", True), + ("bedrock/us.amazon.nova-pro-v1:0", True), + ("bedrock/converse/us.amazon.nova-2-lite-v1:0", True), + ("bedrock/global.openai.gpt-6.1-sol", False), + ("bedrock/qwen.qwen3-vl-235b-a22b", False), + ("bedrock/global.moonshotai.kimi-k3", False), + ("bedrock/mistral.mistral-large-3-675b-instruct", False), + ("bedrock/converse/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", False), + ], +) +def test_place_tool_result_images_nests_only_for_claude_and_nova(model: str, nested: bool): + message: Final = _nested_image_message() + assert _keeps_image_nested(place_tool_result_images([message], model), message) is nested + + +@pytest.mark.parametrize( + ("price_map_key", "flag", "nested"), + [ + ("qwen.qwen3-vl-235b-a22b", True, True), + ("anthropic.claude-haiku-4-5-20251001-v1:0", False, False), + ], +) +def test_price_map_flag_overrides_the_model_family( + monkeypatch: pytest.MonkeyPatch, price_map_key: str, flag: bool, nested: bool +): + entry: Final = {**litellm.model_cost.get(price_map_key, {}), "supports_bedrock_converse_tool_result_images": flag} + monkeypatch.setitem(litellm.model_cost, price_map_key, entry) + message: Final = _nested_image_message() + placed: Final = place_tool_result_images([message], f"bedrock/{price_map_key}") + assert _keeps_image_nested(placed, message) is nested def test_place_tool_result_images_leaves_text_only_tool_results_on_gpt():