From 84ff9f84a92bc43a76e35c13f5d07c074bf14543 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:33:09 +0000 Subject: [PATCH] fix(anthropic): send detail on input_image parts bridged from /v1/messages to Responses (#46720) * fix(anthropic): send detail on input_image parts bridged from /v1/messages to Responses * fix(ui): send detail on playground Responses image parts and code snippets * test(messages): cover image detail on the Responses bridge wire --- .../responses_adapters/transformation.py | 10 +-- ...t_anthropic_messages_openai_bridge_wire.py | 69 +++++++++++++++++++ ...ses_bridge_prompt_cache_breakpoint_wire.py | 1 + .../test_responses_adapters_handler.py | 1 + .../test_responses_adapters_transformation.py | 47 ++++++++++--- .../chat_ui/ResponsesImageUtils.test.tsx | 15 ++++ .../chat_ui/ResponsesImageUtils.tsx | 1 + .../src/components/chat_ui/CodeSnippets.tsx | 13 ++++ .../src/components/chat_ui/types.ts | 1 + 9 files changed, 146 insertions(+), 12 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ResponsesImageUtils.test.tsx diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py index 73978094685..156c8d89d81 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py @@ -96,6 +96,10 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return source.get("url") return None + @staticmethod + def _input_image_part(url: str) -> dict[str, object]: # mutable-ok: API message payload + return {"type": "input_image", "image_url": url, "detail": "auto"} + @staticmethod def _translate_anthropic_document_block_to_file_part( block: Mapping[str, object], @@ -149,9 +153,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if block.get("type") == "image" ) return tuple( - with_prompt_cache_breakpoint({"type": "input_image", "image_url": url}, breakpoint) - for url, breakpoint in sources - if url + with_prompt_cache_breakpoint(cls._input_image_part(url), breakpoint) for url, breakpoint in sources if url ) @classmethod @@ -357,7 +359,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if url: user_parts.append( with_prompt_cache_breakpoint( - {"type": "input_image", "image_url": url}, block.get("prompt_cache_breakpoint") + self._input_image_part(url), block.get("prompt_cache_breakpoint") ) ) elif btype == "document": diff --git a/tests/integration/messages_endpoint/responses_bridge/test_anthropic_messages_openai_bridge_wire.py b/tests/integration/messages_endpoint/responses_bridge/test_anthropic_messages_openai_bridge_wire.py index 72eba1d89a5..cd729978855 100644 --- a/tests/integration/messages_endpoint/responses_bridge/test_anthropic_messages_openai_bridge_wire.py +++ b/tests/integration/messages_endpoint/responses_bridge/test_anthropic_messages_openai_bridge_wire.py @@ -1,3 +1,4 @@ +import itertools import json import uuid from typing import Final @@ -11,6 +12,12 @@ _BACKEND: Final = "gpt-5.4-mini" _API_KEY: Final = "synthetic-openai-key" _CORRECTION: Final = "Stop refactoring the parser and only fix the failing test instead." _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_JSON_OBJECTS: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]) +_PIXEL: Final = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +_PASTED_URL: Final = "https://example.invalid/pasted.png" +_MISSING_DETAIL: Final = json.dumps( + {"error": {"message": "1 validation error: input.0.content.1.detail Field required", "type": "BadRequestError"}} +).encode() def _responses_reply(identity: str, content: str) -> bytes: @@ -35,6 +42,68 @@ def _responses_reply(identity: str, content: str) -> bytes: ).encode() +def _image_parts(body: dict[str, JsonValue]) -> tuple[dict[str, JsonValue], ...]: + contents: Final = ( + _JSON_OBJECTS.validate_python(item.get("content", ())) for item in _JSON_OBJECTS.validate_python(body["input"]) + ) + return tuple(part for part in itertools.chain.from_iterable(contents) if part.get("type") == "input_image") + + +@pytest.mark.parametrize( + ("source", "image_url"), + ( + ({"type": "base64", "media_type": "image/png", "data": _PIXEL}, f"data:image/png;base64,{_PIXEL}"), + ({"type": "url", "url": _PASTED_URL}, _PASTED_URL), + ), + ids=("base64", "url"), +) +def test_pasted_image_reaches_a_responses_server_that_requires_image_detail( + gateway: Gateway, source: dict[str, JsonValue], image_url: str +) -> None: + identity: Final = f"openai-image-detail-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if any("detail" not in part for part in _image_parts(_JSON_OBJECT.validate_json(request.body))): + return Reply(status=400, body=_MISSING_DETAIL) + return Reply(body=_responses_reply(identity, "A red circle.")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"openai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 64, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image", "source": source}, + ], + } + ], + }, + ) + requests: Final = wire.drain() + assert len(requests) == 1, response.text + assert _JSON_OBJECT.validate_json(requests[0].body)["input"] == [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "What is in this image?"}, + {"type": "input_image", "image_url": image_url, "detail": "auto"}, + ], + } + ], requests[0].body + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content)["content"] == [{"type": "text", "text": "A red circle."}], ( + response.text + ) + + @pytest.mark.covers("providers.anthropic_messages_openai_bridge.midturn_system_correction_reaches_the_wire") def test_midturn_system_correction_is_forwarded_to_openai_responses(gateway: Gateway) -> None: identity: Final = f"openai-midturn-system-{uuid.uuid4().hex}" diff --git a/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py b/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py index 40a64af05ed..12a7bfe68e9 100644 --- a/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py +++ b/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py @@ -516,6 +516,7 @@ def test_anthropic_sdk_cache_control_on_a_tool_result_with_a_screenshot_marks_th assert screenshot == { "type": "input_image", "image_url": "data:image/png;base64,Zm9v", + "detail": "auto", "prompt_cache_breakpoint": pcb.EXPLICIT, }, "OpenAI caches only the prefix up to the breakpoint, so the screenshot must sit inside it" assert request.body.count(b"prompt_cache_breakpoint") == 1, request.body diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index 14a1b32f8b7..61efc65f8c2 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -465,6 +465,7 @@ def test_client_cache_control_on_a_screenshot_inside_a_tool_result_reaches_gpt_5 { "type": "input_image", "image_url": "data:image/png;base64,c2hvdA==", + "detail": "auto", "prompt_cache_breakpoint": {"mode": "explicit"}, }, ], diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py index 5d0e3fa7236..829000c0c5f 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -9,7 +9,8 @@ from typing import Any, Dict, Final, List from unittest.mock import MagicMock import pytest - +from openai.types.responses import ResponseInputImageParam +from pydantic import TypeAdapter from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, @@ -400,7 +401,9 @@ class TestTranslateMessagesToResponsesInput: ] result = _translate_messages(messages) assert len(result) == 1 - assert result[0]["content"] == [{"type": "input_image", "image_url": "data:image/png;base64,abc123"}] + assert result[0]["content"] == [ + {"type": "input_image", "image_url": "data:image/png;base64,abc123", "detail": "auto"} + ] def test_user_url_image(self): """User message with URL image source becomes input_image with the URL.""" @@ -416,7 +419,9 @@ class TestTranslateMessagesToResponsesInput: } ] result = _translate_messages(messages) - assert result[0]["content"] == [{"type": "input_image", "image_url": "https://example.com/img.jpg"}] + assert result[0]["content"] == [ + {"type": "input_image", "image_url": "https://example.com/img.jpg", "detail": "auto"} + ] def test_user_base64_image_empty_data_skipped(self): """Base64 image with empty data is skipped (no URL can be formed).""" @@ -745,6 +750,7 @@ class TestTranslateMessagesToResponsesInput: assert result[0]["content"][1] == { "type": "input_image", "image_url": "https://example.com/cat.jpg", + "detail": "auto", } def test_unknown_image_source_type_skipped(self): @@ -1753,6 +1759,24 @@ class TestToolResultImages: and any(part.get("type") == "input_image" for part in item.get("content", [])) ) + def test_every_bridged_image_is_a_valid_responses_input_image(self): + image: Final = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": self.B64_DATA}} + messages: Final = [ + {"role": "user", "content": [{"type": "text", "text": "what is this?"}, image]}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "read", "input": {}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": [image]}], + }, + ] + parts: Final = self._input_images(_ADAPTER.translate_messages_to_responses_input(messages)) + assert len(parts) == 2 + for part in parts: + TypeAdapter(ResponseInputImageParam).validate_python(part) + def test_base64_image_survives(self): items = self._translate( [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": self.B64_DATA}}] @@ -1812,7 +1836,7 @@ class TestToolResultImages: assert self._image_message(items)["content"] == [ {"type": "input_text", "text": TOOL_RESULT_IMAGE_BOUNDARY}, - {"type": "input_image", "image_url": self.DATA_URI}, + {"type": "input_image", "image_url": self.DATA_URI, "detail": "auto"}, ] def test_sibling_user_blocks_stay_out_of_boundary_message(self): @@ -1825,7 +1849,7 @@ class TestToolResultImages: assert self._image_message(items)["content"] == [ {"type": "input_text", "text": TOOL_RESULT_IMAGE_BOUNDARY}, - {"type": "input_image", "image_url": self.DATA_URI}, + {"type": "input_image", "image_url": self.DATA_URI, "detail": "auto"}, ] assert any( part == {"type": "input_text", "text": "what changed?"} @@ -1976,7 +2000,7 @@ class TestToolResultDocuments: ) assert image_message["content"] == [ {"type": "input_text", "text": TOOL_RESULT_IMAGE_BOUNDARY}, - {"type": "input_image", "image_url": f"data:image/png;base64,{self.PNG_B64}"}, + {"type": "input_image", "image_url": f"data:image/png;base64,{self.PNG_B64}", "detail": "auto"}, ] @@ -2146,6 +2170,7 @@ class TestPromptCacheBreakpointToResponses: { "type": "input_image", "image_url": "https://example.com/a.png", + "detail": "auto", "prompt_cache_breakpoint": self.EXPLICIT, }, ], @@ -2238,10 +2263,15 @@ class TestPromptCacheBreakpointToResponses: "output": f"r\n{TOOL_RESULT_IMAGE_PLACEHOLDER}", } *_, first_screenshot, last_screenshot = screenshots["content"] - assert first_screenshot == {"type": "input_image", "image_url": "data:image/png;base64,Zmlyc3Q="} + assert first_screenshot == { + "type": "input_image", + "image_url": "data:image/png;base64,Zmlyc3Q=", + "detail": "auto", + } assert last_screenshot == { "type": "input_image", "image_url": "data:image/png;base64,bGFzdA==", + "detail": "auto", "prompt_cache_breakpoint": self.EXPLICIT, } @@ -2249,6 +2279,7 @@ class TestPromptCacheBreakpointToResponses: MARKED_SCREENSHOT_PART: Final = { "type": "input_image", "image_url": "data:image/png;base64,c2hvdA==", + "detail": "auto", "prompt_cache_breakpoint": EXPLICIT, } LOG_OUTPUT: Final = {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok"} @@ -2320,7 +2351,7 @@ class TestPromptCacheBreakpointToResponses: } ] - SCREENSHOT_PART: Final = {"type": "input_image", "image_url": "data:image/png;base64,c2hvdA=="} + SCREENSHOT_PART: Final = {"type": "input_image", "image_url": "data:image/png;base64,c2hvdA==", "detail": "auto"} MARKED_LOG_TEXT: Final = {"type": "text", "text": "build ok", "prompt_cache_breakpoint": EXPLICIT} LOG_TEXT_WITH_SCREENSHOT: Final = f"build ok\n{TOOL_RESULT_IMAGE_PLACEHOLDER}" diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ResponsesImageUtils.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ResponsesImageUtils.test.tsx new file mode 100644 index 00000000000..d5506a447bf --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ResponsesImageUtils.test.tsx @@ -0,0 +1,15 @@ +import { describe, expect, it } from "vitest"; +import { createMultimodalMessage } from "./ResponsesImageUtils"; + +describe("createMultimodalMessage", () => { + it("sends the uploaded image as a Responses input_image with detail, which strict servers require", async () => { + const file = new File(["test content"], "test.png", { type: "image/png" }); + + const result = await createMultimodalMessage("What is in this image?", file); + + expect(result.content).toEqual([ + { type: "input_text", text: "What is in this image?" }, + { type: "input_image", image_url: expect.stringMatching(/^data:image\/png;base64,/), detail: "auto" }, + ]); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ResponsesImageUtils.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ResponsesImageUtils.tsx index 04dfa39d5c6..7215c39a6be 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ResponsesImageUtils.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ResponsesImageUtils.tsx @@ -28,6 +28,7 @@ export const createMultimodalMessage = async ( { type: "input_image", image_url: `data:${mimeType};base64,${base64Data}`, + detail: "auto", }, ], }; diff --git a/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.tsx b/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.tsx index 9150ba0e632..9bf62e59feb 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.tsx @@ -208,6 +208,7 @@ print(response.output_text) # { # "type": "input_image", # "image_url": f"data:image/jpeg;base64,{base64_file}", # or data:application/pdf;base64,{base64_file} +# "detail": "auto", # }, # ], # } @@ -308,18 +309,22 @@ response = client.responses.create( { "type": "input_image", "image_url": f"data:image/jpeg;base64,{base64_image1}", + "detail": "auto", }, { "type": "input_image", "image_url": f"data:image/jpeg;base64,{base64_image2}", + "detail": "auto", }, { "type": "input_image", "file_id": file_id1, + "detail": "auto", }, { "type": "input_image", "file_id": file_id2, + "detail": "auto", } ], } @@ -392,18 +397,22 @@ response = client.responses.create( { "type": "input_image", "image_url": f"data:image/jpeg;base64,{base64_image1}", + "detail": "auto", }, { "type": "input_image", "image_url": f"data:image/jpeg;base64,{base64_image2}", + "detail": "auto", }, { "type": "input_image", "file_id": file_id1, + "detail": "auto", }, { "type": "input_image", "file_id": file_id2, + "detail": "auto", } ], } @@ -475,18 +484,22 @@ response = client.responses.create( { "type": "input_image", "image_url": f"data:image/jpeg;base64,{base64_image1}", + "detail": "auto", }, { "type": "input_image", "image_url": f"data:image/jpeg;base64,{base64_image2}", + "detail": "auto", }, { "type": "input_image", "file_id": file_id1, + "detail": "auto", }, { "type": "input_image", "file_id": file_id2, + "detail": "auto", } ], } diff --git a/ui/litellm-dashboard/src/components/chat_ui/types.ts b/ui/litellm-dashboard/src/components/chat_ui/types.ts index f32da3f11b7..e208377674a 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/types.ts +++ b/ui/litellm-dashboard/src/components/chat_ui/types.ts @@ -50,4 +50,5 @@ export interface MultimodalContent { type: "input_text" | "input_image"; text?: string; image_url?: string; + detail?: "low" | "high" | "auto"; }