scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-16 04:29:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 139

fix(anthropic): send detail on input_image parts bridged from /v1/messages to Responses (#46720)

Browse source 
* fix(anthropic): send detail on input_image parts bridged from /v1/messages to Responses

* fix(ui): send detail on playground Responses image parts and code snippets

* test(messages): cover image detail on the Responses bridge wire
This commit is contained in:
Mateo Wang 2026-10-15 22:33:09 +00:00 • committed by GitHub
parent 68759a239f
commit 84ff9f84a9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 146 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
litellm/llms/anthropic/pass_through/responses_adapters/transformation.py
View file

@ -96,6 +96,10 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
return source.get("url")
return None
@staticmethod
def _input_image_part(url: str) -> dict[str, object]: # mutable-ok: API message payload
return {"type": "input_image", "image_url": url, "detail": "auto"}
@staticmethod
def _translate_anthropic_document_block_to_file_part(
block: Mapping[str, object],
@ -149,9 +153,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if block.get("type") == "image"
)
return tuple(
with_prompt_cache_breakpoint({"type": "input_image", "image_url": url}, breakpoint)
for url, breakpoint in sources
if url
with_prompt_cache_breakpoint(cls._input_image_part(url), breakpoint) for url, breakpoint in sources if url
)
@classmethod
@ -357,7 +359,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if url:
user_parts.append(
with_prompt_cache_breakpoint(
{"type": "input_image", "image_url": url}, block.get("prompt_cache_breakpoint")
self._input_image_part(url), block.get("prompt_cache_breakpoint")
)
)
elif btype == "document":

69
tests/integration/messages_endpoint/responses_bridge/test_anthropic_messages_openai_bridge_wire.py
View file

@ -1,3 +1,4 @@
import itertools
import json
import uuid
from typing import Final
@ -11,6 +12,12 @@ _BACKEND: Final = "gpt-5.4-mini"
_API_KEY: Final = "synthetic-openai-key"
_CORRECTION: Final = "Stop refactoring the parser and only fix the failing test instead."
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_JSON_OBJECTS: Final = TypeAdapter(tuple[dict[str, JsonValue], ...])
_PIXEL: Final = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
_PASTED_URL: Final = "https://example.invalid/pasted.png"
_MISSING_DETAIL: Final = json.dumps(
{"error": {"message": "1 validation error: input.0.content.1.detail Field required", "type": "BadRequestError"}}
).encode()
def _responses_reply(identity: str, content: str) -> bytes:
@ -35,6 +42,68 @@ def _responses_reply(identity: str, content: str) -> bytes:
).encode()
def _image_parts(body: dict[str, JsonValue]) -> tuple[dict[str, JsonValue], ...]:
contents: Final = (
_JSON_OBJECTS.validate_python(item.get("content", ())) for item in _JSON_OBJECTS.validate_python(body["input"])
)
return tuple(part for part in itertools.chain.from_iterable(contents) if part.get("type") == "input_image")
@pytest.mark.parametrize(
("source", "image_url"),
(
({"type": "base64", "media_type": "image/png", "data": _PIXEL}, f"data:image/png;base64,{_PIXEL}"),
({"type": "url", "url": _PASTED_URL}, _PASTED_URL),
),
ids=("base64", "url"),
)
def test_pasted_image_reaches_a_responses_server_that_requires_image_detail(
gateway: Gateway, source: dict[str, JsonValue], image_url: str
) -> None:
identity: Final = f"openai-image-detail-{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
if any("detail" not in part for part in _image_parts(_JSON_OBJECT.validate_json(request.body))):
return Reply(status=400, body=_MISSING_DETAIL)
return Reply(body=_responses_reply(identity, "A red circle."))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"openai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/messages",
{
"model": model,
"max_tokens": 64,
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "What is in this image?"},
{"type": "image", "source": source},
],
}
],
},
)
requests: Final = wire.drain()
assert len(requests) == 1, response.text
assert _JSON_OBJECT.validate_json(requests[0].body)["input"] == [
{
"type": "message",
"role": "user",
"content": [
{"type": "input_text", "text": "What is in this image?"},
{"type": "input_image", "image_url": image_url, "detail": "auto"},
],
}
], requests[0].body
assert response.status_code == 200, response.text
assert _JSON_OBJECT.validate_json(response.content)["content"] == [{"type": "text", "text": "A red circle."}], (
response.text
)
@pytest.mark.covers("providers.anthropic_messages_openai_bridge.midturn_system_correction_reaches_the_wire")
def test_midturn_system_correction_is_forwarded_to_openai_responses(gateway: Gateway) -> None:
identity: Final = f"openai-midturn-system-{uuid.uuid4().hex}"

1
tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py
View file

@ -516,6 +516,7 @@ def test_anthropic_sdk_cache_control_on_a_tool_result_with_a_screenshot_marks_th
assert screenshot == {
"type": "input_image",
"image_url": "data:image/png;base64,Zm9v",
"detail": "auto",
"prompt_cache_breakpoint": pcb.EXPLICIT,
}, "OpenAI caches only the prefix up to the breakpoint, so the screenshot must sit inside it"
assert request.body.count(b"prompt_cache_breakpoint") == 1, request.body

1
tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py
View file

@ -465,6 +465,7 @@ def test_client_cache_control_on_a_screenshot_inside_a_tool_result_reaches_gpt_5
{
"type": "input_image",
"image_url": "data:image/png;base64,c2hvdA==",
"detail": "auto",
"prompt_cache_breakpoint": {"mode": "explicit"},
},
],

47
tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py
View file

@ -9,7 +9,8 @@ from typing import Any, Dict, Final, List
from unittest.mock import MagicMock
import pytest
from openai.types.responses import ResponseInputImageParam
from pydantic import TypeAdapter
from litellm.constants import (
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
@ -400,7 +401,9 @@ class TestTranslateMessagesToResponsesInput:
]
result = _translate_messages(messages)
assert len(result) == 1
assert result[0]["content"] == [{"type": "input_image", "image_url": "data:image/png;base64,abc123"}]
assert result[0]["content"] == [
{"type": "input_image", "image_url": "data:image/png;base64,abc123", "detail": "auto"}
]
def test_user_url_image(self):
"""User message with URL image source becomes input_image with the URL."""
@ -416,7 +419,9 @@ class TestTranslateMessagesToResponsesInput:
}
]
result = _translate_messages(messages)
assert result[0]["content"] == [{"type": "input_image", "image_url": "https://example.com/img.jpg"}]
assert result[0]["content"] == [
{"type": "input_image", "image_url": "https://example.com/img.jpg", "detail": "auto"}
]
def test_user_base64_image_empty_data_skipped(self):
"""Base64 image with empty data is skipped (no URL can be formed)."""
@ -745,6 +750,7 @@ class TestTranslateMessagesToResponsesInput:
assert result[0]["content"][1] == {
"type": "input_image",
"image_url": "https://example.com/cat.jpg",
"detail": "auto",
}
def test_unknown_image_source_type_skipped(self):
@ -1753,6 +1759,24 @@ class TestToolResultImages:
and any(part.get("type") == "input_image" for part in item.get("content", []))
)
def test_every_bridged_image_is_a_valid_responses_input_image(self):
image: Final = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": self.B64_DATA}}
messages: Final = [
{"role": "user", "content": [{"type": "text", "text": "what is this?"}, image]},
{
"role": "assistant",
"content": [{"type": "tool_use", "id": "toolu_01", "name": "read", "input": {}}],
},
{
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": [image]}],
},
]
parts: Final = self._input_images(_ADAPTER.translate_messages_to_responses_input(messages))
assert len(parts) == 2
for part in parts:
TypeAdapter(ResponseInputImageParam).validate_python(part)
def test_base64_image_survives(self):
items = self._translate(
[{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": self.B64_DATA}}]
@ -1812,7 +1836,7 @@ class TestToolResultImages:
assert self._image_message(items)["content"] == [
{"type": "input_text", "text": TOOL_RESULT_IMAGE_BOUNDARY},
{"type": "input_image", "image_url": self.DATA_URI},
{"type": "input_image", "image_url": self.DATA_URI, "detail": "auto"},
]
def test_sibling_user_blocks_stay_out_of_boundary_message(self):
@ -1825,7 +1849,7 @@ class TestToolResultImages:
assert self._image_message(items)["content"] == [
{"type": "input_text", "text": TOOL_RESULT_IMAGE_BOUNDARY},
{"type": "input_image", "image_url": self.DATA_URI},
{"type": "input_image", "image_url": self.DATA_URI, "detail": "auto"},
]
assert any(
part == {"type": "input_text", "text": "what changed?"}
@ -1976,7 +2000,7 @@ class TestToolResultDocuments:
)
assert image_message["content"] == [
{"type": "input_text", "text": TOOL_RESULT_IMAGE_BOUNDARY},
{"type": "input_image", "image_url": f"data:image/png;base64,{self.PNG_B64}"},
{"type": "input_image", "image_url": f"data:image/png;base64,{self.PNG_B64}", "detail": "auto"},
]
@ -2146,6 +2170,7 @@ class TestPromptCacheBreakpointToResponses:
{
"type": "input_image",
"image_url": "https://example.com/a.png",
"detail": "auto",
"prompt_cache_breakpoint": self.EXPLICIT,
},
],
@ -2238,10 +2263,15 @@ class TestPromptCacheBreakpointToResponses:
"output": f"r\n{TOOL_RESULT_IMAGE_PLACEHOLDER}",
}
*_, first_screenshot, last_screenshot = screenshots["content"]
assert first_screenshot == {"type": "input_image", "image_url": "data:image/png;base64,Zmlyc3Q="}
assert first_screenshot == {
"type": "input_image",
"image_url": "data:image/png;base64,Zmlyc3Q=",
"detail": "auto",
}
assert last_screenshot == {
"type": "input_image",
"image_url": "data:image/png;base64,bGFzdA==",
"detail": "auto",
"prompt_cache_breakpoint": self.EXPLICIT,
}
@ -2249,6 +2279,7 @@ class TestPromptCacheBreakpointToResponses:
MARKED_SCREENSHOT_PART: Final = {
"type": "input_image",
"image_url": "data:image/png;base64,c2hvdA==",
"detail": "auto",
"prompt_cache_breakpoint": EXPLICIT,
}
LOG_OUTPUT: Final = {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok"}
@ -2320,7 +2351,7 @@ class TestPromptCacheBreakpointToResponses:
}
]
SCREENSHOT_PART: Final = {"type": "input_image", "image_url": "data:image/png;base64,c2hvdA=="}
SCREENSHOT_PART: Final = {"type": "input_image", "image_url": "data:image/png;base64,c2hvdA==", "detail": "auto"}
MARKED_LOG_TEXT: Final = {"type": "text", "text": "build ok", "prompt_cache_breakpoint": EXPLICIT}
LOG_TEXT_WITH_SCREENSHOT: Final = f"build ok\n{TOOL_RESULT_IMAGE_PLACEHOLDER}"

15
ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ResponsesImageUtils.test.tsx Normal file
View file

@ -0,0 +1,15 @@
import { describe, expect, it } from "vitest";
import { createMultimodalMessage } from "./ResponsesImageUtils";
describe("createMultimodalMessage", () => {
it("sends the uploaded image as a Responses input_image with detail, which strict servers require", async () => {
const file = new File(["test content"], "test.png", { type: "image/png" });
const result = await createMultimodalMessage("What is in this image?", file);
expect(result.content).toEqual([
{ type: "input_text", text: "What is in this image?" },
{ type: "input_image", image_url: expect.stringMatching(/^data:image\/png;base64,/), detail: "auto" },
]);
});
});

1
ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ResponsesImageUtils.tsx
View file

@ -28,6 +28,7 @@ export const createMultimodalMessage = async (
{
type: "input_image",
image_url: `data:${mimeType};base64,${base64Data}`,
detail: "auto",
},
],
};

13
ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.tsx
View file

@ -208,6 +208,7 @@ print(response.output_text)
# {
# "type": "input_image",
# "image_url": f"data:image/jpeg;base64,{base64_file}", # or data:application/pdf;base64,{base64_file}
# "detail": "auto",
# },
# ],
# }
@ -308,18 +309,22 @@ response = client.responses.create(
{
"type": "input_image",
"image_url": f"data:image/jpeg;base64,{base64_image1}",
"detail": "auto",
},
{
"type": "input_image",
"image_url": f"data:image/jpeg;base64,{base64_image2}",
"detail": "auto",
},
{
"type": "input_image",
"file_id": file_id1,
"detail": "auto",
},
{
"type": "input_image",
"file_id": file_id2,
"detail": "auto",
}
],
}
@ -392,18 +397,22 @@ response = client.responses.create(
{
"type": "input_image",
"image_url": f"data:image/jpeg;base64,{base64_image1}",
"detail": "auto",
},
{
"type": "input_image",
"image_url": f"data:image/jpeg;base64,{base64_image2}",
"detail": "auto",
},
{
"type": "input_image",
"file_id": file_id1,
"detail": "auto",
},
{
"type": "input_image",
"file_id": file_id2,
"detail": "auto",
}
],
}
@ -475,18 +484,22 @@ response = client.responses.create(
{
"type": "input_image",
"image_url": f"data:image/jpeg;base64,{base64_image1}",
"detail": "auto",
},
{
"type": "input_image",
"image_url": f"data:image/jpeg;base64,{base64_image2}",
"detail": "auto",
},
{
"type": "input_image",
"file_id": file_id1,
"detail": "auto",
},
{
"type": "input_image",
"file_id": file_id2,
"detail": "auto",
}
],
}

1
ui/litellm-dashboard/src/components/chat_ui/types.ts
View file

@ -50,4 +50,5 @@ export interface MultimodalContent {
type: "input_text" | "input_image";
text?: string;
image_url?: string;
detail?: "low" | "high" | "auto";
}