fix(anthropic): move block-level cache_control in tool messages onto the tool_result block

Anthropic rejects cache_control inside tool_result.content. convert_to_anthropic_tool_result copied markers from text, image, document and file blocks of an OpenAI tool message into tool_result.content, so any client sending block-level markers on tool output got a 400. The last block marker now lands on the tool_result itself, and a message-level marker still takes precedence.
This commit is contained in:
lkshrk 2026-10-03 08:16:47 +02:00
parent b9478108b0
commit 666cb25ce7
No known key found for this signature in database
2 changed files with 87 additions and 40 deletions

View file

@ -1802,6 +1802,19 @@ def _is_anthropic_document_data_uri(url: str) -> bool:
return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES
def _last_block_cache_control(
content: str | Iterable[Mapping[str, object]] | None,
) -> ChatCompletionCachedContent | None:
if not isinstance(content, list):
return None
markers: Final = tuple(
cast(ChatCompletionCachedContent, marker)
for block in content
if isinstance(marker := block.get("cache_control"), dict)
)
return markers[-1] if markers else None
def convert_to_anthropic_tool_result(
message: ChatCompletionToolMessage | ChatCompletionFunctionMessage,
force_base64: bool = False,
@ -1858,15 +1871,10 @@ def convert_to_anthropic_tool_result(
] = []
for content in content_list:
if content["type"] == "text":
# Only include cache_control if explicitly set and not None
# to avoid sending "cache_control": null which breaks some API channels
text_content: AnthropicMessagesToolResultContent = {
"type": "text",
"text": content["text"],
}
cache_control_value = content.get("cache_control")
if cache_control_value is not None:
text_content["cache_control"] = cache_control_value
anthropic_content_list.append(text_content)
elif content["type"] == "image_url":
image_url_value = content["image_url"]
@ -1881,10 +1889,6 @@ def convert_to_anthropic_tool_result(
"file": {"file_data": url_str},
}
_document_block = anthropic_process_openai_file_message(synth_file_message)
_document_block = add_cache_control_to_content(
anthropic_content_element=cast(AnthropicMessagesDocumentParam, _document_block),
original_content_element=content,
)
anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _document_block))
else:
_anthropic_image_param = create_anthropic_image_param(
@ -1892,10 +1896,6 @@ def convert_to_anthropic_tool_result(
format=format,
is_bedrock_invoke=force_base64,
)
_anthropic_image_param = add_cache_control_to_content(
anthropic_content_element=_anthropic_image_param,
original_content_element=content,
)
anthropic_content_list.append(
cast(
AnthropicMessagesImageParam | AnthropicMessagesDocumentParam,
@ -1906,16 +1906,13 @@ def convert_to_anthropic_tool_result(
anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
elif content["type"] == "file":
_file_block = anthropic_process_openai_file_message(content)
_file_block = add_cache_control_to_content(
anthropic_content_element=cast(AnthropicMessagesDocumentParam, _file_block),
original_content_element=content,
)
anthropic_content_list.append(_file_block)
anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _file_block))
anthropic_content = anthropic_content_list
anthropic_tool_result: AnthropicMessagesToolResultParam | None = None
## PROMPT CACHING CHECK ##
cache_control: Final = message.get("cache_control", None)
# Anthropic rejects cache_control inside tool_result.content, so block-level markers move onto the tool_result
cache_control: Final = message.get("cache_control") or _last_block_cache_control(message["content"])
if message["role"] == "tool":
tool_message: Final[ChatCompletionToolMessage] = message
tool_call_id: str = tool_message["tool_call_id"]

View file

@ -3144,9 +3144,8 @@ def test_bedrock_tools_pt_keeps_anthropic_input_schema_tools():
def test_convert_to_anthropic_tool_result_image_with_cache_control():
"""
Test that cache_control is properly applied to image content in tool results.
This tests the functionality added in the uncommitted changes where
add_cache_control_to_content is called for image_url content types.
A cache_control on an image inside a tool message lands on the tool_result block,
since Anthropic rejects cache_control within tool_result.content.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
@ -3181,12 +3180,11 @@ def test_convert_to_anthropic_tool_result_image_with_cache_control():
assert result["content"][0]["type"] == "text"
assert result["content"][0]["text"] == "Here is the image you requested:"
# Verify image content with cache_control
assert result["content"][1]["type"] == "image"
assert result["content"][1]["source"]["type"] == "base64"
assert result["content"][1]["source"]["media_type"] == "image/jpeg"
assert "cache_control" in result["content"][1]
assert result["content"][1]["cache_control"]["type"] == "ephemeral"
assert result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in result["content"])
def test_convert_to_anthropic_tool_result_image_without_cache_control():
@ -3260,23 +3258,75 @@ def test_convert_to_anthropic_tool_result_mixed_content_with_cache_control():
assert result["type"] == "tool_result"
assert isinstance(result["content"], list)
assert len(result["content"]) == 4
assert [block["type"] for block in result["content"]] == ["text", "image", "text", "image"]
assert result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in result["content"])
# First text with cache_control
assert result["content"][0]["type"] == "text"
assert result["content"][0]["cache_control"]["type"] == "ephemeral"
# First image with cache_control
assert result["content"][1]["type"] == "image"
assert result["content"][1]["cache_control"]["type"] == "ephemeral"
def test_convert_to_anthropic_tool_result_last_block_marker_wins():
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
)
# Second text without cache_control (cache_control will be None if not set)
assert result["content"][2]["type"] == "text"
assert result["content"][2].get("cache_control") is None
message = {
"role": "tool",
"tool_call_id": "call_ttl",
"content": [
{"type": "text", "text": "older", "cache_control": {"type": "ephemeral", "ttl": "1h"}},
{"type": "text", "text": "newest", "cache_control": {"type": "ephemeral"}},
],
}
# Second image without cache_control (cache_control will be None if not set)
assert result["content"][3]["type"] == "image"
assert result["content"][3].get("cache_control") is None
result = convert_to_anthropic_tool_result(message)
assert result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in result["content"])
def test_convert_to_anthropic_tool_result_message_marker_wins_over_block_marker():
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
)
message = {
"role": "tool",
"tool_call_id": "call_msg",
"cache_control": {"type": "ephemeral", "ttl": "1h"},
"content": [{"type": "text", "text": "out", "cache_control": {"type": "ephemeral"}}],
}
result = convert_to_anthropic_tool_result(message)
assert result["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
assert "cache_control" not in result["content"][0]
def test_anthropic_messages_pt_tool_text_block_marker_stays_out_of_tool_result_content():
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
messages = [
{"role": "user", "content": "list files"},
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "toolu_01", "type": "function", "function": {"name": "ls", "arguments": "{}"}}],
},
{
"role": "tool",
"tool_call_id": "toolu_01",
"content": [
{"type": "text", "text": "file1"},
{"type": "text", "text": "file2", "cache_control": {"type": "ephemeral"}},
],
},
]
anthropic_messages = anthropic_messages_pt(model="claude-sonnet-4-5", messages=messages, llm_provider="anthropic")
tool_result = anthropic_messages[-1]["content"][0]
assert tool_result["type"] == "tool_result"
assert tool_result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in tool_result["content"])
def test_convert_to_anthropic_tool_result_image_url_as_http():
@ -3301,11 +3351,11 @@ def test_convert_to_anthropic_tool_result_image_url_as_http():
result = convert_to_anthropic_tool_result(message)
# Verify image is passed as URL reference with cache_control
assert result["content"][0]["type"] == "image"
assert result["content"][0]["source"]["type"] == "url"
assert result["content"][0]["source"]["url"] == "https://example.com/image.jpg"
assert result["content"][0]["cache_control"]["type"] == "ephemeral"
assert result["cache_control"] == {"type": "ephemeral"}
assert "cache_control" not in result["content"][0]
def test_anthropic_messages_pt_server_tool_use_passthrough():