scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

fix(anthropic): move block-level cache_control in tool messages onto the tool_result block

Browse source 
Anthropic rejects cache_control inside tool_result.content. convert_to_anthropic_tool_result copied markers from text, image, document and file blocks of an OpenAI tool message into tool_result.content, so any client sending block-level markers on tool output got a 400. The last block marker now lands on the tool_result itself, and a message-level marker still takes precedence.
This commit is contained in:
lkshrk 2026-10-03 08:16:47 +02:00
parent 75014255cf
commit bfde0fcc59
No known key found for this signature in database
2 changed files with 92 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -6,7 +6,7 @@ import json
import mimetypes
import re
import xml.etree.ElementTree as ET
from collections.abc import Iterator, Mapping, Sequence
from collections.abc import Iterable, Iterator, Mapping, Sequence
from enum import Enum
from types import MappingProxyType
from typing import Any, Final, TypeAlias, TypedDict, cast, overload
@ -1563,6 +1563,19 @@ def _is_anthropic_document_data_uri(url: str) -> bool:
return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES
def _last_block_cache_control(
content: str | Iterable[Mapping[str, object]] | None,
) -> ChatCompletionCachedContent | None:
if not isinstance(content, list):
return None
markers: Final = tuple(
cast(ChatCompletionCachedContent, marker)
for block in content
if isinstance(marker := block.get("cache_control"), dict)
)
return markers[-1] if markers else None
def convert_to_anthropic_tool_result(
message: ChatCompletionToolMessage | ChatCompletionFunctionMessage,
force_base64: bool = False,
@ -1619,15 +1632,10 @@ def convert_to_anthropic_tool_result(
] = []
for content in content_list:
if content["type"] == "text":
# Only include cache_control if explicitly set and not None
# to avoid sending "cache_control": null which breaks some API channels
text_content: AnthropicMessagesToolResultContent = {
"type": "text",
"text": content["text"],
}
cache_control_value = content.get("cache_control")
if cache_control_value is not None:
text_content["cache_control"] = cache_control_value
anthropic_content_list.append(text_content)
elif content["type"] == "image_url":
image_url_value = content["image_url"]
@ -1642,10 +1650,6 @@ def convert_to_anthropic_tool_result(
"file": {"file_data": url_str},
}
_document_block = anthropic_process_openai_file_message(synth_file_message)
_document_block = add_cache_control_to_content(
anthropic_content_element=cast(AnthropicMessagesDocumentParam, _document_block),
original_content_element=content,
)
anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _document_block))
else:
_anthropic_image_param = create_anthropic_image_param(
@ -1653,26 +1657,19 @@ def convert_to_anthropic_tool_result(
format=format,
is_bedrock_invoke=force_base64,
)
_anthropic_image_param = add_cache_control_to_content(
anthropic_content_element=_anthropic_image_param,
original_content_element=content,
)
anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
anthropic_content_list.append(_anthropic_image_param)
elif content["type"] == "tool_reference":
anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
elif content["type"] == "file":
file_content = cast(ChatCompletionFileObject, content)
_file_block = anthropic_process_openai_file_message(file_content)
_file_block = add_cache_control_to_content(
anthropic_content_element=cast(AnthropicMessagesDocumentParam, _file_block),
original_content_element=content,
)
anthropic_content_list.append(_file_block)
anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _file_block))
anthropic_content = anthropic_content_list
anthropic_tool_result: AnthropicMessagesToolResultParam | None = None
## PROMPT CACHING CHECK ##
cache_control: Final = message.get("cache_control", None)
# Anthropic rejects cache_control inside tool_result.content, so block-level markers move onto the tool_result
cache_control: Final = message.get("cache_control") or _last_block_cache_control(message["content"])
if message["role"] == "tool":
tool_message: Final[ChatCompletionToolMessage] = message
tool_call_id: str = tool_message["tool_call_id"]

96
tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
View file

@ -1670,9 +1670,8 @@ def test_bedrock_tools_pt_keeps_anthropic_input_schema_tools():
def test_convert_to_anthropic_tool_result_image_with_cache_control():
"""
Test that cache_control is properly applied to image content in tool results.
This tests the functionality added in the uncommitted changes where
add_cache_control_to_content is called for image_url content types.
A cache_control on an image inside a tool message lands on the tool_result block,
since Anthropic rejects cache_control within tool_result.content.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
@ -1707,12 +1706,11 @@ def test_convert_to_anthropic_tool_result_image_with_cache_control():
assert result["content"][0]["type"] == "text"
assert result["content"][0]["text"] == "Here is the image you requested:"
# Verify image content with cache_control
assert result["content"][1]["type"] == "image"
assert result["content"][1]["source"]["type"] == "base64"
assert result["content"][1]["source"]["media_type"] == "image/jpeg"
assert "cache_control" in result["content"][1]
assert result["content"][1]["cache_control"]["type"] == "ephemeral"
assert result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in result["content"])
def test_convert_to_anthropic_tool_result_image_without_cache_control():
@ -1786,23 +1784,75 @@ def test_convert_to_anthropic_tool_result_mixed_content_with_cache_control():
assert result["type"] == "tool_result"
assert isinstance(result["content"], list)
assert len(result["content"]) == 4
assert [block["type"] for block in result["content"]] == ["text", "image", "text", "image"]
assert result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in result["content"])
# First text with cache_control
assert result["content"][0]["type"] == "text"
assert result["content"][0]["cache_control"]["type"] == "ephemeral"
# First image with cache_control
assert result["content"][1]["type"] == "image"
assert result["content"][1]["cache_control"]["type"] == "ephemeral"
def test_convert_to_anthropic_tool_result_last_block_marker_wins():
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
)
# Second text without cache_control (cache_control will be None if not set)
assert result["content"][2]["type"] == "text"
assert result["content"][2].get("cache_control") is None
message = {
"role": "tool",
"tool_call_id": "call_ttl",
"content": [
{"type": "text", "text": "older", "cache_control": {"type": "ephemeral", "ttl": "1h"}},
{"type": "text", "text": "newest", "cache_control": {"type": "ephemeral"}},
],
}
# Second image without cache_control (cache_control will be None if not set)
assert result["content"][3]["type"] == "image"
assert result["content"][3].get("cache_control") is None
result = convert_to_anthropic_tool_result(message)
assert result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in result["content"])
def test_convert_to_anthropic_tool_result_message_marker_wins_over_block_marker():
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
)
message = {
"role": "tool",
"tool_call_id": "call_msg",
"cache_control": {"type": "ephemeral", "ttl": "1h"},
"content": [{"type": "text", "text": "out", "cache_control": {"type": "ephemeral"}}],
}
result = convert_to_anthropic_tool_result(message)
assert result["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
assert "cache_control" not in result["content"][0]
def test_anthropic_messages_pt_tool_text_block_marker_stays_out_of_tool_result_content():
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
messages = [
{"role": "user", "content": "list files"},
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "toolu_01", "type": "function", "function": {"name": "ls", "arguments": "{}"}}],
},
{
"role": "tool",
"tool_call_id": "toolu_01",
"content": [
{"type": "text", "text": "file1"},
{"type": "text", "text": "file2", "cache_control": {"type": "ephemeral"}},
],
},
]
anthropic_messages = anthropic_messages_pt(model="claude-sonnet-4-5", messages=messages, llm_provider="anthropic")
tool_result = anthropic_messages[-1]["content"][0]
assert tool_result["type"] == "tool_result"
assert tool_result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in tool_result["content"])
def test_convert_to_anthropic_tool_result_image_url_as_http():
@ -1827,11 +1877,11 @@ def test_convert_to_anthropic_tool_result_image_url_as_http():
result = convert_to_anthropic_tool_result(message)
# Verify image is passed as URL reference with cache_control
assert result["content"][0]["type"] == "image"
assert result["content"][0]["source"]["type"] == "url"
assert result["content"][0]["source"]["url"] == "https://example.com/image.jpg"
assert result["content"][0]["cache_control"]["type"] == "ephemeral"
assert result["cache_control"] == {"type": "ephemeral"}
assert "cache_control" not in result["content"][0]
def test_anthropic_messages_pt_server_tool_use_passthrough():
@ -4139,7 +4189,9 @@ def test_bedrock_converse_messages_pt_user_message_without_content_adds_no_block
assert _bedrock_converse_messages_pt(
messages=with_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock"
) == _bedrock_converse_messages_pt(messages=without_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock")
) == _bedrock_converse_messages_pt(
messages=without_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock"
)
@pytest.mark.asyncio