This commit is contained in:
lkshrk 2026-10-31 15:27:19 +00:00 • committed by GitHub
commit fb99a3a74c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 191 additions and 42 deletions

View file

@ -974,8 +974,10 @@ class AnthropicCacheControlHook(CustomPromptManagement):
message_content: Final = message.get("content", None)
# 1. if string, insert cache control in the message
if isinstance(message_content, str):
# 1. string content, or a tool message: insert cache control at message level.
# Anthropic only accepts cache_control on the tool_result block itself, never
# inside tool_result.content; the message-level marker maps onto that block.
if message["role"] == "tool" or isinstance(message_content, str):
message["cache_control"] = control
# 2. list of objects - only apply to the last item Claude accepts a breakpoint on
elif isinstance(message_content, list):

View file

@ -1817,6 +1817,19 @@ def _is_anthropic_document_data_uri(url: str) -> bool:
return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES
def _last_block_cache_control(
content: str | Iterable[Mapping[str, object]] | None,
) -> ChatCompletionCachedContent | None:
if not isinstance(content, list):
return None
markers: Final = tuple(
cast(ChatCompletionCachedContent, marker)
for block in content
if isinstance(marker := block.get("cache_control"), dict)
)
return markers[-1] if markers else None
def convert_to_anthropic_tool_result(
message: ChatCompletionToolMessage | ChatCompletionFunctionMessage,
force_base64: bool = False,
@ -1873,15 +1886,10 @@ def convert_to_anthropic_tool_result(
] = []
for content in content_list:
if content["type"] == "text":
# Only include cache_control if explicitly set and not None
# to avoid sending "cache_control": null which breaks some API channels
text_content: AnthropicMessagesToolResultContent = {
"type": "text",
"text": content["text"],
}
cache_control_value = content.get("cache_control")
if cache_control_value is not None:
text_content["cache_control"] = cache_control_value
anthropic_content_list.append(text_content)
elif content["type"] == "image_url":
image_url_value = content["image_url"]
@ -1896,10 +1904,6 @@ def convert_to_anthropic_tool_result(
"file": {"file_data": url_str},
}
_document_block = anthropic_process_openai_file_message(synth_file_message)
_document_block = add_cache_control_to_content(
anthropic_content_element=cast(AnthropicMessagesDocumentParam, _document_block),
original_content_element=content,
)
anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _document_block))
else:
_anthropic_image_param = create_anthropic_image_param(
@ -1907,10 +1911,6 @@ def convert_to_anthropic_tool_result(
format=format,
is_bedrock_invoke=force_base64,
)
_anthropic_image_param = add_cache_control_to_content(
anthropic_content_element=_anthropic_image_param,
original_content_element=content,
)
anthropic_content_list.append(
cast(
AnthropicMessagesImageParam | AnthropicMessagesDocumentParam,
@ -1921,16 +1921,13 @@ def convert_to_anthropic_tool_result(
anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
elif content["type"] == "file":
_file_block = anthropic_process_openai_file_message(content)
_file_block = add_cache_control_to_content(
anthropic_content_element=cast(AnthropicMessagesDocumentParam, _file_block),
original_content_element=content,
)
anthropic_content_list.append(_file_block)
anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _file_block))
anthropic_content = anthropic_content_list
anthropic_tool_result: AnthropicMessagesToolResultParam | None = None
## PROMPT CACHING CHECK ##
cache_control: Final = message.get("cache_control", None)
# Anthropic rejects cache_control inside tool_result.content, so block-level markers move onto the tool_result
cache_control: Final = message.get("cache_control") or _last_block_cache_control(message["content"])
if message["role"] == "tool":
tool_message: Final[ChatCompletionToolMessage] = message
tool_call_id: str = tool_message["tool_call_id"]

View file

@ -5034,3 +5034,59 @@ def test_cache_control_hook_injects_next_to_a_malformed_message_instead_of_raisi
{"type": "ephemeral"},
{"type": "ephemeral"},
]
def test_tool_message_injection_lands_on_tool_result_not_inside_its_content():
"""
Anthropic rejects cache_control inside tool_result.content
("cache_control may not be specified within `tool_result.content`").
The default trailing-turn breakpoint must therefore mark a tool message
at message level, which the Anthropic transformation maps onto the
tool_result block.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
anthropic_messages_pt,
)
hook = AnthropicCacheControlHook()
messages: list[AllMessageValues] = [
{"role": "user", "content": [{"type": "text", "text": "list files"}]},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "toolu_01",
"type": "function",
"function": {"name": "ls", "arguments": "{}"},
}
],
},
{
"role": "tool",
"tool_call_id": "toolu_01",
"content": [
{"type": "text", "text": "file1"},
{"type": "text", "text": "file2"},
],
},
]
hook._safe_insert_cache_control_in_message(
message=messages[-1],
control={"type": "ephemeral"},
)
assert messages[-1]["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in messages[-1]["content"])
assert hook._message_has_cache_control(messages[-1], frozenset()) is True
anthropic_messages = anthropic_messages_pt(
model="claude-sonnet-4-5",
messages=messages,
llm_provider="anthropic",
)
tool_result = anthropic_messages[-1]["content"][0]
assert tool_result["type"] == "tool_result"
assert tool_result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in tool_result["content"])

View file

@ -3144,9 +3144,8 @@ def test_bedrock_tools_pt_keeps_anthropic_input_schema_tools():
def test_convert_to_anthropic_tool_result_image_with_cache_control():
"""
Test that cache_control is properly applied to image content in tool results.
This tests the functionality added in the uncommitted changes where
add_cache_control_to_content is called for image_url content types.
A cache_control on an image inside a tool message lands on the tool_result block,
since Anthropic rejects cache_control within tool_result.content.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
@ -3181,12 +3180,11 @@ def test_convert_to_anthropic_tool_result_image_with_cache_control():
assert result["content"][0]["type"] == "text"
assert result["content"][0]["text"] == "Here is the image you requested:"
# Verify image content with cache_control
assert result["content"][1]["type"] == "image"
assert result["content"][1]["source"]["type"] == "base64"
assert result["content"][1]["source"]["media_type"] == "image/jpeg"
assert "cache_control" in result["content"][1]
assert result["content"][1]["cache_control"]["type"] == "ephemeral"
assert result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in result["content"])
def test_convert_to_anthropic_tool_result_image_without_cache_control():
@ -3260,23 +3258,75 @@ def test_convert_to_anthropic_tool_result_mixed_content_with_cache_control():
assert result["type"] == "tool_result"
assert isinstance(result["content"], list)
assert len(result["content"]) == 4
assert [block["type"] for block in result["content"]] == ["text", "image", "text", "image"]
assert result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in result["content"])
# First text with cache_control
assert result["content"][0]["type"] == "text"
assert result["content"][0]["cache_control"]["type"] == "ephemeral"
# First image with cache_control
assert result["content"][1]["type"] == "image"
assert result["content"][1]["cache_control"]["type"] == "ephemeral"
def test_convert_to_anthropic_tool_result_last_block_marker_wins():
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
)
# Second text without cache_control (cache_control will be None if not set)
assert result["content"][2]["type"] == "text"
assert result["content"][2].get("cache_control") is None
message = {
"role": "tool",
"tool_call_id": "call_ttl",
"content": [
{"type": "text", "text": "older", "cache_control": {"type": "ephemeral", "ttl": "1h"}},
{"type": "text", "text": "newest", "cache_control": {"type": "ephemeral"}},
],
}
# Second image without cache_control (cache_control will be None if not set)
assert result["content"][3]["type"] == "image"
assert result["content"][3].get("cache_control") is None
result = convert_to_anthropic_tool_result(message)
assert result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in result["content"])
def test_convert_to_anthropic_tool_result_message_marker_wins_over_block_marker():
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
)
message = {
"role": "tool",
"tool_call_id": "call_msg",
"cache_control": {"type": "ephemeral", "ttl": "1h"},
"content": [{"type": "text", "text": "out", "cache_control": {"type": "ephemeral"}}],
}
result = convert_to_anthropic_tool_result(message)
assert result["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
assert "cache_control" not in result["content"][0]
def test_anthropic_messages_pt_tool_text_block_marker_stays_out_of_tool_result_content():
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
messages = [
{"role": "user", "content": "list files"},
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "toolu_01", "type": "function", "function": {"name": "ls", "arguments": "{}"}}],
},
{
"role": "tool",
"tool_call_id": "toolu_01",
"content": [
{"type": "text", "text": "file1"},
{"type": "text", "text": "file2", "cache_control": {"type": "ephemeral"}},
],
},
]
anthropic_messages = anthropic_messages_pt(model="claude-sonnet-4-5", messages=messages, llm_provider="anthropic")
tool_result = anthropic_messages[-1]["content"][0]
assert tool_result["type"] == "tool_result"
assert tool_result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in tool_result["content"])
def test_convert_to_anthropic_tool_result_image_url_as_http():
@ -3301,11 +3351,11 @@ def test_convert_to_anthropic_tool_result_image_url_as_http():
result = convert_to_anthropic_tool_result(message)
# Verify image is passed as URL reference with cache_control
assert result["content"][0]["type"] == "image"
assert result["content"][0]["source"]["type"] == "url"
assert result["content"][0]["source"]["url"] == "https://example.com/image.jpg"
assert result["content"][0]["cache_control"]["type"] == "ephemeral"
assert result["cache_control"] == {"type": "ephemeral"}
assert "cache_control" not in result["content"][0]
def test_anthropic_messages_pt_server_tool_use_passthrough():

View file

@ -2459,3 +2459,47 @@ async def test_anthropic_messages_strict_tools_rejection_is_sent_again_without_s
assert first["tools"] == [tool]
assert second == {**first, "tools": [{"name": "lookup_record", "input_schema": {"type": "object"}}]}
assert _JSON_BODY.validate_python(response)["id"] == "msg_strict_retry"
@pytest.mark.asyncio
@pytest.mark.parametrize("client_marker", [False, True])
async def test_messages_bridge_keeps_tool_result_cache_control_on_tool_result(monkeypatch, client_marker):
from litellm.llms.anthropic.pass_through.adapters.handler import (
LiteLLMMessagesToCompletionTransformationHandler,
)
monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
captured = {}
async def fake_send(self, request, **kwargs):
captured["body"] = json.loads(request.content)
raise httpx.ConnectError("cut at the wire", request=request)
tool_result: dict[str, Any] = {
"type": "tool_result",
"tool_use_id": "toolu_01",
"content": [{"type": "text", "text": "file1"}, {"type": "text", "text": "file2"}],
}
if client_marker:
tool_result["cache_control"] = {"type": "ephemeral"}
with (
patch.object(httpx.AsyncClient, "send", fake_send),
pytest.raises(litellm.exceptions.InternalServerError),
):
await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler(
max_tokens=8,
messages=[
{"role": "user", "content": [{"type": "text", "text": "list files"}]},
{"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01", "name": "ls", "input": {}}]},
{"role": "user", "content": [tool_result]},
],
model="anthropic/claude-sonnet-4-5",
tools=[{"name": "ls", "description": "list", "input_schema": {"type": "object", "properties": {}}}],
api_key="test-api-key",
)
sent_tool_result = captured["body"]["messages"][-1]["content"][0]
assert sent_tool_result["type"] == "tool_result"
assert sent_tool_result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in sent_tool_result["content"])