From 75014255cf011ae612d9e351a32ec267ee98a024 Mon Sep 17 00:00:00 2001 From: lkshrk Date: Sat, 19 Sep 2026 13:26:02 +0200 Subject: [PATCH 1/2] fix(anthropic): keep auto-injected cache_control on the tool_result block The default trailing-turn breakpoint from enable_anthropic_prompt_caching marked the last content block of a tool message. convert_to_anthropic_tool_result copies block-level markers into tool_result.content, which the Anthropic API rejects: "cache_control may not be specified within `tool_result.content`". Mark tool messages at message level instead; the transformation maps that onto the tool_result block itself. Fixes #41954 --- .../anthropic_cache_control_hook.py | 6 +- .../test_anthropic_cache_control_hook.py | 56 +++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 4487aa31e3b..80d59feb009 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -531,8 +531,10 @@ class AnthropicCacheControlHook(CustomPromptManagement): message_content: Final = message.get("content", None) - # 1. if string, insert cache control in the message - if isinstance(message_content, str): + # 1. string content, or a tool message: insert cache control at message level. + # Anthropic only accepts cache_control on the tool_result block itself, never + # inside tool_result.content; the message-level marker maps onto that block. + if message.get("role") == "tool" or isinstance(message_content, str): message["cache_control"] = control # 2. list of objects - only apply to last item per Anthropic spec elif isinstance(message_content, list): diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 4c9fd3cfdf2..b7356c15867 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -3890,3 +3890,59 @@ class TestRecordGatewayInjection: custom_llm_provider="anthropic", ) assert self.KEY not in kwargs["litellm_metadata"] + + +def test_tool_message_injection_lands_on_tool_result_not_inside_its_content(): + """ + Anthropic rejects cache_control inside tool_result.content + ("cache_control may not be specified within `tool_result.content`"). + The default trailing-turn breakpoint must therefore mark a tool message + at message level, which the Anthropic transformation maps onto the + tool_result block. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + anthropic_messages_pt, + ) + + hook = AnthropicCacheControlHook() + messages: List[AllMessageValues] = [ + {"role": "user", "content": [{"type": "text", "text": "list files"}]}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "toolu_01", + "type": "function", + "function": {"name": "ls", "arguments": "{}"}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "toolu_01", + "content": [ + {"type": "text", "text": "file1"}, + {"type": "text", "text": "file2"}, + ], + }, + ] + + hook._safe_insert_cache_control_in_message( + message=messages[-1], + control={"type": "ephemeral"}, + ) + + assert messages[-1]["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in messages[-1]["content"]) + assert hook._message_has_cache_control(messages[-1]) is True + + anthropic_messages = anthropic_messages_pt( + model="claude-sonnet-4-5", + messages=messages, + llm_provider="anthropic", + ) + tool_result = anthropic_messages[-1]["content"][0] + assert tool_result["type"] == "tool_result" + assert tool_result["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in tool_result["content"]) From bfde0fcc59372c003761af73e212a72ee516384e Mon Sep 17 00:00:00 2001 From: lkshrk Date: Sat, 3 Oct 2026 08:16:47 +0200 Subject: [PATCH 2/2] fix(anthropic): move block-level cache_control in tool messages onto the tool_result block Anthropic rejects cache_control inside tool_result.content. convert_to_anthropic_tool_result copied markers from text, image, document and file blocks of an OpenAI tool message into tool_result.content, so any client sending block-level markers on tool output got a 400. The last block marker now lands on the tool_result itself, and a message-level marker still takes precedence. --- .../prompt_templates/factory.py | 39 ++++---- ...llm_core_utils_prompt_templates_factory.py | 96 ++++++++++++++----- 2 files changed, 92 insertions(+), 43 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 90a60a9e46c..8ae22ee5c16 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -6,7 +6,7 @@ import json import mimetypes import re import xml.etree.ElementTree as ET -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Iterable, Iterator, Mapping, Sequence from enum import Enum from types import MappingProxyType from typing import Any, Final, TypeAlias, TypedDict, cast, overload @@ -1563,6 +1563,19 @@ def _is_anthropic_document_data_uri(url: str) -> bool: return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES +def _last_block_cache_control( + content: str | Iterable[Mapping[str, object]] | None, +) -> ChatCompletionCachedContent | None: + if not isinstance(content, list): + return None + markers: Final = tuple( + cast(ChatCompletionCachedContent, marker) + for block in content + if isinstance(marker := block.get("cache_control"), dict) + ) + return markers[-1] if markers else None + + def convert_to_anthropic_tool_result( message: ChatCompletionToolMessage | ChatCompletionFunctionMessage, force_base64: bool = False, @@ -1619,15 +1632,10 @@ def convert_to_anthropic_tool_result( ] = [] for content in content_list: if content["type"] == "text": - # Only include cache_control if explicitly set and not None - # to avoid sending "cache_control": null which breaks some API channels text_content: AnthropicMessagesToolResultContent = { "type": "text", "text": content["text"], } - cache_control_value = content.get("cache_control") - if cache_control_value is not None: - text_content["cache_control"] = cache_control_value anthropic_content_list.append(text_content) elif content["type"] == "image_url": image_url_value = content["image_url"] @@ -1642,10 +1650,6 @@ def convert_to_anthropic_tool_result( "file": {"file_data": url_str}, } _document_block = anthropic_process_openai_file_message(synth_file_message) - _document_block = add_cache_control_to_content( - anthropic_content_element=cast(AnthropicMessagesDocumentParam, _document_block), - original_content_element=content, - ) anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _document_block)) else: _anthropic_image_param = create_anthropic_image_param( @@ -1653,26 +1657,19 @@ def convert_to_anthropic_tool_result( format=format, is_bedrock_invoke=force_base64, ) - _anthropic_image_param = add_cache_control_to_content( - anthropic_content_element=_anthropic_image_param, - original_content_element=content, - ) - anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param)) + anthropic_content_list.append(_anthropic_image_param) elif content["type"] == "tool_reference": anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"])) elif content["type"] == "file": file_content = cast(ChatCompletionFileObject, content) _file_block = anthropic_process_openai_file_message(file_content) - _file_block = add_cache_control_to_content( - anthropic_content_element=cast(AnthropicMessagesDocumentParam, _file_block), - original_content_element=content, - ) - anthropic_content_list.append(_file_block) + anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _file_block)) anthropic_content = anthropic_content_list anthropic_tool_result: AnthropicMessagesToolResultParam | None = None ## PROMPT CACHING CHECK ## - cache_control: Final = message.get("cache_control", None) + # Anthropic rejects cache_control inside tool_result.content, so block-level markers move onto the tool_result + cache_control: Final = message.get("cache_control") or _last_block_cache_control(message["content"]) if message["role"] == "tool": tool_message: Final[ChatCompletionToolMessage] = message tool_call_id: str = tool_message["tool_call_id"] diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 8d2e6b9fd0c..4720896e84d 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1670,9 +1670,8 @@ def test_bedrock_tools_pt_keeps_anthropic_input_schema_tools(): def test_convert_to_anthropic_tool_result_image_with_cache_control(): """ - Test that cache_control is properly applied to image content in tool results. - This tests the functionality added in the uncommitted changes where - add_cache_control_to_content is called for image_url content types. + A cache_control on an image inside a tool message lands on the tool_result block, + since Anthropic rejects cache_control within tool_result.content. """ from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_tool_result, @@ -1707,12 +1706,11 @@ def test_convert_to_anthropic_tool_result_image_with_cache_control(): assert result["content"][0]["type"] == "text" assert result["content"][0]["text"] == "Here is the image you requested:" - # Verify image content with cache_control assert result["content"][1]["type"] == "image" assert result["content"][1]["source"]["type"] == "base64" assert result["content"][1]["source"]["media_type"] == "image/jpeg" - assert "cache_control" in result["content"][1] - assert result["content"][1]["cache_control"]["type"] == "ephemeral" + assert result["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in result["content"]) def test_convert_to_anthropic_tool_result_image_without_cache_control(): @@ -1786,23 +1784,75 @@ def test_convert_to_anthropic_tool_result_mixed_content_with_cache_control(): assert result["type"] == "tool_result" assert isinstance(result["content"], list) - assert len(result["content"]) == 4 + assert [block["type"] for block in result["content"]] == ["text", "image", "text", "image"] + assert result["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in result["content"]) - # First text with cache_control - assert result["content"][0]["type"] == "text" - assert result["content"][0]["cache_control"]["type"] == "ephemeral" - # First image with cache_control - assert result["content"][1]["type"] == "image" - assert result["content"][1]["cache_control"]["type"] == "ephemeral" +def test_convert_to_anthropic_tool_result_last_block_marker_wins(): + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) - # Second text without cache_control (cache_control will be None if not set) - assert result["content"][2]["type"] == "text" - assert result["content"][2].get("cache_control") is None + message = { + "role": "tool", + "tool_call_id": "call_ttl", + "content": [ + {"type": "text", "text": "older", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + {"type": "text", "text": "newest", "cache_control": {"type": "ephemeral"}}, + ], + } - # Second image without cache_control (cache_control will be None if not set) - assert result["content"][3]["type"] == "image" - assert result["content"][3].get("cache_control") is None + result = convert_to_anthropic_tool_result(message) + + assert result["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in result["content"]) + + +def test_convert_to_anthropic_tool_result_message_marker_wins_over_block_marker(): + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + message = { + "role": "tool", + "tool_call_id": "call_msg", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + "content": [{"type": "text", "text": "out", "cache_control": {"type": "ephemeral"}}], + } + + result = convert_to_anthropic_tool_result(message) + + assert result["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + assert "cache_control" not in result["content"][0] + + +def test_anthropic_messages_pt_tool_text_block_marker_stays_out_of_tool_result_content(): + from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt + + messages = [ + {"role": "user", "content": "list files"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "toolu_01", "type": "function", "function": {"name": "ls", "arguments": "{}"}}], + }, + { + "role": "tool", + "tool_call_id": "toolu_01", + "content": [ + {"type": "text", "text": "file1"}, + {"type": "text", "text": "file2", "cache_control": {"type": "ephemeral"}}, + ], + }, + ] + + anthropic_messages = anthropic_messages_pt(model="claude-sonnet-4-5", messages=messages, llm_provider="anthropic") + + tool_result = anthropic_messages[-1]["content"][0] + assert tool_result["type"] == "tool_result" + assert tool_result["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in tool_result["content"]) def test_convert_to_anthropic_tool_result_image_url_as_http(): @@ -1827,11 +1877,11 @@ def test_convert_to_anthropic_tool_result_image_url_as_http(): result = convert_to_anthropic_tool_result(message) - # Verify image is passed as URL reference with cache_control assert result["content"][0]["type"] == "image" assert result["content"][0]["source"]["type"] == "url" assert result["content"][0]["source"]["url"] == "https://example.com/image.jpg" - assert result["content"][0]["cache_control"]["type"] == "ephemeral" + assert result["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in result["content"][0] def test_anthropic_messages_pt_server_tool_use_passthrough(): @@ -4139,7 +4189,9 @@ def test_bedrock_converse_messages_pt_user_message_without_content_adds_no_block assert _bedrock_converse_messages_pt( messages=with_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock" - ) == _bedrock_converse_messages_pt(messages=without_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock") + ) == _bedrock_converse_messages_pt( + messages=without_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock" + ) @pytest.mark.asyncio