From b9478108b0ea9dc61a1818af283ca56748f8473a Mon Sep 17 00:00:00 2001 From: lkshrk Date: Sat, 19 Sep 2026 13:26:02 +0200 Subject: [PATCH 1/4] fix(anthropic): keep auto-injected cache_control on the tool_result block The default trailing-turn breakpoint from enable_anthropic_prompt_caching marked the last content block of a tool message. convert_to_anthropic_tool_result copies block-level markers into tool_result.content, which the Anthropic API rejects: "cache_control may not be specified within `tool_result.content`". Mark tool messages at message level instead; the transformation maps that onto the tool_result block itself. Fixes #41954 --- .../anthropic_cache_control_hook.py | 6 +- .../test_anthropic_cache_control_hook.py | 56 +++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 45c82589d2d..431b5e10c42 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -788,8 +788,10 @@ class AnthropicCacheControlHook(CustomPromptManagement): message_content: Final = message.get("content", None) - # 1. if string, insert cache control in the message - if isinstance(message_content, str): + # 1. string content, or a tool message: insert cache control at message level. + # Anthropic only accepts cache_control on the tool_result block itself, never + # inside tool_result.content; the message-level marker maps onto that block. + if message.get("role") == "tool" or isinstance(message_content, str): message["cache_control"] = control # 2. list of objects - only apply to the last item Claude accepts a breakpoint on elif isinstance(message_content, list): diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index c70c7080ab8..6a297295b80 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -4963,3 +4963,59 @@ def test_cache_control_hook_injects_next_to_a_malformed_message_instead_of_raisi {"type": "ephemeral"}, {"type": "ephemeral"}, ] + + +def test_tool_message_injection_lands_on_tool_result_not_inside_its_content(): + """ + Anthropic rejects cache_control inside tool_result.content + ("cache_control may not be specified within `tool_result.content`"). + The default trailing-turn breakpoint must therefore mark a tool message + at message level, which the Anthropic transformation maps onto the + tool_result block. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + anthropic_messages_pt, + ) + + hook = AnthropicCacheControlHook() + messages: list[AllMessageValues] = [ + {"role": "user", "content": [{"type": "text", "text": "list files"}]}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "toolu_01", + "type": "function", + "function": {"name": "ls", "arguments": "{}"}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "toolu_01", + "content": [ + {"type": "text", "text": "file1"}, + {"type": "text", "text": "file2"}, + ], + }, + ] + + hook._safe_insert_cache_control_in_message( + message=messages[-1], + control={"type": "ephemeral"}, + ) + + assert messages[-1]["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in messages[-1]["content"]) + assert hook._message_has_cache_control(messages[-1], frozenset()) is True + + anthropic_messages = anthropic_messages_pt( + model="claude-sonnet-4-5", + messages=messages, + llm_provider="anthropic", + ) + tool_result = anthropic_messages[-1]["content"][0] + assert tool_result["type"] == "tool_result" + assert tool_result["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in tool_result["content"]) From 666cb25ce75ea41b0a34a40ae92f8d2312d113ba Mon Sep 17 00:00:00 2001 From: lkshrk Date: Sat, 3 Oct 2026 08:16:47 +0200 Subject: [PATCH 2/4] fix(anthropic): move block-level cache_control in tool messages onto the tool_result block Anthropic rejects cache_control inside tool_result.content. convert_to_anthropic_tool_result copied markers from text, image, document and file blocks of an OpenAI tool message into tool_result.content, so any client sending block-level markers on tool output got a 400. The last block marker now lands on the tool_result itself, and a message-level marker still takes precedence. --- .../prompt_templates/factory.py | 35 ++++--- ...llm_core_utils_prompt_templates_factory.py | 92 ++++++++++++++----- 2 files changed, 87 insertions(+), 40 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index de896cde4dc..ee6e5328a98 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1802,6 +1802,19 @@ def _is_anthropic_document_data_uri(url: str) -> bool: return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES +def _last_block_cache_control( + content: str | Iterable[Mapping[str, object]] | None, +) -> ChatCompletionCachedContent | None: + if not isinstance(content, list): + return None + markers: Final = tuple( + cast(ChatCompletionCachedContent, marker) + for block in content + if isinstance(marker := block.get("cache_control"), dict) + ) + return markers[-1] if markers else None + + def convert_to_anthropic_tool_result( message: ChatCompletionToolMessage | ChatCompletionFunctionMessage, force_base64: bool = False, @@ -1858,15 +1871,10 @@ def convert_to_anthropic_tool_result( ] = [] for content in content_list: if content["type"] == "text": - # Only include cache_control if explicitly set and not None - # to avoid sending "cache_control": null which breaks some API channels text_content: AnthropicMessagesToolResultContent = { "type": "text", "text": content["text"], } - cache_control_value = content.get("cache_control") - if cache_control_value is not None: - text_content["cache_control"] = cache_control_value anthropic_content_list.append(text_content) elif content["type"] == "image_url": image_url_value = content["image_url"] @@ -1881,10 +1889,6 @@ def convert_to_anthropic_tool_result( "file": {"file_data": url_str}, } _document_block = anthropic_process_openai_file_message(synth_file_message) - _document_block = add_cache_control_to_content( - anthropic_content_element=cast(AnthropicMessagesDocumentParam, _document_block), - original_content_element=content, - ) anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _document_block)) else: _anthropic_image_param = create_anthropic_image_param( @@ -1892,10 +1896,6 @@ def convert_to_anthropic_tool_result( format=format, is_bedrock_invoke=force_base64, ) - _anthropic_image_param = add_cache_control_to_content( - anthropic_content_element=_anthropic_image_param, - original_content_element=content, - ) anthropic_content_list.append( cast( AnthropicMessagesImageParam | AnthropicMessagesDocumentParam, @@ -1906,16 +1906,13 @@ def convert_to_anthropic_tool_result( anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"])) elif content["type"] == "file": _file_block = anthropic_process_openai_file_message(content) - _file_block = add_cache_control_to_content( - anthropic_content_element=cast(AnthropicMessagesDocumentParam, _file_block), - original_content_element=content, - ) - anthropic_content_list.append(_file_block) + anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _file_block)) anthropic_content = anthropic_content_list anthropic_tool_result: AnthropicMessagesToolResultParam | None = None ## PROMPT CACHING CHECK ## - cache_control: Final = message.get("cache_control", None) + # Anthropic rejects cache_control inside tool_result.content, so block-level markers move onto the tool_result + cache_control: Final = message.get("cache_control") or _last_block_cache_control(message["content"]) if message["role"] == "tool": tool_message: Final[ChatCompletionToolMessage] = message tool_call_id: str = tool_message["tool_call_id"] diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 778c9d68be5..bf6cee0fa65 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -3144,9 +3144,8 @@ def test_bedrock_tools_pt_keeps_anthropic_input_schema_tools(): def test_convert_to_anthropic_tool_result_image_with_cache_control(): """ - Test that cache_control is properly applied to image content in tool results. - This tests the functionality added in the uncommitted changes where - add_cache_control_to_content is called for image_url content types. + A cache_control on an image inside a tool message lands on the tool_result block, + since Anthropic rejects cache_control within tool_result.content. """ from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_tool_result, @@ -3181,12 +3180,11 @@ def test_convert_to_anthropic_tool_result_image_with_cache_control(): assert result["content"][0]["type"] == "text" assert result["content"][0]["text"] == "Here is the image you requested:" - # Verify image content with cache_control assert result["content"][1]["type"] == "image" assert result["content"][1]["source"]["type"] == "base64" assert result["content"][1]["source"]["media_type"] == "image/jpeg" - assert "cache_control" in result["content"][1] - assert result["content"][1]["cache_control"]["type"] == "ephemeral" + assert result["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in result["content"]) def test_convert_to_anthropic_tool_result_image_without_cache_control(): @@ -3260,23 +3258,75 @@ def test_convert_to_anthropic_tool_result_mixed_content_with_cache_control(): assert result["type"] == "tool_result" assert isinstance(result["content"], list) - assert len(result["content"]) == 4 + assert [block["type"] for block in result["content"]] == ["text", "image", "text", "image"] + assert result["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in result["content"]) - # First text with cache_control - assert result["content"][0]["type"] == "text" - assert result["content"][0]["cache_control"]["type"] == "ephemeral" - # First image with cache_control - assert result["content"][1]["type"] == "image" - assert result["content"][1]["cache_control"]["type"] == "ephemeral" +def test_convert_to_anthropic_tool_result_last_block_marker_wins(): + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) - # Second text without cache_control (cache_control will be None if not set) - assert result["content"][2]["type"] == "text" - assert result["content"][2].get("cache_control") is None + message = { + "role": "tool", + "tool_call_id": "call_ttl", + "content": [ + {"type": "text", "text": "older", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + {"type": "text", "text": "newest", "cache_control": {"type": "ephemeral"}}, + ], + } - # Second image without cache_control (cache_control will be None if not set) - assert result["content"][3]["type"] == "image" - assert result["content"][3].get("cache_control") is None + result = convert_to_anthropic_tool_result(message) + + assert result["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in result["content"]) + + +def test_convert_to_anthropic_tool_result_message_marker_wins_over_block_marker(): + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + message = { + "role": "tool", + "tool_call_id": "call_msg", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + "content": [{"type": "text", "text": "out", "cache_control": {"type": "ephemeral"}}], + } + + result = convert_to_anthropic_tool_result(message) + + assert result["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + assert "cache_control" not in result["content"][0] + + +def test_anthropic_messages_pt_tool_text_block_marker_stays_out_of_tool_result_content(): + from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt + + messages = [ + {"role": "user", "content": "list files"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "toolu_01", "type": "function", "function": {"name": "ls", "arguments": "{}"}}], + }, + { + "role": "tool", + "tool_call_id": "toolu_01", + "content": [ + {"type": "text", "text": "file1"}, + {"type": "text", "text": "file2", "cache_control": {"type": "ephemeral"}}, + ], + }, + ] + + anthropic_messages = anthropic_messages_pt(model="claude-sonnet-4-5", messages=messages, llm_provider="anthropic") + + tool_result = anthropic_messages[-1]["content"][0] + assert tool_result["type"] == "tool_result" + assert tool_result["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in tool_result["content"]) def test_convert_to_anthropic_tool_result_image_url_as_http(): @@ -3301,11 +3351,11 @@ def test_convert_to_anthropic_tool_result_image_url_as_http(): result = convert_to_anthropic_tool_result(message) - # Verify image is passed as URL reference with cache_control assert result["content"][0]["type"] == "image" assert result["content"][0]["source"]["type"] == "url" assert result["content"][0]["source"]["url"] == "https://example.com/image.jpg" - assert result["content"][0]["cache_control"]["type"] == "ephemeral" + assert result["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in result["content"][0] def test_anthropic_messages_pt_server_tool_use_passthrough(): From b1c624d4939cda6e74259f4b050bcd2bfb8fb1ec Mon Sep 17 00:00:00 2001 From: lkshrk Date: Sat, 10 Oct 2026 02:46:15 +0200 Subject: [PATCH 3/4] test(anthropic): pin tool_result cache_control placement on the /v1/messages bridge With enable_anthropic_prompt_caching on, a /v1/messages request routed through the chat completions bridge got the trailing-turn breakpoint nested inside tool_result.content. Cover both the hook-injected and the client-sent marker. --- .../native/test_anthropic_messages_handler.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index 5ce6d03f477..bc77e51fa66 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -2455,3 +2455,47 @@ async def test_anthropic_messages_strict_tools_rejection_is_sent_again_without_s assert first["tools"] == [tool] assert second == {**first, "tools": [{"name": "lookup_record", "input_schema": {"type": "object"}}]} assert _JSON_BODY.validate_python(response)["id"] == "msg_strict_retry" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("client_marker", [False, True]) +async def test_messages_bridge_keeps_tool_result_cache_control_on_tool_result(monkeypatch, client_marker): + from litellm.llms.anthropic.pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + captured = {} + + async def fake_send(self, request, **kwargs): + captured["body"] = json.loads(request.content) + raise httpx.ConnectError("cut at the wire", request=request) + + tool_result: dict[str, Any] = { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [{"type": "text", "text": "file1"}, {"type": "text", "text": "file2"}], + } + if client_marker: + tool_result["cache_control"] = {"type": "ephemeral"} + + with ( + patch.object(httpx.AsyncClient, "send", fake_send), + pytest.raises(litellm.exceptions.InternalServerError), + ): + await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler( + max_tokens=8, + messages=[ + {"role": "user", "content": [{"type": "text", "text": "list files"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01", "name": "ls", "input": {}}]}, + {"role": "user", "content": [tool_result]}, + ], + model="anthropic/claude-sonnet-4-5", + tools=[{"name": "ls", "description": "list", "input_schema": {"type": "object", "properties": {}}}], + api_key="test-api-key", + ) + + sent_tool_result = captured["body"]["messages"][-1]["content"][0] + assert sent_tool_result["type"] == "tool_result" + assert sent_tool_result["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in sent_tool_result["content"]) From 15a80c31f42917bf012f5e85e9074d52e4d64e73 Mon Sep 17 00:00:00 2001 From: lkshrk Date: Sat, 10 Oct 2026 03:46:06 +0200 Subject: [PATCH 4/4] fix(anthropic): index message role directly in cache control hook --- litellm/integrations/anthropic_cache_control_hook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 431b5e10c42..5a047a09137 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -791,7 +791,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): # 1. string content, or a tool message: insert cache control at message level. # Anthropic only accepts cache_control on the tool_result block itself, never # inside tool_result.content; the message-level marker maps onto that block. - if message.get("role") == "tool" or isinstance(message_content, str): + if message["role"] == "tool" or isinstance(message_content, str): message["cache_control"] = control # 2. list of objects - only apply to the last item Claude accepts a breakpoint on elif isinstance(message_content, list):