diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 90a60a9e46c..8ae22ee5c16 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -6,7 +6,7 @@ import json import mimetypes import re import xml.etree.ElementTree as ET -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Iterable, Iterator, Mapping, Sequence from enum import Enum from types import MappingProxyType from typing import Any, Final, TypeAlias, TypedDict, cast, overload @@ -1563,6 +1563,19 @@ def _is_anthropic_document_data_uri(url: str) -> bool: return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES +def _last_block_cache_control( + content: str | Iterable[Mapping[str, object]] | None, +) -> ChatCompletionCachedContent | None: + if not isinstance(content, list): + return None + markers: Final = tuple( + cast(ChatCompletionCachedContent, marker) + for block in content + if isinstance(marker := block.get("cache_control"), dict) + ) + return markers[-1] if markers else None + + def convert_to_anthropic_tool_result( message: ChatCompletionToolMessage | ChatCompletionFunctionMessage, force_base64: bool = False, @@ -1619,15 +1632,10 @@ def convert_to_anthropic_tool_result( ] = [] for content in content_list: if content["type"] == "text": - # Only include cache_control if explicitly set and not None - # to avoid sending "cache_control": null which breaks some API channels text_content: AnthropicMessagesToolResultContent = { "type": "text", "text": content["text"], } - cache_control_value = content.get("cache_control") - if cache_control_value is not None: - text_content["cache_control"] = cache_control_value anthropic_content_list.append(text_content) elif content["type"] == "image_url": image_url_value = content["image_url"] @@ -1642,10 +1650,6 @@ def convert_to_anthropic_tool_result( "file": {"file_data": url_str}, } _document_block = anthropic_process_openai_file_message(synth_file_message) - _document_block = add_cache_control_to_content( - anthropic_content_element=cast(AnthropicMessagesDocumentParam, _document_block), - original_content_element=content, - ) anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _document_block)) else: _anthropic_image_param = create_anthropic_image_param( @@ -1653,26 +1657,19 @@ def convert_to_anthropic_tool_result( format=format, is_bedrock_invoke=force_base64, ) - _anthropic_image_param = add_cache_control_to_content( - anthropic_content_element=_anthropic_image_param, - original_content_element=content, - ) - anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param)) + anthropic_content_list.append(_anthropic_image_param) elif content["type"] == "tool_reference": anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"])) elif content["type"] == "file": file_content = cast(ChatCompletionFileObject, content) _file_block = anthropic_process_openai_file_message(file_content) - _file_block = add_cache_control_to_content( - anthropic_content_element=cast(AnthropicMessagesDocumentParam, _file_block), - original_content_element=content, - ) - anthropic_content_list.append(_file_block) + anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _file_block)) anthropic_content = anthropic_content_list anthropic_tool_result: AnthropicMessagesToolResultParam | None = None ## PROMPT CACHING CHECK ## - cache_control: Final = message.get("cache_control", None) + # Anthropic rejects cache_control inside tool_result.content, so block-level markers move onto the tool_result + cache_control: Final = message.get("cache_control") or _last_block_cache_control(message["content"]) if message["role"] == "tool": tool_message: Final[ChatCompletionToolMessage] = message tool_call_id: str = tool_message["tool_call_id"] diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 8d2e6b9fd0c..4720896e84d 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1670,9 +1670,8 @@ def test_bedrock_tools_pt_keeps_anthropic_input_schema_tools(): def test_convert_to_anthropic_tool_result_image_with_cache_control(): """ - Test that cache_control is properly applied to image content in tool results. - This tests the functionality added in the uncommitted changes where - add_cache_control_to_content is called for image_url content types. + A cache_control on an image inside a tool message lands on the tool_result block, + since Anthropic rejects cache_control within tool_result.content. """ from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_tool_result, @@ -1707,12 +1706,11 @@ def test_convert_to_anthropic_tool_result_image_with_cache_control(): assert result["content"][0]["type"] == "text" assert result["content"][0]["text"] == "Here is the image you requested:" - # Verify image content with cache_control assert result["content"][1]["type"] == "image" assert result["content"][1]["source"]["type"] == "base64" assert result["content"][1]["source"]["media_type"] == "image/jpeg" - assert "cache_control" in result["content"][1] - assert result["content"][1]["cache_control"]["type"] == "ephemeral" + assert result["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in result["content"]) def test_convert_to_anthropic_tool_result_image_without_cache_control(): @@ -1786,23 +1784,75 @@ def test_convert_to_anthropic_tool_result_mixed_content_with_cache_control(): assert result["type"] == "tool_result" assert isinstance(result["content"], list) - assert len(result["content"]) == 4 + assert [block["type"] for block in result["content"]] == ["text", "image", "text", "image"] + assert result["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in result["content"]) - # First text with cache_control - assert result["content"][0]["type"] == "text" - assert result["content"][0]["cache_control"]["type"] == "ephemeral" - # First image with cache_control - assert result["content"][1]["type"] == "image" - assert result["content"][1]["cache_control"]["type"] == "ephemeral" +def test_convert_to_anthropic_tool_result_last_block_marker_wins(): + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) - # Second text without cache_control (cache_control will be None if not set) - assert result["content"][2]["type"] == "text" - assert result["content"][2].get("cache_control") is None + message = { + "role": "tool", + "tool_call_id": "call_ttl", + "content": [ + {"type": "text", "text": "older", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + {"type": "text", "text": "newest", "cache_control": {"type": "ephemeral"}}, + ], + } - # Second image without cache_control (cache_control will be None if not set) - assert result["content"][3]["type"] == "image" - assert result["content"][3].get("cache_control") is None + result = convert_to_anthropic_tool_result(message) + + assert result["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in result["content"]) + + +def test_convert_to_anthropic_tool_result_message_marker_wins_over_block_marker(): + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + message = { + "role": "tool", + "tool_call_id": "call_msg", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + "content": [{"type": "text", "text": "out", "cache_control": {"type": "ephemeral"}}], + } + + result = convert_to_anthropic_tool_result(message) + + assert result["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + assert "cache_control" not in result["content"][0] + + +def test_anthropic_messages_pt_tool_text_block_marker_stays_out_of_tool_result_content(): + from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt + + messages = [ + {"role": "user", "content": "list files"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "toolu_01", "type": "function", "function": {"name": "ls", "arguments": "{}"}}], + }, + { + "role": "tool", + "tool_call_id": "toolu_01", + "content": [ + {"type": "text", "text": "file1"}, + {"type": "text", "text": "file2", "cache_control": {"type": "ephemeral"}}, + ], + }, + ] + + anthropic_messages = anthropic_messages_pt(model="claude-sonnet-4-5", messages=messages, llm_provider="anthropic") + + tool_result = anthropic_messages[-1]["content"][0] + assert tool_result["type"] == "tool_result" + assert tool_result["cache_control"] == {"type": "ephemeral"} + assert all("cache_control" not in block for block in tool_result["content"]) def test_convert_to_anthropic_tool_result_image_url_as_http(): @@ -1827,11 +1877,11 @@ def test_convert_to_anthropic_tool_result_image_url_as_http(): result = convert_to_anthropic_tool_result(message) - # Verify image is passed as URL reference with cache_control assert result["content"][0]["type"] == "image" assert result["content"][0]["source"]["type"] == "url" assert result["content"][0]["source"]["url"] == "https://example.com/image.jpg" - assert result["content"][0]["cache_control"]["type"] == "ephemeral" + assert result["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in result["content"][0] def test_anthropic_messages_pt_server_tool_use_passthrough(): @@ -4139,7 +4189,9 @@ def test_bedrock_converse_messages_pt_user_message_without_content_adds_no_block assert _bedrock_converse_messages_pt( messages=with_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock" - ) == _bedrock_converse_messages_pt(messages=without_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock") + ) == _bedrock_converse_messages_pt( + messages=without_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock" + ) @pytest.mark.asyncio