scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 109

Merge bfde0fcc59 into a66adb4ff8

Browse source
This commit is contained in:
lkshrk 2026-10-04 00:42:33 +00:00 • committed by GitHub
commit f0a4e15bf4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 152 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -531,8 +531,10 @@ class AnthropicCacheControlHook(CustomPromptManagement):
message_content: Final = message.get("content", None)
# 1. if string, insert cache control in the message
if isinstance(message_content, str):
# 1. string content, or a tool message: insert cache control at message level.
# Anthropic only accepts cache_control on the tool_result block itself, never
# inside tool_result.content; the message-level marker maps onto that block.
if message.get("role") == "tool" or isinstance(message_content, str):
message["cache_control"] = control
# 2. list of objects - only apply to last item per Anthropic spec
elif isinstance(message_content, list):

39
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -6,7 +6,7 @@ import json
import mimetypes
import re
import xml.etree.ElementTree as ET
from collections.abc import Iterator, Mapping, Sequence
from collections.abc import Iterable, Iterator, Mapping, Sequence
from enum import Enum
from types import MappingProxyType
from typing import Any, Final, TypeAlias, TypedDict, cast, overload
@ -1563,6 +1563,19 @@ def _is_anthropic_document_data_uri(url: str) -> bool:
return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES
def _last_block_cache_control(
content: str | Iterable[Mapping[str, object]] | None,
) -> ChatCompletionCachedContent | None:
if not isinstance(content, list):
return None
markers: Final = tuple(
cast(ChatCompletionCachedContent, marker)
for block in content
if isinstance(marker := block.get("cache_control"), dict)
)
return markers[-1] if markers else None
def convert_to_anthropic_tool_result(
message: ChatCompletionToolMessage | ChatCompletionFunctionMessage,
force_base64: bool = False,
@ -1619,15 +1632,10 @@ def convert_to_anthropic_tool_result(
] = []
for content in content_list:
if content["type"] == "text":
# Only include cache_control if explicitly set and not None
# to avoid sending "cache_control": null which breaks some API channels
text_content: AnthropicMessagesToolResultContent = {
"type": "text",
"text": content["text"],
}
cache_control_value = content.get("cache_control")
if cache_control_value is not None:
text_content["cache_control"] = cache_control_value
anthropic_content_list.append(text_content)
elif content["type"] == "image_url":
image_url_value = content["image_url"]
@ -1642,10 +1650,6 @@ def convert_to_anthropic_tool_result(
"file": {"file_data": url_str},
}
_document_block = anthropic_process_openai_file_message(synth_file_message)
_document_block = add_cache_control_to_content(
anthropic_content_element=cast(AnthropicMessagesDocumentParam, _document_block),
original_content_element=content,
)
anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _document_block))
else:
_anthropic_image_param = create_anthropic_image_param(
@ -1653,26 +1657,19 @@ def convert_to_anthropic_tool_result(
format=format,
is_bedrock_invoke=force_base64,
)
_anthropic_image_param = add_cache_control_to_content(
anthropic_content_element=_anthropic_image_param,
original_content_element=content,
)
anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
anthropic_content_list.append(_anthropic_image_param)
elif content["type"] == "tool_reference":
anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
elif content["type"] == "file":
file_content = cast(ChatCompletionFileObject, content)
_file_block = anthropic_process_openai_file_message(file_content)
_file_block = add_cache_control_to_content(
anthropic_content_element=cast(AnthropicMessagesDocumentParam, _file_block),
original_content_element=content,
)
anthropic_content_list.append(_file_block)
anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _file_block))
anthropic_content = anthropic_content_list
anthropic_tool_result: AnthropicMessagesToolResultParam | None = None
## PROMPT CACHING CHECK ##
cache_control: Final = message.get("cache_control", None)
# Anthropic rejects cache_control inside tool_result.content, so block-level markers move onto the tool_result
cache_control: Final = message.get("cache_control") or _last_block_cache_control(message["content"])
if message["role"] == "tool":
tool_message: Final[ChatCompletionToolMessage] = message
tool_call_id: str = tool_message["tool_call_id"]

56
tests/unit/integrations/test_anthropic_cache_control_hook.py
View file

@ -3890,3 +3890,59 @@ class TestRecordGatewayInjection:
custom_llm_provider="anthropic",
)
assert self.KEY not in kwargs["litellm_metadata"]
def test_tool_message_injection_lands_on_tool_result_not_inside_its_content():
"""
Anthropic rejects cache_control inside tool_result.content
("cache_control may not be specified within `tool_result.content`").
The default trailing-turn breakpoint must therefore mark a tool message
at message level, which the Anthropic transformation maps onto the
tool_result block.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
anthropic_messages_pt,
)
hook = AnthropicCacheControlHook()
messages: List[AllMessageValues] = [
{"role": "user", "content": [{"type": "text", "text": "list files"}]},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "toolu_01",
"type": "function",
"function": {"name": "ls", "arguments": "{}"},
}
],
},
{
"role": "tool",
"tool_call_id": "toolu_01",
"content": [
{"type": "text", "text": "file1"},
{"type": "text", "text": "file2"},
],
},
]
hook._safe_insert_cache_control_in_message(
message=messages[-1],
control={"type": "ephemeral"},
)
assert messages[-1]["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in messages[-1]["content"])
assert hook._message_has_cache_control(messages[-1]) is True
anthropic_messages = anthropic_messages_pt(
model="claude-sonnet-4-5",
messages=messages,
llm_provider="anthropic",
)
tool_result = anthropic_messages[-1]["content"][0]
assert tool_result["type"] == "tool_result"
assert tool_result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in tool_result["content"])

96
tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
View file

@ -1670,9 +1670,8 @@ def test_bedrock_tools_pt_keeps_anthropic_input_schema_tools():
def test_convert_to_anthropic_tool_result_image_with_cache_control():
"""
Test that cache_control is properly applied to image content in tool results.
This tests the functionality added in the uncommitted changes where
add_cache_control_to_content is called for image_url content types.
A cache_control on an image inside a tool message lands on the tool_result block,
since Anthropic rejects cache_control within tool_result.content.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
@ -1707,12 +1706,11 @@ def test_convert_to_anthropic_tool_result_image_with_cache_control():
assert result["content"][0]["type"] == "text"
assert result["content"][0]["text"] == "Here is the image you requested:"
# Verify image content with cache_control
assert result["content"][1]["type"] == "image"
assert result["content"][1]["source"]["type"] == "base64"
assert result["content"][1]["source"]["media_type"] == "image/jpeg"
assert "cache_control" in result["content"][1]
assert result["content"][1]["cache_control"]["type"] == "ephemeral"
assert result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in result["content"])
def test_convert_to_anthropic_tool_result_image_without_cache_control():
@ -1786,23 +1784,75 @@ def test_convert_to_anthropic_tool_result_mixed_content_with_cache_control():
assert result["type"] == "tool_result"
assert isinstance(result["content"], list)
assert len(result["content"]) == 4
assert [block["type"] for block in result["content"]] == ["text", "image", "text", "image"]
assert result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in result["content"])
# First text with cache_control
assert result["content"][0]["type"] == "text"
assert result["content"][0]["cache_control"]["type"] == "ephemeral"
# First image with cache_control
assert result["content"][1]["type"] == "image"
assert result["content"][1]["cache_control"]["type"] == "ephemeral"
def test_convert_to_anthropic_tool_result_last_block_marker_wins():
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
)
# Second text without cache_control (cache_control will be None if not set)
assert result["content"][2]["type"] == "text"
assert result["content"][2].get("cache_control") is None
message = {
"role": "tool",
"tool_call_id": "call_ttl",
"content": [
{"type": "text", "text": "older", "cache_control": {"type": "ephemeral", "ttl": "1h"}},
{"type": "text", "text": "newest", "cache_control": {"type": "ephemeral"}},
],
}
# Second image without cache_control (cache_control will be None if not set)
assert result["content"][3]["type"] == "image"
assert result["content"][3].get("cache_control") is None
result = convert_to_anthropic_tool_result(message)
assert result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in result["content"])
def test_convert_to_anthropic_tool_result_message_marker_wins_over_block_marker():
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
)
message = {
"role": "tool",
"tool_call_id": "call_msg",
"cache_control": {"type": "ephemeral", "ttl": "1h"},
"content": [{"type": "text", "text": "out", "cache_control": {"type": "ephemeral"}}],
}
result = convert_to_anthropic_tool_result(message)
assert result["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
assert "cache_control" not in result["content"][0]
def test_anthropic_messages_pt_tool_text_block_marker_stays_out_of_tool_result_content():
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
messages = [
{"role": "user", "content": "list files"},
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "toolu_01", "type": "function", "function": {"name": "ls", "arguments": "{}"}}],
},
{
"role": "tool",
"tool_call_id": "toolu_01",
"content": [
{"type": "text", "text": "file1"},
{"type": "text", "text": "file2", "cache_control": {"type": "ephemeral"}},
],
},
]
anthropic_messages = anthropic_messages_pt(model="claude-sonnet-4-5", messages=messages, llm_provider="anthropic")
tool_result = anthropic_messages[-1]["content"][0]
assert tool_result["type"] == "tool_result"
assert tool_result["cache_control"] == {"type": "ephemeral"}
assert all("cache_control" not in block for block in tool_result["content"])
def test_convert_to_anthropic_tool_result_image_url_as_http():
@ -1827,11 +1877,11 @@ def test_convert_to_anthropic_tool_result_image_url_as_http():
result = convert_to_anthropic_tool_result(message)
# Verify image is passed as URL reference with cache_control
assert result["content"][0]["type"] == "image"
assert result["content"][0]["source"]["type"] == "url"
assert result["content"][0]["source"]["url"] == "https://example.com/image.jpg"
assert result["content"][0]["cache_control"]["type"] == "ephemeral"
assert result["cache_control"] == {"type": "ephemeral"}
assert "cache_control" not in result["content"][0]
def test_anthropic_messages_pt_server_tool_use_passthrough():
@ -4139,7 +4189,9 @@ def test_bedrock_converse_messages_pt_user_message_without_content_adds_no_block
assert _bedrock_converse_messages_pt(
messages=with_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock"
) == _bedrock_converse_messages_pt(messages=without_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock")
) == _bedrock_converse_messages_pt(
messages=without_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock"
)
@pytest.mark.asyncio