mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 15a80c31f4 into 8ff495be96
This commit is contained in:
commit
fb99a3a74c
5 changed files with 191 additions and 42 deletions
|
|
@ -974,8 +974,10 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
|
||||
message_content: Final = message.get("content", None)
|
||||
|
||||
# 1. if string, insert cache control in the message
|
||||
if isinstance(message_content, str):
|
||||
# 1. string content, or a tool message: insert cache control at message level.
|
||||
# Anthropic only accepts cache_control on the tool_result block itself, never
|
||||
# inside tool_result.content; the message-level marker maps onto that block.
|
||||
if message["role"] == "tool" or isinstance(message_content, str):
|
||||
message["cache_control"] = control
|
||||
# 2. list of objects - only apply to the last item Claude accepts a breakpoint on
|
||||
elif isinstance(message_content, list):
|
||||
|
|
|
|||
|
|
@ -1817,6 +1817,19 @@ def _is_anthropic_document_data_uri(url: str) -> bool:
|
|||
return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES
|
||||
|
||||
|
||||
def _last_block_cache_control(
|
||||
content: str | Iterable[Mapping[str, object]] | None,
|
||||
) -> ChatCompletionCachedContent | None:
|
||||
if not isinstance(content, list):
|
||||
return None
|
||||
markers: Final = tuple(
|
||||
cast(ChatCompletionCachedContent, marker)
|
||||
for block in content
|
||||
if isinstance(marker := block.get("cache_control"), dict)
|
||||
)
|
||||
return markers[-1] if markers else None
|
||||
|
||||
|
||||
def convert_to_anthropic_tool_result(
|
||||
message: ChatCompletionToolMessage | ChatCompletionFunctionMessage,
|
||||
force_base64: bool = False,
|
||||
|
|
@ -1873,15 +1886,10 @@ def convert_to_anthropic_tool_result(
|
|||
] = []
|
||||
for content in content_list:
|
||||
if content["type"] == "text":
|
||||
# Only include cache_control if explicitly set and not None
|
||||
# to avoid sending "cache_control": null which breaks some API channels
|
||||
text_content: AnthropicMessagesToolResultContent = {
|
||||
"type": "text",
|
||||
"text": content["text"],
|
||||
}
|
||||
cache_control_value = content.get("cache_control")
|
||||
if cache_control_value is not None:
|
||||
text_content["cache_control"] = cache_control_value
|
||||
anthropic_content_list.append(text_content)
|
||||
elif content["type"] == "image_url":
|
||||
image_url_value = content["image_url"]
|
||||
|
|
@ -1896,10 +1904,6 @@ def convert_to_anthropic_tool_result(
|
|||
"file": {"file_data": url_str},
|
||||
}
|
||||
_document_block = anthropic_process_openai_file_message(synth_file_message)
|
||||
_document_block = add_cache_control_to_content(
|
||||
anthropic_content_element=cast(AnthropicMessagesDocumentParam, _document_block),
|
||||
original_content_element=content,
|
||||
)
|
||||
anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _document_block))
|
||||
else:
|
||||
_anthropic_image_param = create_anthropic_image_param(
|
||||
|
|
@ -1907,10 +1911,6 @@ def convert_to_anthropic_tool_result(
|
|||
format=format,
|
||||
is_bedrock_invoke=force_base64,
|
||||
)
|
||||
_anthropic_image_param = add_cache_control_to_content(
|
||||
anthropic_content_element=_anthropic_image_param,
|
||||
original_content_element=content,
|
||||
)
|
||||
anthropic_content_list.append(
|
||||
cast(
|
||||
AnthropicMessagesImageParam | AnthropicMessagesDocumentParam,
|
||||
|
|
@ -1921,16 +1921,13 @@ def convert_to_anthropic_tool_result(
|
|||
anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
|
||||
elif content["type"] == "file":
|
||||
_file_block = anthropic_process_openai_file_message(content)
|
||||
_file_block = add_cache_control_to_content(
|
||||
anthropic_content_element=cast(AnthropicMessagesDocumentParam, _file_block),
|
||||
original_content_element=content,
|
||||
)
|
||||
anthropic_content_list.append(_file_block)
|
||||
anthropic_content_list.append(cast(AnthropicMessagesDocumentParam, _file_block))
|
||||
|
||||
anthropic_content = anthropic_content_list
|
||||
anthropic_tool_result: AnthropicMessagesToolResultParam | None = None
|
||||
## PROMPT CACHING CHECK ##
|
||||
cache_control: Final = message.get("cache_control", None)
|
||||
# Anthropic rejects cache_control inside tool_result.content, so block-level markers move onto the tool_result
|
||||
cache_control: Final = message.get("cache_control") or _last_block_cache_control(message["content"])
|
||||
if message["role"] == "tool":
|
||||
tool_message: Final[ChatCompletionToolMessage] = message
|
||||
tool_call_id: str = tool_message["tool_call_id"]
|
||||
|
|
|
|||
|
|
@ -5034,3 +5034,59 @@ def test_cache_control_hook_injects_next_to_a_malformed_message_instead_of_raisi
|
|||
{"type": "ephemeral"},
|
||||
{"type": "ephemeral"},
|
||||
]
|
||||
|
||||
|
||||
def test_tool_message_injection_lands_on_tool_result_not_inside_its_content():
|
||||
"""
|
||||
Anthropic rejects cache_control inside tool_result.content
|
||||
("cache_control may not be specified within `tool_result.content`").
|
||||
The default trailing-turn breakpoint must therefore mark a tool message
|
||||
at message level, which the Anthropic transformation maps onto the
|
||||
tool_result block.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
anthropic_messages_pt,
|
||||
)
|
||||
|
||||
hook = AnthropicCacheControlHook()
|
||||
messages: list[AllMessageValues] = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "list files"}]},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "toolu_01",
|
||||
"type": "function",
|
||||
"function": {"name": "ls", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "toolu_01",
|
||||
"content": [
|
||||
{"type": "text", "text": "file1"},
|
||||
{"type": "text", "text": "file2"},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
hook._safe_insert_cache_control_in_message(
|
||||
message=messages[-1],
|
||||
control={"type": "ephemeral"},
|
||||
)
|
||||
|
||||
assert messages[-1]["cache_control"] == {"type": "ephemeral"}
|
||||
assert all("cache_control" not in block for block in messages[-1]["content"])
|
||||
assert hook._message_has_cache_control(messages[-1], frozenset()) is True
|
||||
|
||||
anthropic_messages = anthropic_messages_pt(
|
||||
model="claude-sonnet-4-5",
|
||||
messages=messages,
|
||||
llm_provider="anthropic",
|
||||
)
|
||||
tool_result = anthropic_messages[-1]["content"][0]
|
||||
assert tool_result["type"] == "tool_result"
|
||||
assert tool_result["cache_control"] == {"type": "ephemeral"}
|
||||
assert all("cache_control" not in block for block in tool_result["content"])
|
||||
|
|
|
|||
|
|
@ -3144,9 +3144,8 @@ def test_bedrock_tools_pt_keeps_anthropic_input_schema_tools():
|
|||
|
||||
def test_convert_to_anthropic_tool_result_image_with_cache_control():
|
||||
"""
|
||||
Test that cache_control is properly applied to image content in tool results.
|
||||
This tests the functionality added in the uncommitted changes where
|
||||
add_cache_control_to_content is called for image_url content types.
|
||||
A cache_control on an image inside a tool message lands on the tool_result block,
|
||||
since Anthropic rejects cache_control within tool_result.content.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
convert_to_anthropic_tool_result,
|
||||
|
|
@ -3181,12 +3180,11 @@ def test_convert_to_anthropic_tool_result_image_with_cache_control():
|
|||
assert result["content"][0]["type"] == "text"
|
||||
assert result["content"][0]["text"] == "Here is the image you requested:"
|
||||
|
||||
# Verify image content with cache_control
|
||||
assert result["content"][1]["type"] == "image"
|
||||
assert result["content"][1]["source"]["type"] == "base64"
|
||||
assert result["content"][1]["source"]["media_type"] == "image/jpeg"
|
||||
assert "cache_control" in result["content"][1]
|
||||
assert result["content"][1]["cache_control"]["type"] == "ephemeral"
|
||||
assert result["cache_control"] == {"type": "ephemeral"}
|
||||
assert all("cache_control" not in block for block in result["content"])
|
||||
|
||||
|
||||
def test_convert_to_anthropic_tool_result_image_without_cache_control():
|
||||
|
|
@ -3260,23 +3258,75 @@ def test_convert_to_anthropic_tool_result_mixed_content_with_cache_control():
|
|||
|
||||
assert result["type"] == "tool_result"
|
||||
assert isinstance(result["content"], list)
|
||||
assert len(result["content"]) == 4
|
||||
assert [block["type"] for block in result["content"]] == ["text", "image", "text", "image"]
|
||||
assert result["cache_control"] == {"type": "ephemeral"}
|
||||
assert all("cache_control" not in block for block in result["content"])
|
||||
|
||||
# First text with cache_control
|
||||
assert result["content"][0]["type"] == "text"
|
||||
assert result["content"][0]["cache_control"]["type"] == "ephemeral"
|
||||
|
||||
# First image with cache_control
|
||||
assert result["content"][1]["type"] == "image"
|
||||
assert result["content"][1]["cache_control"]["type"] == "ephemeral"
|
||||
def test_convert_to_anthropic_tool_result_last_block_marker_wins():
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
convert_to_anthropic_tool_result,
|
||||
)
|
||||
|
||||
# Second text without cache_control (cache_control will be None if not set)
|
||||
assert result["content"][2]["type"] == "text"
|
||||
assert result["content"][2].get("cache_control") is None
|
||||
message = {
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_ttl",
|
||||
"content": [
|
||||
{"type": "text", "text": "older", "cache_control": {"type": "ephemeral", "ttl": "1h"}},
|
||||
{"type": "text", "text": "newest", "cache_control": {"type": "ephemeral"}},
|
||||
],
|
||||
}
|
||||
|
||||
# Second image without cache_control (cache_control will be None if not set)
|
||||
assert result["content"][3]["type"] == "image"
|
||||
assert result["content"][3].get("cache_control") is None
|
||||
result = convert_to_anthropic_tool_result(message)
|
||||
|
||||
assert result["cache_control"] == {"type": "ephemeral"}
|
||||
assert all("cache_control" not in block for block in result["content"])
|
||||
|
||||
|
||||
def test_convert_to_anthropic_tool_result_message_marker_wins_over_block_marker():
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
convert_to_anthropic_tool_result,
|
||||
)
|
||||
|
||||
message = {
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_msg",
|
||||
"cache_control": {"type": "ephemeral", "ttl": "1h"},
|
||||
"content": [{"type": "text", "text": "out", "cache_control": {"type": "ephemeral"}}],
|
||||
}
|
||||
|
||||
result = convert_to_anthropic_tool_result(message)
|
||||
|
||||
assert result["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
|
||||
assert "cache_control" not in result["content"][0]
|
||||
|
||||
|
||||
def test_anthropic_messages_pt_tool_text_block_marker_stays_out_of_tool_result_content():
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "list files"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "toolu_01", "type": "function", "function": {"name": "ls", "arguments": "{}"}}],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "toolu_01",
|
||||
"content": [
|
||||
{"type": "text", "text": "file1"},
|
||||
{"type": "text", "text": "file2", "cache_control": {"type": "ephemeral"}},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
anthropic_messages = anthropic_messages_pt(model="claude-sonnet-4-5", messages=messages, llm_provider="anthropic")
|
||||
|
||||
tool_result = anthropic_messages[-1]["content"][0]
|
||||
assert tool_result["type"] == "tool_result"
|
||||
assert tool_result["cache_control"] == {"type": "ephemeral"}
|
||||
assert all("cache_control" not in block for block in tool_result["content"])
|
||||
|
||||
|
||||
def test_convert_to_anthropic_tool_result_image_url_as_http():
|
||||
|
|
@ -3301,11 +3351,11 @@ def test_convert_to_anthropic_tool_result_image_url_as_http():
|
|||
|
||||
result = convert_to_anthropic_tool_result(message)
|
||||
|
||||
# Verify image is passed as URL reference with cache_control
|
||||
assert result["content"][0]["type"] == "image"
|
||||
assert result["content"][0]["source"]["type"] == "url"
|
||||
assert result["content"][0]["source"]["url"] == "https://example.com/image.jpg"
|
||||
assert result["content"][0]["cache_control"]["type"] == "ephemeral"
|
||||
assert result["cache_control"] == {"type": "ephemeral"}
|
||||
assert "cache_control" not in result["content"][0]
|
||||
|
||||
|
||||
def test_anthropic_messages_pt_server_tool_use_passthrough():
|
||||
|
|
|
|||
|
|
@ -2459,3 +2459,47 @@ async def test_anthropic_messages_strict_tools_rejection_is_sent_again_without_s
|
|||
assert first["tools"] == [tool]
|
||||
assert second == {**first, "tools": [{"name": "lookup_record", "input_schema": {"type": "object"}}]}
|
||||
assert _JSON_BODY.validate_python(response)["id"] == "msg_strict_retry"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("client_marker", [False, True])
|
||||
async def test_messages_bridge_keeps_tool_result_cache_control_on_tool_result(monkeypatch, client_marker):
|
||||
from litellm.llms.anthropic.pass_through.adapters.handler import (
|
||||
LiteLLMMessagesToCompletionTransformationHandler,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
|
||||
captured = {}
|
||||
|
||||
async def fake_send(self, request, **kwargs):
|
||||
captured["body"] = json.loads(request.content)
|
||||
raise httpx.ConnectError("cut at the wire", request=request)
|
||||
|
||||
tool_result: dict[str, Any] = {
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_01",
|
||||
"content": [{"type": "text", "text": "file1"}, {"type": "text", "text": "file2"}],
|
||||
}
|
||||
if client_marker:
|
||||
tool_result["cache_control"] = {"type": "ephemeral"}
|
||||
|
||||
with (
|
||||
patch.object(httpx.AsyncClient, "send", fake_send),
|
||||
pytest.raises(litellm.exceptions.InternalServerError),
|
||||
):
|
||||
await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler(
|
||||
max_tokens=8,
|
||||
messages=[
|
||||
{"role": "user", "content": [{"type": "text", "text": "list files"}]},
|
||||
{"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01", "name": "ls", "input": {}}]},
|
||||
{"role": "user", "content": [tool_result]},
|
||||
],
|
||||
model="anthropic/claude-sonnet-4-5",
|
||||
tools=[{"name": "ls", "description": "list", "input_schema": {"type": "object", "properties": {}}}],
|
||||
api_key="test-api-key",
|
||||
)
|
||||
|
||||
sent_tool_result = captured["body"]["messages"][-1]["content"][0]
|
||||
assert sent_tool_result["type"] == "tool_result"
|
||||
assert sent_tool_result["cache_control"] == {"type": "ephemeral"}
|
||||
assert all("cache_control" not in block for block in sent_tool_result["content"])
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue