mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-17 23:51:30 +00:00
fix(bedrock): preserve cache_control TTL on tools for Claude 4.5+
Bedrock enforces non-increasing TTL ordering across cache_control blocks (tools → system → messages). The tool cache_control TTL was being unconditionally dropped to the default 5m, while system blocks preserved the user-specified TTL for Claude 4.5+ models. This mismatch caused "a ttl='1h' block must not come after a ttl='5m' block" errors when users set ttl='1h' on both tools and system. Converse path: add_cache_point_tool_block() now accepts a model param and preserves TTL for Claude 4.5+, matching _get_cache_point_block(). Invoke path: _remove_ttl_from_cache_control() now also processes tools (was only processing system and messages). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Shubham Arora
parent
72a461ba4a
commit
231210d9cf
5 changed files with 366 additions and 129 deletions
|
|
@ -1393,10 +1393,10 @@ def convert_to_gemini_tool_call_invoke(
|
|||
if tool_calls is not None:
|
||||
for idx, tool in enumerate(tool_calls):
|
||||
if "function" in tool:
|
||||
gemini_function_call: Optional[
|
||||
VertexFunctionCall
|
||||
] = _gemini_tool_call_invoke_helper(
|
||||
function_call_params=tool["function"]
|
||||
gemini_function_call: Optional[VertexFunctionCall] = (
|
||||
_gemini_tool_call_invoke_helper(
|
||||
function_call_params=tool["function"]
|
||||
)
|
||||
)
|
||||
if gemini_function_call is not None:
|
||||
part_dict: VertexPartType = {
|
||||
|
|
@ -1574,9 +1574,7 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915
|
|||
file_data = (
|
||||
file_content.get("file_data", "")
|
||||
if isinstance(file_content, dict)
|
||||
else file_content
|
||||
if isinstance(file_content, str)
|
||||
else ""
|
||||
else file_content if isinstance(file_content, str) else ""
|
||||
)
|
||||
|
||||
if file_data:
|
||||
|
|
@ -2081,9 +2079,9 @@ def _sanitize_empty_text_content(
|
|||
if isinstance(content, str):
|
||||
if not content or not content.strip():
|
||||
message = cast(AllMessageValues, dict(message)) # Make a copy
|
||||
message[
|
||||
"content"
|
||||
] = "[System: Empty message content sanitised to satisfy protocol]"
|
||||
message["content"] = (
|
||||
"[System: Empty message content sanitised to satisfy protocol]"
|
||||
)
|
||||
verbose_logger.debug(
|
||||
f"_sanitize_empty_text_content: Replaced empty text content in {message.get('role')} message"
|
||||
)
|
||||
|
|
@ -2423,9 +2421,9 @@ def anthropic_messages_pt( # noqa: PLR0915
|
|||
# Convert ChatCompletionImageUrlObject to dict if needed
|
||||
image_url_value = m["image_url"]
|
||||
if isinstance(image_url_value, str):
|
||||
image_url_input: Union[
|
||||
str, dict[str, Any]
|
||||
] = image_url_value
|
||||
image_url_input: Union[str, dict[str, Any]] = (
|
||||
image_url_value
|
||||
)
|
||||
else:
|
||||
# ChatCompletionImageUrlObject or dict case - convert to dict
|
||||
image_url_input = {
|
||||
|
|
@ -2452,9 +2450,9 @@ def anthropic_messages_pt( # noqa: PLR0915
|
|||
)
|
||||
|
||||
if "cache_control" in _content_element:
|
||||
_anthropic_content_element[
|
||||
"cache_control"
|
||||
] = _content_element["cache_control"]
|
||||
_anthropic_content_element["cache_control"] = (
|
||||
_content_element["cache_control"]
|
||||
)
|
||||
user_content.append(_anthropic_content_element)
|
||||
elif m.get("type", "") == "text":
|
||||
m = cast(ChatCompletionTextObject, m)
|
||||
|
|
@ -2514,9 +2512,9 @@ def anthropic_messages_pt( # noqa: PLR0915
|
|||
)
|
||||
|
||||
if "cache_control" in _content_element:
|
||||
_anthropic_content_text_element[
|
||||
"cache_control"
|
||||
] = _content_element["cache_control"]
|
||||
_anthropic_content_text_element["cache_control"] = (
|
||||
_content_element["cache_control"]
|
||||
)
|
||||
|
||||
user_content.append(_anthropic_content_text_element)
|
||||
|
||||
|
|
@ -2649,9 +2647,9 @@ def anthropic_messages_pt( # noqa: PLR0915
|
|||
original_content_element=dict(assistant_content_block),
|
||||
)
|
||||
if "cache_control" in _content_element:
|
||||
_anthropic_text_content_element[
|
||||
"cache_control"
|
||||
] = _content_element["cache_control"]
|
||||
_anthropic_text_content_element["cache_control"] = (
|
||||
_content_element["cache_control"]
|
||||
)
|
||||
text_element = _anthropic_text_content_element
|
||||
|
||||
# Interleave: each thinking block precedes its server tool group.
|
||||
|
|
@ -2811,9 +2809,9 @@ def anthropic_messages_pt( # noqa: PLR0915
|
|||
)
|
||||
|
||||
if "cache_control" in _content_element:
|
||||
_anthropic_text_content_element[
|
||||
"cache_control"
|
||||
] = _content_element["cache_control"]
|
||||
_anthropic_text_content_element["cache_control"] = (
|
||||
_content_element["cache_control"]
|
||||
)
|
||||
|
||||
assistant_content.append(_anthropic_text_content_element)
|
||||
|
||||
|
|
@ -5060,12 +5058,25 @@ def make_valid_bedrock_tool_name(input_tool_name: str) -> str:
|
|||
return valid_string
|
||||
|
||||
|
||||
def add_cache_point_tool_block(tool: dict) -> Optional[BedrockToolBlock]:
|
||||
def add_cache_point_tool_block(
|
||||
tool: dict, model: Optional[str] = None
|
||||
) -> Optional[BedrockToolBlock]:
|
||||
from litellm.llms.bedrock.common_utils import is_claude_4_5_on_bedrock
|
||||
|
||||
cache_control = tool.get("cache_control", None)
|
||||
if cache_control is not None:
|
||||
cache_point = cache_control.get("type", "ephemeral")
|
||||
if cache_point == "ephemeral":
|
||||
return {"cachePoint": {"type": "default"}}
|
||||
cache_point_block: CachePointBlock = {"type": "default"}
|
||||
if isinstance(cache_control, dict) and "ttl" in cache_control:
|
||||
ttl = cache_control["ttl"]
|
||||
if (
|
||||
ttl in ["5m", "1h"]
|
||||
and model is not None
|
||||
and is_claude_4_5_on_bedrock(model)
|
||||
):
|
||||
cache_point_block["ttl"] = ttl
|
||||
return {"cachePoint": cache_point_block}
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -5095,7 +5106,9 @@ def _is_bedrock_tool_block(tool: dict) -> bool:
|
|||
)
|
||||
|
||||
|
||||
def _bedrock_tools_pt(tools: List) -> List[BedrockToolBlock]:
|
||||
def _bedrock_tools_pt(
|
||||
tools: List, model: Optional[str] = None
|
||||
) -> List[BedrockToolBlock]:
|
||||
"""
|
||||
OpenAI tools looks like:
|
||||
tools = [
|
||||
|
|
@ -5211,7 +5224,7 @@ def _bedrock_tools_pt(tools: List) -> List[BedrockToolBlock]:
|
|||
tool_block_list.append(tool_block)
|
||||
|
||||
## ADD CACHE POINT TOOL BLOCK ##
|
||||
cache_point_tool_block = add_cache_point_tool_block(tool)
|
||||
cache_point_tool_block = add_cache_point_tool_block(tool, model=model)
|
||||
if cache_point_tool_block is not None:
|
||||
tool_block_list.append(cache_point_tool_block)
|
||||
|
||||
|
|
@ -5278,9 +5291,7 @@ def default_response_schema_prompt(response_schema: dict) -> str:
|
|||
prompt_str = """Use this JSON schema:
|
||||
```json
|
||||
{}
|
||||
```""".format(
|
||||
response_schema
|
||||
)
|
||||
```""".format(response_schema)
|
||||
return prompt_str
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1293,7 +1293,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
)
|
||||
|
||||
# Process regular function tools using existing logic
|
||||
bedrock_tools = _bedrock_tools_pt(regular_tools)
|
||||
bedrock_tools = _bedrock_tools_pt(regular_tools, model=model)
|
||||
|
||||
# Add computer use tools and anthropic_beta if needed (only when computer use tools are present)
|
||||
if computer_use_tools:
|
||||
|
|
@ -1357,7 +1357,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
additional_request_params["tools"] = transformed_computer_tools
|
||||
else:
|
||||
# No computer use tools, process all tools as regular tools
|
||||
bedrock_tools = _bedrock_tools_pt(filtered_tools)
|
||||
bedrock_tools = _bedrock_tools_pt(filtered_tools, model=model)
|
||||
|
||||
# Append pre-formatted tools (systemTool etc.) after transformation
|
||||
bedrock_tools.extend(pre_formatted_tools)
|
||||
|
|
@ -1744,9 +1744,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
|
||||
return message, returned_finish_reason
|
||||
|
||||
def _translate_message_content(
|
||||
self, content_blocks: List[ContentBlock]
|
||||
) -> Tuple[
|
||||
def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tuple[
|
||||
str,
|
||||
List[ChatCompletionToolCallChunk],
|
||||
Optional[List[BedrockConverseReasoningContentBlock]],
|
||||
|
|
@ -1763,9 +1761,9 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"""
|
||||
content_str = ""
|
||||
tools: List[ChatCompletionToolCallChunk] = []
|
||||
reasoningContentBlocks: Optional[
|
||||
List[BedrockConverseReasoningContentBlock]
|
||||
] = None
|
||||
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
|
||||
None
|
||||
)
|
||||
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
|
||||
for idx, content in enumerate(content_blocks):
|
||||
"""
|
||||
|
|
@ -1976,9 +1974,9 @@ class AmazonConverseConfig(BaseConfig):
|
|||
chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
|
||||
content_str = ""
|
||||
tools: List[ChatCompletionToolCallChunk] = []
|
||||
reasoningContentBlocks: Optional[
|
||||
List[BedrockConverseReasoningContentBlock]
|
||||
] = None
|
||||
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
|
||||
None
|
||||
)
|
||||
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
|
||||
|
||||
if message is not None:
|
||||
|
|
@ -1997,17 +1995,17 @@ class AmazonConverseConfig(BaseConfig):
|
|||
provider_specific_fields["citationsContent"] = citationsContentBlocks
|
||||
|
||||
if provider_specific_fields:
|
||||
chat_completion_message[
|
||||
"provider_specific_fields"
|
||||
] = provider_specific_fields
|
||||
chat_completion_message["provider_specific_fields"] = (
|
||||
provider_specific_fields
|
||||
)
|
||||
|
||||
if reasoningContentBlocks is not None:
|
||||
chat_completion_message[
|
||||
"reasoning_content"
|
||||
] = self._transform_reasoning_content(reasoningContentBlocks)
|
||||
chat_completion_message[
|
||||
"thinking_blocks"
|
||||
] = self._transform_thinking_blocks(reasoningContentBlocks)
|
||||
chat_completion_message["reasoning_content"] = (
|
||||
self._transform_reasoning_content(reasoningContentBlocks)
|
||||
)
|
||||
chat_completion_message["thinking_blocks"] = (
|
||||
self._transform_thinking_blocks(reasoningContentBlocks)
|
||||
)
|
||||
chat_completion_message["content"] = content_str
|
||||
filtered_tools = self._filter_json_mode_tools(
|
||||
json_mode=json_mode,
|
||||
|
|
|
|||
|
|
@ -125,7 +125,7 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
- `scope` (e.g., "global") - always removed
|
||||
- `ttl` - removed for older models; Claude 4.5+ supports "5m" and "1h"
|
||||
|
||||
Processes both `system` and `messages` content blocks.
|
||||
Processes `tools`, `system`, and `messages` content blocks.
|
||||
|
||||
Args:
|
||||
anthropic_messages_request: The request dictionary to modify in-place
|
||||
|
|
@ -152,6 +152,12 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
if isinstance(item, dict) and "cache_control" in item:
|
||||
_sanitize_cache_control(item["cache_control"])
|
||||
|
||||
# Process tools
|
||||
if "tools" in anthropic_messages_request:
|
||||
for tool in anthropic_messages_request["tools"]:
|
||||
if isinstance(tool, dict) and "cache_control" in tool:
|
||||
_sanitize_cache_control(tool["cache_control"])
|
||||
|
||||
# Process system (list of content blocks)
|
||||
if "system" in anthropic_messages_request:
|
||||
system = anthropic_messages_request["system"]
|
||||
|
|
@ -394,9 +400,9 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
|
||||
# 1. anthropic_version is required for all claude models
|
||||
if "anthropic_version" not in anthropic_messages_request:
|
||||
anthropic_messages_request[
|
||||
"anthropic_version"
|
||||
] = self.DEFAULT_BEDROCK_ANTHROPIC_API_VERSION
|
||||
anthropic_messages_request["anthropic_version"] = (
|
||||
self.DEFAULT_BEDROCK_ANTHROPIC_API_VERSION
|
||||
)
|
||||
|
||||
# 2. `stream` is not allowed in request body for bedrock invoke
|
||||
if "stream" in anthropic_messages_request:
|
||||
|
|
@ -573,7 +579,9 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
|
||||
raw_input = stop_usage.get("input_tokens")
|
||||
if raw_input is not None:
|
||||
delta_usage["input_tokens"] = raw_input if isinstance(raw_input, int) else 0
|
||||
delta_usage["input_tokens"] = (
|
||||
raw_input if isinstance(raw_input, int) else 0
|
||||
)
|
||||
|
||||
if delta_usage:
|
||||
pending_delta["usage"] = delta_usage # type: ignore[arg-type]
|
||||
|
|
|
|||
|
|
@ -543,7 +543,12 @@ def test_convert_gemini_tool_call_result_with_image_url():
|
|||
message_dict_format = ChatCompletionToolMessage(
|
||||
role="tool",
|
||||
tool_call_id="call_456",
|
||||
content=[{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQ"}}],
|
||||
content=[
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/jpeg;base64,/9j/4AAQ"},
|
||||
}
|
||||
],
|
||||
)
|
||||
last_message_with_tool_calls["tool_calls"][0]["id"] = "call_456"
|
||||
|
||||
|
|
@ -617,11 +622,19 @@ def test_convert_gemini_tool_call_result_with_multiple_anthropic_image_blocks():
|
|||
{"type": "text", "text": "here are two images"},
|
||||
{
|
||||
"type": "image",
|
||||
"source": {"type": "base64", "media_type": "image/png", "data": png_b64},
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/png",
|
||||
"data": png_b64,
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "image",
|
||||
"source": {"type": "base64", "media_type": "image/jpeg", "data": jpeg_b64},
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/jpeg",
|
||||
"data": jpeg_b64,
|
||||
},
|
||||
},
|
||||
],
|
||||
)
|
||||
|
|
@ -644,7 +657,9 @@ def test_convert_gemini_tool_call_result_with_multiple_anthropic_image_blocks():
|
|||
)
|
||||
assert isinstance(result, list), "expected a list of parts"
|
||||
inline_parts = [p for p in result if "inline_data" in p]
|
||||
assert len(inline_parts) == 2, f"expected 2 inline_data parts, got {len(inline_parts)}"
|
||||
assert (
|
||||
len(inline_parts) == 2
|
||||
), f"expected 2 inline_data parts, got {len(inline_parts)}"
|
||||
mime_types = {p["inline_data"]["mime_type"] for p in inline_parts}
|
||||
assert mime_types == {"image/png", "image/jpeg"}
|
||||
|
||||
|
|
@ -681,7 +696,9 @@ def test_convert_gemini_tool_call_result_with_data_url_string():
|
|||
)
|
||||
assert isinstance(result, list), "expected a list of parts"
|
||||
inline_parts = [p for p in result if "inline_data" in p]
|
||||
assert len(inline_parts) == 1, "data-URL image string was not converted to inline_data"
|
||||
assert (
|
||||
len(inline_parts) == 1
|
||||
), "data-URL image string was not converted to inline_data"
|
||||
assert inline_parts[0]["inline_data"]["mime_type"] == "image/png"
|
||||
assert inline_parts[0]["inline_data"]["data"] == tiny_png_b64
|
||||
|
||||
|
|
@ -718,9 +735,9 @@ def test_convert_gemini_tool_call_result_with_data_url_extra_params():
|
|||
assert isinstance(result, list), "expected a list of parts"
|
||||
inline_parts = [p for p in result if "inline_data" in p]
|
||||
assert len(inline_parts) == 1
|
||||
assert inline_parts[0]["inline_data"]["mime_type"] == "image/png", (
|
||||
f"expected clean 'image/png', got '{inline_parts[0]['inline_data']['mime_type']}'"
|
||||
)
|
||||
assert (
|
||||
inline_parts[0]["inline_data"]["mime_type"] == "image/png"
|
||||
), f"expected clean 'image/png', got '{inline_parts[0]['inline_data']['mime_type']}'"
|
||||
|
||||
|
||||
def test_bedrock_tools_unpack_defs():
|
||||
|
|
@ -1007,8 +1024,14 @@ def test_bedrock_image_processor_content_type_document_formats():
|
|||
test_cases = [
|
||||
("https://example.com/doc.pdf", "application/pdf"),
|
||||
("https://example.com/sheet.csv", "text/csv"),
|
||||
("https://example.com/doc.docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"),
|
||||
("https://example.com/sheet.xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"),
|
||||
(
|
||||
"https://example.com/doc.docx",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
),
|
||||
(
|
||||
"https://example.com/sheet.xlsx",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
),
|
||||
("https://example.com/page.html", "text/html"),
|
||||
("https://example.com/readme.txt", "text/plain"),
|
||||
]
|
||||
|
|
@ -1017,7 +1040,9 @@ def test_bedrock_image_processor_content_type_document_formats():
|
|||
_, content_type = BedrockImageProcessor._post_call_image_processing(
|
||||
mock_response, url
|
||||
)
|
||||
assert content_type == expected_mime, f"Expected {expected_mime} for {url}, got {content_type}"
|
||||
assert (
|
||||
content_type == expected_mime
|
||||
), f"Expected {expected_mime} for {url}, got {content_type}"
|
||||
|
||||
|
||||
def test_bedrock_image_processor_content_type_s3_pdf_with_query():
|
||||
|
|
@ -1084,6 +1109,7 @@ def test_bedrock_tools_pt_empty_description():
|
|||
assert tool_spec.get("name") == "get_weather"
|
||||
assert tool_spec.get("description") == "get_weather"
|
||||
|
||||
|
||||
def test_bedrock_create_bedrock_block_deterministic_document_hash():
|
||||
"""
|
||||
Test that _create_bedrock_block generates deterministic document names
|
||||
|
|
@ -1283,7 +1309,9 @@ def test_bedrock_create_bedrock_block_document_name_format():
|
|||
|
||||
# Check format: DocumentPDFmessages_{16_hex_chars}_{format}
|
||||
pattern = r"^DocumentPDFmessages_[0-9a-f]{16}_pdf$"
|
||||
assert re.match(pattern, document_name), f"Document name format mismatch: {document_name}"
|
||||
assert re.match(
|
||||
pattern, document_name
|
||||
), f"Document name format mismatch: {document_name}"
|
||||
|
||||
|
||||
def test_bedrock_create_bedrock_block_different_document_formats():
|
||||
|
|
@ -1313,6 +1341,7 @@ def test_bedrock_create_bedrock_block_different_document_formats():
|
|||
assert block["document"]["name"].endswith(f"_{format_type}")
|
||||
assert block["document"]["format"] == format_type
|
||||
|
||||
|
||||
def test_bedrock_nova_web_search_options_mapping():
|
||||
"""
|
||||
Test that web_search_options is correctly mapped to Nova grounding.
|
||||
|
|
@ -1336,8 +1365,7 @@ def test_bedrock_nova_web_search_options_mapping():
|
|||
|
||||
# Test with search_context_size (should be ignored for Nova)
|
||||
result2 = config._map_web_search_options(
|
||||
{"search_context_size": "high"},
|
||||
"us.amazon.nova-premier-v1:0"
|
||||
{"search_context_size": "high"}, "us.amazon.nova-premier-v1:0"
|
||||
)
|
||||
|
||||
assert result2 is not None
|
||||
|
|
@ -1346,6 +1374,7 @@ def test_bedrock_nova_web_search_options_mapping():
|
|||
assert system_tool2["name"] == "nova_grounding"
|
||||
# Nova doesn't support search_context_size, so it's just ignored
|
||||
|
||||
|
||||
def test_bedrock_tools_pt_does_not_handle_system_tool():
|
||||
"""
|
||||
Verify that _bedrock_tools_pt does NOT handle system_tool format.
|
||||
|
|
@ -1365,12 +1394,10 @@ def test_bedrock_tools_pt_does_not_handle_system_tool():
|
|||
"description": "Get the current weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
}
|
||||
"properties": {"location": {"type": "string"}},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
|
@ -1381,6 +1408,7 @@ def test_bedrock_tools_pt_does_not_handle_system_tool():
|
|||
assert tool_spec is not None
|
||||
assert tool_spec["name"] == "get_weather"
|
||||
|
||||
|
||||
def test_convert_to_anthropic_tool_result_image_with_cache_control():
|
||||
"""
|
||||
Test that cache_control is properly applied to image content in tool results.
|
||||
|
|
@ -1545,6 +1573,8 @@ def test_convert_to_anthropic_tool_result_image_url_as_http():
|
|||
assert result["content"][0]["source"]["type"] == "url"
|
||||
assert result["content"][0]["source"]["url"] == "https://example.com/image.jpg"
|
||||
assert result["content"][0]["cache_control"]["type"] == "ephemeral"
|
||||
|
||||
|
||||
def test_anthropic_messages_pt_server_tool_use_passthrough():
|
||||
"""
|
||||
Test that anthropic_messages_pt passes through server_tool_use and
|
||||
|
|
@ -1555,13 +1585,12 @@ def test_anthropic_messages_pt_server_tool_use_passthrough():
|
|||
|
||||
Fixes: https://github.com/BerriAI/litellm/issues/XXXXX
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
anthropic_messages_pt,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "I need help with time information."
|
||||
},
|
||||
{"role": "user", "content": "I need help with time information."},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
|
|
@ -1569,7 +1598,7 @@ def test_anthropic_messages_pt_server_tool_use_passthrough():
|
|||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_01ABC123",
|
||||
"name": "tool_search_tool_regex",
|
||||
"input": {"query": ".*time.*"}
|
||||
"input": {"query": ".*time.*"},
|
||||
},
|
||||
{
|
||||
"type": "tool_search_tool_result",
|
||||
|
|
@ -1578,19 +1607,13 @@ def test_anthropic_messages_pt_server_tool_use_passthrough():
|
|||
"type": "tool_search_tool_search_result",
|
||||
"tool_references": [
|
||||
{"type": "tool_reference", "tool_name": "get_time"}
|
||||
]
|
||||
}
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "I found the time tool. How can I help you?"
|
||||
}
|
||||
{"type": "text", "text": "I found the time tool. How can I help you?"},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the time in New York?"
|
||||
},
|
||||
{"role": "user", "content": "What's the time in New York?"},
|
||||
]
|
||||
|
||||
result = anthropic_messages_pt(
|
||||
|
|
@ -1622,7 +1645,9 @@ def test_anthropic_messages_pt_server_tool_use_passthrough():
|
|||
# Verify tool_search_tool_result block is preserved
|
||||
assert "tool_search_tool_result" in content_types
|
||||
tool_result_block = next(
|
||||
b for b in assistant_msg["content"] if b.get("type") == "tool_search_tool_result"
|
||||
b
|
||||
for b in assistant_msg["content"]
|
||||
if b.get("type") == "tool_search_tool_result"
|
||||
)
|
||||
assert tool_result_block["tool_use_id"] == "srvtoolu_01ABC123"
|
||||
assert tool_result_block["content"]["type"] == "tool_search_tool_search_result"
|
||||
|
|
@ -1630,9 +1655,7 @@ def test_anthropic_messages_pt_server_tool_use_passthrough():
|
|||
|
||||
# Verify text block is also preserved
|
||||
assert "text" in content_types
|
||||
text_block = next(
|
||||
b for b in assistant_msg["content"] if b.get("type") == "text"
|
||||
)
|
||||
text_block = next(b for b in assistant_msg["content"] if b.get("type") == "text")
|
||||
assert text_block["text"] == "I found the time tool. How can I help you?"
|
||||
|
||||
|
||||
|
|
@ -1663,7 +1686,10 @@ def test_bedrock_tools_unpack_defs_no_oom_with_nested_refs():
|
|||
"Expression": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {"type": "string", "enum": ["and", "or", "not", "comparison"]},
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["and", "or", "not", "comparison"],
|
||||
},
|
||||
"left": {"$ref": "#/$defs/Operand"},
|
||||
"right": {"$ref": "#/$defs/Operand"},
|
||||
"operator": {"$ref": "#/$defs/Operator"},
|
||||
|
|
@ -1674,7 +1700,9 @@ def test_bedrock_tools_unpack_defs_no_oom_with_nested_refs():
|
|||
"anyOf": [
|
||||
{"$ref": "#/$defs/Literal"},
|
||||
{"$ref": "#/$defs/FieldRef"},
|
||||
{"$ref": "#/$defs/Expression"}, # Circular: Operand -> Expression -> Operand
|
||||
{
|
||||
"$ref": "#/$defs/Expression"
|
||||
}, # Circular: Operand -> Expression -> Operand
|
||||
],
|
||||
},
|
||||
"Literal": {
|
||||
|
|
@ -1808,9 +1836,9 @@ def test_anthropic_messages_pt_file_block_preserves_cache_control():
|
|||
|
||||
file_block = content_blocks[0]
|
||||
assert file_block["type"] == "document"
|
||||
assert "cache_control" in file_block, (
|
||||
"cache_control should be preserved on file/document content blocks"
|
||||
)
|
||||
assert (
|
||||
"cache_control" in file_block
|
||||
), "cache_control should be preserved on file/document content blocks"
|
||||
assert file_block["cache_control"]["type"] == "ephemeral"
|
||||
|
||||
text_block = content_blocks[1]
|
||||
|
|
@ -2056,7 +2084,9 @@ def test_sanitize_messages_deduplicates_tool_results():
|
|||
|
||||
# Count tool messages with this ID — should be exactly 1
|
||||
tool_results = [
|
||||
m for m in result if m.get("role") == "tool" and m.get("tool_call_id") == "call_abc123"
|
||||
m
|
||||
for m in result
|
||||
if m.get("role") == "tool" and m.get("tool_call_id") == "call_abc123"
|
||||
]
|
||||
assert len(tool_results) == 1
|
||||
# Should keep the LAST occurrence (most complete)
|
||||
|
|
@ -2193,7 +2223,8 @@ def test_sanitize_messages_dedup_scoped_per_turn_preserves_cross_turn():
|
|||
|
||||
# Both tool results must survive — one per turn
|
||||
tool_results = [
|
||||
m for m in result
|
||||
m
|
||||
for m in result
|
||||
if m.get("role") == "tool" and m.get("tool_call_id") == "call_X"
|
||||
]
|
||||
assert len(tool_results) == 2, (
|
||||
|
|
@ -2252,38 +2283,35 @@ def test_sanitize_messages_combined_case_a_and_case_d():
|
|||
missing_results = [
|
||||
m for m in tool_results if m.get("tool_call_id") == "call_missing"
|
||||
]
|
||||
assert len(missing_results) == 1, (
|
||||
f"Expected 1 dummy result for call_missing (Case A), got {len(missing_results)}"
|
||||
)
|
||||
assert (
|
||||
len(missing_results) == 1
|
||||
), f"Expected 1 dummy result for call_missing (Case A), got {len(missing_results)}"
|
||||
|
||||
# Case D: call_duped should have exactly 1 result (the fresh one)
|
||||
duped_results = [
|
||||
m for m in tool_results if m.get("tool_call_id") == "call_duped"
|
||||
]
|
||||
assert len(duped_results) == 1, (
|
||||
f"Expected 1 result for call_duped after dedup (Case D), got {len(duped_results)}"
|
||||
)
|
||||
assert duped_results[0]["content"] == "fresh_result", (
|
||||
f"Expected last-wins 'fresh_result', got '{duped_results[0]['content']}'"
|
||||
)
|
||||
assert (
|
||||
len(duped_results) == 1
|
||||
), f"Expected 1 result for call_duped after dedup (Case D), got {len(duped_results)}"
|
||||
assert (
|
||||
duped_results[0]["content"] == "fresh_result"
|
||||
), f"Expected last-wins 'fresh_result', got '{duped_results[0]['content']}'"
|
||||
|
||||
# Verify tool results immediately follow the assistant message
|
||||
asst_idx = next(
|
||||
i for i, m in enumerate(result) if m.get("role") == "assistant"
|
||||
)
|
||||
asst_idx = next(i for i, m in enumerate(result) if m.get("role") == "assistant")
|
||||
tool_msgs_after_asst = [
|
||||
m
|
||||
for m in result[asst_idx + 1 :]
|
||||
if m.get("role") in ("tool", "function")
|
||||
m for m in result[asst_idx + 1 :] if m.get("role") in ("tool", "function")
|
||||
]
|
||||
assert len(tool_msgs_after_asst) == 2, (
|
||||
f"Expected 2 tool results after assistant, got {len(tool_msgs_after_asst)}"
|
||||
)
|
||||
assert (
|
||||
len(tool_msgs_after_asst) == 2
|
||||
), f"Expected 2 tool results after assistant, got {len(tool_msgs_after_asst)}"
|
||||
# Both tool_call_ids should be present (order may vary)
|
||||
tool_ids = {m["tool_call_id"] for m in tool_msgs_after_asst}
|
||||
assert tool_ids == {"call_missing", "call_duped"}, (
|
||||
f"Expected tool_call_ids {{call_missing, call_duped}}, got {tool_ids}"
|
||||
)
|
||||
assert tool_ids == {
|
||||
"call_missing",
|
||||
"call_duped",
|
||||
}, f"Expected tool_call_ids {{call_missing, call_duped}}, got {tool_ids}"
|
||||
finally:
|
||||
litellm.modify_params = original
|
||||
|
||||
|
|
@ -2329,9 +2357,9 @@ def test_anthropic_messages_pt_file_block_preserves_cache_control():
|
|||
# Document block (from file) should preserve cache_control
|
||||
doc_block = content_blocks[0]
|
||||
assert doc_block["type"] == "document"
|
||||
assert "cache_control" in doc_block, (
|
||||
"cache_control was dropped from file/document block"
|
||||
)
|
||||
assert (
|
||||
"cache_control" in doc_block
|
||||
), "cache_control was dropped from file/document block"
|
||||
assert doc_block["cache_control"]["type"] == "ephemeral"
|
||||
|
||||
# Text block should also preserve cache_control
|
||||
|
|
@ -2339,3 +2367,112 @@ def test_anthropic_messages_pt_file_block_preserves_cache_control():
|
|||
assert text_block["type"] == "text"
|
||||
assert "cache_control" in text_block
|
||||
assert text_block["cache_control"]["type"] == "ephemeral"
|
||||
|
||||
|
||||
def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5():
|
||||
"""
|
||||
Tools with cache_control ttl should preserve the ttl in the cachePoint
|
||||
block for Claude 4.5+ models on Bedrock, matching the behavior of system
|
||||
block cache_control.
|
||||
|
||||
Without this fix, tool cachePoint is always {"type": "default"} (5m),
|
||||
while system blocks can have ttl="1h", violating Bedrock's non-increasing
|
||||
TTL ordering constraint (tools -> system -> messages).
|
||||
|
||||
Ref: https://github.com/BerriAI/litellm/issues/XXXXX
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
add_cache_point_tool_block,
|
||||
)
|
||||
|
||||
tool_with_1h = {
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "parameters": {"type": "object"}},
|
||||
"cache_control": {"type": "ephemeral", "ttl": "1h"},
|
||||
}
|
||||
|
||||
# Claude 4.5 model: ttl should be preserved
|
||||
result = add_cache_point_tool_block(
|
||||
tool_with_1h, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0"
|
||||
)
|
||||
assert result is not None
|
||||
assert result["cachePoint"]["type"] == "default"
|
||||
assert result["cachePoint"]["ttl"] == "1h"
|
||||
|
||||
# Claude 4.5 model with 5m ttl: also preserved
|
||||
tool_with_5m = {
|
||||
"cache_control": {"type": "ephemeral", "ttl": "5m"},
|
||||
}
|
||||
result_5m = add_cache_point_tool_block(
|
||||
tool_with_5m, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0"
|
||||
)
|
||||
assert result_5m is not None
|
||||
assert result_5m["cachePoint"]["ttl"] == "5m"
|
||||
|
||||
# Older model: ttl should be stripped
|
||||
result_old = add_cache_point_tool_block(
|
||||
tool_with_1h, model="anthropic.claude-3-5-sonnet-20241022-v2:0"
|
||||
)
|
||||
assert result_old is not None
|
||||
assert result_old["cachePoint"]["type"] == "default"
|
||||
assert "ttl" not in result_old["cachePoint"]
|
||||
|
||||
# No model provided: ttl should be stripped (safe default)
|
||||
result_no_model = add_cache_point_tool_block(tool_with_1h, model=None)
|
||||
assert result_no_model is not None
|
||||
assert "ttl" not in result_no_model["cachePoint"]
|
||||
|
||||
# No cache_control: returns None (unchanged behavior)
|
||||
tool_no_cache = {
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "parameters": {"type": "object"}},
|
||||
}
|
||||
assert add_cache_point_tool_block(tool_no_cache) is None
|
||||
|
||||
# cache_control without ttl: returns default cachePoint (unchanged behavior)
|
||||
tool_no_ttl = {"cache_control": {"type": "ephemeral"}}
|
||||
result_no_ttl = add_cache_point_tool_block(
|
||||
tool_no_ttl, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0"
|
||||
)
|
||||
assert result_no_ttl is not None
|
||||
assert result_no_ttl["cachePoint"]["type"] == "default"
|
||||
assert "ttl" not in result_no_ttl["cachePoint"]
|
||||
|
||||
|
||||
def test_bedrock_tools_pt_passes_ttl_for_claude_4_5():
|
||||
"""
|
||||
End-to-end: _bedrock_tools_pt should produce cachePoint blocks with ttl
|
||||
for Claude 4.5+ models when tools have cache_control with ttl.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_tools_pt
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
"cache_control": {"type": "ephemeral", "ttl": "1h"},
|
||||
}
|
||||
]
|
||||
|
||||
# Claude 4.5: cachePoint should have ttl
|
||||
result = _bedrock_tools_pt(
|
||||
tools, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0"
|
||||
)
|
||||
cache_blocks = [b for b in result if "cachePoint" in b]
|
||||
assert len(cache_blocks) == 1
|
||||
assert cache_blocks[0]["cachePoint"]["ttl"] == "1h"
|
||||
|
||||
# Older model: cachePoint should not have ttl
|
||||
result_old = _bedrock_tools_pt(
|
||||
tools, model="anthropic.claude-3-5-sonnet-20241022-v2:0"
|
||||
)
|
||||
cache_blocks_old = [b for b in result_old if "cachePoint" in b]
|
||||
assert len(cache_blocks_old) == 1
|
||||
assert "ttl" not in cache_blocks_old[0]["cachePoint"]
|
||||
|
|
|
|||
|
|
@ -315,7 +315,10 @@ def test_normalize_tool_input_schema_types_for_bedrock_invoke():
|
|||
"type": "custom",
|
||||
"additionalProperties": False,
|
||||
"properties": {
|
||||
"nested": {"type": "custom", "properties": {"x": {"type": "string"}}}
|
||||
"nested": {
|
||||
"type": "custom",
|
||||
"properties": {"x": {"type": "string"}},
|
||||
}
|
||||
},
|
||||
"required": ["nested"],
|
||||
},
|
||||
|
|
@ -436,6 +439,86 @@ def test_bedrock_invoke_messages_transform_converts_custom_tool_schema_type_to_o
|
|||
assert result["tools"][0]["type"] == "custom"
|
||||
|
||||
|
||||
def test_remove_ttl_from_cache_control_processes_tools():
|
||||
"""
|
||||
Ensure _remove_ttl_from_cache_control also sanitizes cache_control on tools.
|
||||
|
||||
Without this, tools keep unsupported ttl values while system/messages have
|
||||
them stripped, causing TTL ordering violations on Bedrock.
|
||||
"""
|
||||
|
||||
cfg = AmazonAnthropicClaudeMessagesConfig()
|
||||
|
||||
# Tools with ttl should have it stripped for non-Claude-4.5 models
|
||||
request = {
|
||||
"tools": [
|
||||
{
|
||||
"name": "get_weather",
|
||||
"input_schema": {"type": "object"},
|
||||
"cache_control": {"type": "ephemeral", "ttl": "1h"},
|
||||
},
|
||||
{
|
||||
"name": "get_time",
|
||||
"input_schema": {"type": "object"},
|
||||
},
|
||||
],
|
||||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are helpful.",
|
||||
"cache_control": {"type": "ephemeral", "ttl": "1h"},
|
||||
}
|
||||
],
|
||||
"messages": [],
|
||||
}
|
||||
|
||||
cfg._remove_ttl_from_cache_control(
|
||||
request, model="anthropic.claude-3-5-sonnet-20241022-v2:0"
|
||||
)
|
||||
|
||||
# Tool ttl should be stripped
|
||||
assert "ttl" not in request["tools"][0]["cache_control"]
|
||||
assert request["tools"][0]["cache_control"]["type"] == "ephemeral"
|
||||
# Tool without cache_control should be unchanged
|
||||
assert "cache_control" not in request["tools"][1]
|
||||
# System ttl should also be stripped
|
||||
assert "ttl" not in request["system"][0]["cache_control"]
|
||||
|
||||
|
||||
def test_remove_ttl_from_cache_control_preserves_tools_ttl_for_claude_4_5():
|
||||
"""
|
||||
For Claude 4.5+ models, ttl in ["5m", "1h"] should be preserved on tools,
|
||||
just like it is for system and messages.
|
||||
"""
|
||||
|
||||
cfg = AmazonAnthropicClaudeMessagesConfig()
|
||||
|
||||
request = {
|
||||
"tools": [
|
||||
{
|
||||
"name": "get_weather",
|
||||
"input_schema": {"type": "object"},
|
||||
"cache_control": {"type": "ephemeral", "ttl": "1h"},
|
||||
},
|
||||
],
|
||||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are helpful.",
|
||||
"cache_control": {"type": "ephemeral", "ttl": "1h"},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
cfg._remove_ttl_from_cache_control(
|
||||
request, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0"
|
||||
)
|
||||
|
||||
# Both tools and system should preserve ttl for Claude 4.5
|
||||
assert request["tools"][0]["cache_control"]["ttl"] == "1h"
|
||||
assert request["system"][0]["cache_control"]["ttl"] == "1h"
|
||||
|
||||
|
||||
def test_remove_scope_from_cache_control():
|
||||
"""Ensure scope field is removed from cache_control for Bedrock (not supported)."""
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue