mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-06 08:16:43 +00:00
fix: restore cachePoint in assistant messages, add tool_use warning
Live Bedrock testing confirmed that cachePoint IS valid in assistant messages (both after text and after toolUse blocks). The actual rule is: cachePoint must not appear BEFORE a toolResult in a user message. Changes: - Restore cachePoint generation in assistant content (text and toolUse) for both sync and async paths - Restore cachePoint in _convert_to_bedrock_tool_call_invoke - Keep cachePoint skipped only for tool_result/tool_use elements in user messages (where it would land before toolResult) - Add verbose_logger.warning when tool_use appears in user message content (wrong role — silently dropped with explanation) - Remove unused FastAPI TestClient import from test file - Restore original test assertions for assistant cachePoint All 96 tests pass. Made-with: Cursor
This commit is contained in:
Rodrigo Diaz Leven
parent
666a239501
commit
8a3e850430
2 changed files with 66 additions and 34 deletions
|
|
@ -3881,8 +3881,6 @@ def _convert_to_bedrock_tool_call_invoke(
|
|||
_parts_list.append(
|
||||
BedrockContentBlock(toolUse=bedrock_tool)
|
||||
)
|
||||
# cache_control applies to the whole original
|
||||
# tool call; attach after the last split block.
|
||||
if tool.get("cache_control", None) is not None:
|
||||
_parts_list.append(
|
||||
BedrockContentBlock(
|
||||
|
|
@ -3899,7 +3897,6 @@ def _convert_to_bedrock_tool_call_invoke(
|
|||
bedrock_content_block = BedrockContentBlock(toolUse=bedrock_tool)
|
||||
_parts_list.append(bedrock_content_block)
|
||||
|
||||
# Check for cache_control and add a separate cachePoint block
|
||||
if tool.get("cache_control", None) is not None:
|
||||
cache_point_block = BedrockContentBlock(
|
||||
cachePoint=CachePointBlock(type="default")
|
||||
|
|
@ -4459,6 +4456,13 @@ class BedrockConverseMessagesProcessor:
|
|||
toolUseId=tool_use_id,
|
||||
)
|
||||
_parts.append(BedrockContentBlock(toolResult=bedrock_tool_result))
|
||||
else:
|
||||
verbose_logger.warning(
|
||||
"Unexpected tool_use block in user message content — "
|
||||
"tool_use belongs in assistant messages. Dropping block."
|
||||
)
|
||||
# Skip cachePoint — Bedrock rejects cachePoint placed
|
||||
# before toolResult ("nothing available to cache").
|
||||
continue
|
||||
_cache_point_block = (
|
||||
litellm.AmazonConverseConfig()._get_cache_point_block(
|
||||
|
|
@ -4637,6 +4641,20 @@ class BedrockConverseMessagesProcessor:
|
|||
assistants_parts.append(
|
||||
BedrockContentBlock(toolUse=bedrock_tool_use)
|
||||
)
|
||||
# Skip cachePoint for tool_use — a cachePoint here
|
||||
# would land before the user's toolResult, which
|
||||
# Bedrock rejects with "nothing available to cache".
|
||||
continue
|
||||
_cache_point_block = (
|
||||
litellm.AmazonConverseConfig()._get_cache_point_block(
|
||||
message_block=cast(
|
||||
OpenAIMessageContentListBlock, element
|
||||
),
|
||||
block_type="content_block",
|
||||
)
|
||||
)
|
||||
if _cache_point_block is not None:
|
||||
assistants_parts.append(_cache_point_block)
|
||||
assistant_content.extend(assistants_parts)
|
||||
elif _assistant_content is not None and isinstance(
|
||||
_assistant_content, str
|
||||
|
|
@ -4646,6 +4664,13 @@ class BedrockConverseMessagesProcessor:
|
|||
assistant_content.append(
|
||||
BedrockContentBlock(text=_assistant_content)
|
||||
)
|
||||
_cache_point_block = (
|
||||
litellm.AmazonConverseConfig()._get_cache_point_block(
|
||||
assistant_message_block, block_type="content_block"
|
||||
)
|
||||
)
|
||||
if _cache_point_block is not None:
|
||||
assistant_content.append(_cache_point_block)
|
||||
|
||||
_tool_calls = assistant_message_block.get("tool_calls", [])
|
||||
if _tool_calls:
|
||||
|
|
@ -4832,11 +4857,6 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
|
|||
)
|
||||
_parts.append(_part)
|
||||
elif element["type"] in ("tool_result", "tool_use"):
|
||||
# Anthropic-format tool results/uses embedded in user messages
|
||||
# (e.g. sent by Cursor IDE after a tool call). Convert the
|
||||
# tool_result to a Bedrock toolResult block. Skip the
|
||||
# cachePoint — Bedrock rejects cachePoint blocks placed
|
||||
# immediately after toolResult content with no preceding text.
|
||||
if element["type"] == "tool_result":
|
||||
tool_use_id = element.get("tool_use_id", str(uuid.uuid4()))
|
||||
inner_content = element.get("content", "")
|
||||
|
|
@ -4856,8 +4876,13 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
|
|||
toolUseId=tool_use_id,
|
||||
)
|
||||
_parts.append(BedrockContentBlock(toolResult=bedrock_tool_result))
|
||||
# Skip cachePoint for tool_result/tool_use — add it at
|
||||
# message level (after all content) if needed, handled below.
|
||||
else:
|
||||
verbose_logger.warning(
|
||||
"Unexpected tool_use block in user message content — "
|
||||
"tool_use belongs in assistant messages. Dropping block."
|
||||
)
|
||||
# Skip cachePoint — Bedrock rejects cachePoint placed
|
||||
# before toolResult ("nothing available to cache").
|
||||
continue
|
||||
_cache_point_block = (
|
||||
litellm.AmazonConverseConfig()._get_cache_point_block(
|
||||
|
|
@ -5017,8 +5042,7 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
|
|||
elif element["type"] == "tool_use":
|
||||
# Anthropic-format tool invocation embedded in assistant
|
||||
# content list (e.g. from Cursor IDE). Convert to Bedrock
|
||||
# toolUse ContentBlock directly. Skip cachePoint — Bedrock
|
||||
# rejects cachePoint immediately after toolUse blocks.
|
||||
# toolUse ContentBlock directly.
|
||||
tool_input = element.get("input", {})
|
||||
if isinstance(tool_input, str):
|
||||
try:
|
||||
|
|
@ -5033,11 +5057,20 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
|
|||
assistants_parts.append(
|
||||
BedrockContentBlock(toolUse=bedrock_tool_use)
|
||||
)
|
||||
continue # skip _get_cache_point_block for tool_use
|
||||
# Anthropic prompt caching only supports cachePoint blocks in
|
||||
# system, tools, and USER messages — NOT in assistant messages.
|
||||
# Bedrock rejects cachePoint in assistant content with
|
||||
# "There is nothing available to cache." Skip it here.
|
||||
# Skip cachePoint for tool_use — a cachePoint here
|
||||
# would land before the user's toolResult, which
|
||||
# Bedrock rejects with "nothing available to cache".
|
||||
continue
|
||||
_cache_point_block = (
|
||||
litellm.AmazonConverseConfig()._get_cache_point_block(
|
||||
message_block=cast(
|
||||
OpenAIMessageContentListBlock, element
|
||||
),
|
||||
block_type="content_block",
|
||||
)
|
||||
)
|
||||
if _cache_point_block is not None:
|
||||
assistants_parts.append(_cache_point_block)
|
||||
assistant_content.extend(assistants_parts)
|
||||
elif _assistant_content is not None and isinstance(_assistant_content, str):
|
||||
# Skip completely empty strings to avoid blank content blocks
|
||||
|
|
@ -5045,6 +5078,13 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
|
|||
assistant_content.append(
|
||||
BedrockContentBlock(text=_assistant_content)
|
||||
)
|
||||
_cache_point_block = (
|
||||
litellm.AmazonConverseConfig()._get_cache_point_block(
|
||||
assistant_message_block, block_type="content_block"
|
||||
)
|
||||
)
|
||||
if _cache_point_block is not None:
|
||||
assistant_content.append(_cache_point_block)
|
||||
_tool_calls = assistant_message_block.get("tool_calls", [])
|
||||
if _tool_calls:
|
||||
assistant_content.extend(
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import os
|
|||
import sys
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../../../../..")) # Adds the parent directory to the system path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
|
@ -1282,21 +1281,17 @@ async def test_assistant_message_cache_control():
|
|||
assert result[0]["role"] == "user"
|
||||
assert result[1]["role"] == "assistant"
|
||||
|
||||
# Bedrock rejects cachePoint in assistant messages ("There is nothing
|
||||
# available to cache"), so assistant content must NOT contain cachePoint.
|
||||
# Assistant message should have text content and cachePoint
|
||||
assistant_content = result[1]["content"]
|
||||
assert len(assistant_content) == 1
|
||||
assert len(assistant_content) == 2
|
||||
assert assistant_content[0]["text"] == "Hi there!"
|
||||
assert "cachePoint" in assistant_content[1]
|
||||
assert assistant_content[1]["cachePoint"]["type"] == "default"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assistant_message_list_content_cache_control():
|
||||
"""Test that cache_control on assistant list content does NOT produce cachePoint blocks.
|
||||
|
||||
Bedrock rejects cachePoint in assistant messages with 'There is nothing
|
||||
available to cache'. Prompt caching only supports cache points in system,
|
||||
tools, and user messages.
|
||||
"""
|
||||
"""Test assistant messages with list content and cache_control."""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
BedrockConverseMessagesProcessor,
|
||||
_bedrock_converse_messages_pt,
|
||||
|
|
@ -1320,10 +1315,12 @@ async def test_assistant_message_list_content_cache_control():
|
|||
|
||||
assert result == async_result
|
||||
|
||||
# No cachePoint in assistant content
|
||||
# Assistant message should have text content and cachePoint
|
||||
assistant_content = result[1]["content"]
|
||||
assert len(assistant_content) == 1
|
||||
assert len(assistant_content) == 2
|
||||
assert assistant_content[0]["text"] == "This should be cached"
|
||||
assert "cachePoint" in assistant_content[1]
|
||||
assert assistant_content[1]["cachePoint"]["type"] == "default"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1462,12 +1459,10 @@ async def test_assistant_tool_calls_cache_control():
|
|||
assistant_content = result[1]["content"]
|
||||
assert len(assistant_content) == 2
|
||||
|
||||
# First should be tool use
|
||||
assert "toolUse" in assistant_content[0]
|
||||
assert assistant_content[0]["toolUse"]["name"] == "calc"
|
||||
assert assistant_content[0]["toolUse"]["toolUseId"] == "call_proxy_123"
|
||||
|
||||
# Second should be cachePoint
|
||||
assert "cachePoint" in assistant_content[1]
|
||||
assert assistant_content[1]["cachePoint"]["type"] == "default"
|
||||
|
||||
|
|
@ -1516,15 +1511,12 @@ async def test_multiple_tool_calls_with_mixed_cache_control():
|
|||
assistant_content = result[1]["content"]
|
||||
assert len(assistant_content) == 3
|
||||
|
||||
# First tool use with cache
|
||||
assert "toolUse" in assistant_content[0]
|
||||
assert assistant_content[0]["toolUse"]["toolUseId"] == "call_1"
|
||||
|
||||
# Cache point for first tool
|
||||
assert "cachePoint" in assistant_content[1]
|
||||
assert assistant_content[1]["cachePoint"]["type"] == "default"
|
||||
|
||||
# Second tool use without cache
|
||||
assert "toolUse" in assistant_content[2]
|
||||
assert assistant_content[2]["toolUse"]["toolUseId"] == "call_2"
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue