From 8a3e8504306c69d05640f7e545f4fe55bd81b429 Mon Sep 17 00:00:00 2001 From: Rodrigo Diaz Leven Date: Sun, 5 Apr 2026 20:43:54 +0200 Subject: [PATCH] fix: restore cachePoint in assistant messages, add tool_use warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live Bedrock testing confirmed that cachePoint IS valid in assistant messages (both after text and after toolUse blocks). The actual rule is: cachePoint must not appear BEFORE a toolResult in a user message. Changes: - Restore cachePoint generation in assistant content (text and toolUse) for both sync and async paths - Restore cachePoint in _convert_to_bedrock_tool_call_invoke - Keep cachePoint skipped only for tool_result/tool_use elements in user messages (where it would land before toolResult) - Add verbose_logger.warning when tool_use appears in user message content (wrong role — silently dropped with explanation) - Remove unused FastAPI TestClient import from test file - Restore original test assertions for assistant cachePoint All 96 tests pass. Made-with: Cursor --- .../prompt_templates/factory.py | 74 ++++++++++++++----- .../chat/test_converse_transformation.py | 26 +++---- 2 files changed, 66 insertions(+), 34 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index e4e11f012f6..d2c1bb229da 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -3881,8 +3881,6 @@ def _convert_to_bedrock_tool_call_invoke( _parts_list.append( BedrockContentBlock(toolUse=bedrock_tool) ) - # cache_control applies to the whole original - # tool call; attach after the last split block. if tool.get("cache_control", None) is not None: _parts_list.append( BedrockContentBlock( @@ -3899,7 +3897,6 @@ def _convert_to_bedrock_tool_call_invoke( bedrock_content_block = BedrockContentBlock(toolUse=bedrock_tool) _parts_list.append(bedrock_content_block) - # Check for cache_control and add a separate cachePoint block if tool.get("cache_control", None) is not None: cache_point_block = BedrockContentBlock( cachePoint=CachePointBlock(type="default") @@ -4459,6 +4456,13 @@ class BedrockConverseMessagesProcessor: toolUseId=tool_use_id, ) _parts.append(BedrockContentBlock(toolResult=bedrock_tool_result)) + else: + verbose_logger.warning( + "Unexpected tool_use block in user message content — " + "tool_use belongs in assistant messages. Dropping block." + ) + # Skip cachePoint — Bedrock rejects cachePoint placed + # before toolResult ("nothing available to cache"). continue _cache_point_block = ( litellm.AmazonConverseConfig()._get_cache_point_block( @@ -4637,6 +4641,20 @@ class BedrockConverseMessagesProcessor: assistants_parts.append( BedrockContentBlock(toolUse=bedrock_tool_use) ) + # Skip cachePoint for tool_use — a cachePoint here + # would land before the user's toolResult, which + # Bedrock rejects with "nothing available to cache". + continue + _cache_point_block = ( + litellm.AmazonConverseConfig()._get_cache_point_block( + message_block=cast( + OpenAIMessageContentListBlock, element + ), + block_type="content_block", + ) + ) + if _cache_point_block is not None: + assistants_parts.append(_cache_point_block) assistant_content.extend(assistants_parts) elif _assistant_content is not None and isinstance( _assistant_content, str @@ -4646,6 +4664,13 @@ class BedrockConverseMessagesProcessor: assistant_content.append( BedrockContentBlock(text=_assistant_content) ) + _cache_point_block = ( + litellm.AmazonConverseConfig()._get_cache_point_block( + assistant_message_block, block_type="content_block" + ) + ) + if _cache_point_block is not None: + assistant_content.append(_cache_point_block) _tool_calls = assistant_message_block.get("tool_calls", []) if _tool_calls: @@ -4832,11 +4857,6 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 ) _parts.append(_part) elif element["type"] in ("tool_result", "tool_use"): - # Anthropic-format tool results/uses embedded in user messages - # (e.g. sent by Cursor IDE after a tool call). Convert the - # tool_result to a Bedrock toolResult block. Skip the - # cachePoint — Bedrock rejects cachePoint blocks placed - # immediately after toolResult content with no preceding text. if element["type"] == "tool_result": tool_use_id = element.get("tool_use_id", str(uuid.uuid4())) inner_content = element.get("content", "") @@ -4856,8 +4876,13 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 toolUseId=tool_use_id, ) _parts.append(BedrockContentBlock(toolResult=bedrock_tool_result)) - # Skip cachePoint for tool_result/tool_use — add it at - # message level (after all content) if needed, handled below. + else: + verbose_logger.warning( + "Unexpected tool_use block in user message content — " + "tool_use belongs in assistant messages. Dropping block." + ) + # Skip cachePoint — Bedrock rejects cachePoint placed + # before toolResult ("nothing available to cache"). continue _cache_point_block = ( litellm.AmazonConverseConfig()._get_cache_point_block( @@ -5017,8 +5042,7 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 elif element["type"] == "tool_use": # Anthropic-format tool invocation embedded in assistant # content list (e.g. from Cursor IDE). Convert to Bedrock - # toolUse ContentBlock directly. Skip cachePoint — Bedrock - # rejects cachePoint immediately after toolUse blocks. + # toolUse ContentBlock directly. tool_input = element.get("input", {}) if isinstance(tool_input, str): try: @@ -5033,11 +5057,20 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 assistants_parts.append( BedrockContentBlock(toolUse=bedrock_tool_use) ) - continue # skip _get_cache_point_block for tool_use - # Anthropic prompt caching only supports cachePoint blocks in - # system, tools, and USER messages — NOT in assistant messages. - # Bedrock rejects cachePoint in assistant content with - # "There is nothing available to cache." Skip it here. + # Skip cachePoint for tool_use — a cachePoint here + # would land before the user's toolResult, which + # Bedrock rejects with "nothing available to cache". + continue + _cache_point_block = ( + litellm.AmazonConverseConfig()._get_cache_point_block( + message_block=cast( + OpenAIMessageContentListBlock, element + ), + block_type="content_block", + ) + ) + if _cache_point_block is not None: + assistants_parts.append(_cache_point_block) assistant_content.extend(assistants_parts) elif _assistant_content is not None and isinstance(_assistant_content, str): # Skip completely empty strings to avoid blank content blocks @@ -5045,6 +5078,13 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 assistant_content.append( BedrockContentBlock(text=_assistant_content) ) + _cache_point_block = ( + litellm.AmazonConverseConfig()._get_cache_point_block( + assistant_message_block, block_type="content_block" + ) + ) + if _cache_point_block is not None: + assistant_content.append(_cache_point_block) _tool_calls = assistant_message_block.get("tool_calls", []) if _tool_calls: assistant_content.extend( diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 546ec0cf9db..7806ce05bce 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -4,7 +4,6 @@ import os import sys import pytest -from fastapi.testclient import TestClient sys.path.insert(0, os.path.abspath("../../../../..")) # Adds the parent directory to the system path from unittest.mock import MagicMock, patch @@ -1282,21 +1281,17 @@ async def test_assistant_message_cache_control(): assert result[0]["role"] == "user" assert result[1]["role"] == "assistant" - # Bedrock rejects cachePoint in assistant messages ("There is nothing - # available to cache"), so assistant content must NOT contain cachePoint. + # Assistant message should have text content and cachePoint assistant_content = result[1]["content"] - assert len(assistant_content) == 1 + assert len(assistant_content) == 2 assert assistant_content[0]["text"] == "Hi there!" + assert "cachePoint" in assistant_content[1] + assert assistant_content[1]["cachePoint"]["type"] == "default" @pytest.mark.asyncio async def test_assistant_message_list_content_cache_control(): - """Test that cache_control on assistant list content does NOT produce cachePoint blocks. - - Bedrock rejects cachePoint in assistant messages with 'There is nothing - available to cache'. Prompt caching only supports cache points in system, - tools, and user messages. - """ + """Test assistant messages with list content and cache_control.""" from litellm.litellm_core_utils.prompt_templates.factory import ( BedrockConverseMessagesProcessor, _bedrock_converse_messages_pt, @@ -1320,10 +1315,12 @@ async def test_assistant_message_list_content_cache_control(): assert result == async_result - # No cachePoint in assistant content + # Assistant message should have text content and cachePoint assistant_content = result[1]["content"] - assert len(assistant_content) == 1 + assert len(assistant_content) == 2 assert assistant_content[0]["text"] == "This should be cached" + assert "cachePoint" in assistant_content[1] + assert assistant_content[1]["cachePoint"]["type"] == "default" @pytest.mark.asyncio @@ -1462,12 +1459,10 @@ async def test_assistant_tool_calls_cache_control(): assistant_content = result[1]["content"] assert len(assistant_content) == 2 - # First should be tool use assert "toolUse" in assistant_content[0] assert assistant_content[0]["toolUse"]["name"] == "calc" assert assistant_content[0]["toolUse"]["toolUseId"] == "call_proxy_123" - # Second should be cachePoint assert "cachePoint" in assistant_content[1] assert assistant_content[1]["cachePoint"]["type"] == "default" @@ -1516,15 +1511,12 @@ async def test_multiple_tool_calls_with_mixed_cache_control(): assistant_content = result[1]["content"] assert len(assistant_content) == 3 - # First tool use with cache assert "toolUse" in assistant_content[0] assert assistant_content[0]["toolUse"]["toolUseId"] == "call_1" - # Cache point for first tool assert "cachePoint" in assistant_content[1] assert assistant_content[1]["cachePoint"]["type"] == "default" - # Second tool use without cache assert "toolUse" in assistant_content[2] assert assistant_content[2]["toolUse"]["toolUseId"] == "call_2"