fix(bedrock): reserve only the tool cache points Converse sends when trimming Claude cache marks under drop_params

litellm/integrations/anthropic_cache_control_hook.py

@@ -1982,9 +1982,17 @@ def messages_within_cache_control_cap(
 
     Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
     """
-    reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
-        _validated_object_list(tools), cache_control, request_kwargs
+    return messages_within_reserved_cache_control_cap(
+        messages,
+        AnthropicCacheControlHook.count_external_cache_breakpoints(
+            _validated_object_list(tools), cache_control, request_kwargs
+        ),
     )
+
+
+def messages_within_reserved_cache_control_cap(
+    messages: Sequence[_MessageT], reserved_marks: int
+) -> Sequence[_MessageT]:
     kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
     dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
     if dropped > 0:

litellm/llms/bedrock/chat/converse_transformation.py

@@ -2241,22 +2241,27 @@ class AmazonConverseConfig(BaseConfig):
 
         return data
 
-    @staticmethod
     def _claude_messages_within_cache_control_cap(
+        self,
         messages: Sequence[AllMessageValues],
         capability_model: str,
+        deployment_model: str,
         optional_params: Mapping[str, object],
         litellm_params: Mapping[str, object],
+        model_info: Mapping[str, object] | None,
     ) -> Sequence[AllMessageValues]:
         if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower():
             return messages
         from litellm.integrations.anthropic_cache_control_hook import (
-            messages_within_cache_control_cap,
+            messages_within_reserved_cache_control_cap,
             tool_message_with_single_cache_mark,
         )
 
-        return messages_within_cache_control_cap(
-            tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools")
+        return messages_within_reserved_cache_control_cap(
+            tuple(map(tool_message_with_single_cache_mark, messages)),
+            self._tool_cache_points_on_the_wire(
+                capability_model, deployment_model, optional_params, litellm_params, model_info
+            ),
         )
 
     async def _async_transform_request(
@@ -2285,7 +2290,7 @@ class AmazonConverseConfig(BaseConfig):
         messages, system_content_blocks = self._transform_system_message(
             list(
                 self._claude_messages_within_cache_control_cap(
-                    marked_messages, capability_model, json_optional_params, litellm_params
+                    marked_messages, capability_model, model, json_optional_params, litellm_params, model_info
                 )
             ),
             model=capability_model,
@@ -2399,7 +2404,7 @@ class AmazonConverseConfig(BaseConfig):
         messages, system_content_blocks = self._transform_system_message(
             list(
                 self._claude_messages_within_cache_control_cap(
-                    marked_messages, capability_model, json_optional_params, litellm_params
+                    marked_messages, capability_model, model, json_optional_params, litellm_params, model_info
                 )
             ),
             model=capability_model,

tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py

@@ -4434,7 +4434,7 @@ def _anthropic_body(messages: list, litellm_params: dict | None = None, tools: l
 
 
 def test_transform_request_carries_prompt_cache_breakpoint_to_cache_control():
-    body = _anthropic_body(
+    body: Final = _anthropic_body(
         [
             {
                 "role": "system",
@@ -4477,28 +4477,28 @@ def test_transform_request_carries_prompt_cache_breakpoint_to_cache_control():
     ],
 )
 def test_transform_request_prompt_cache_breakpoint_without_claude_equivalent(marker, expected_block):
-    messages = [{"role": "user", "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": marker}]}]
+    messages: Final = [{"role": "user", "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": marker}]}]
 
     with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"):
         _anthropic_body(copy.deepcopy(messages))
 
-    body = _anthropic_body(messages, litellm_params={"drop_params": True})
+    body: Final = _anthropic_body(messages, litellm_params={"drop_params": True})
     assert body["messages"] == [{"role": "user", "content": [expected_block]}]
 
 
 def test_transform_request_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_cap():
-    client_marked_part = {"type": "text", "text": "part 0", "cache_control": {"type": "ephemeral"}}
-    marked_parts = [
+    client_marked_part: Final = {"type": "text", "text": "part 0", "cache_control": {"type": "ephemeral"}}
+    marked_parts: Final = [
         {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
         for index in range(1, 5)
     ]
-    messages = [{"role": "user", "content": [client_marked_part, *marked_parts]}]
-    tools = [{"name": "lookup", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}}]
+    messages: Final = [{"role": "user", "content": [client_marked_part, *marked_parts]}]
+    tools: Final = [{"name": "lookup", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}}]
 
     with pytest.raises(litellm.utils.UnsupportedParamsError, match="at most 4 cache breakpoints"):
         _anthropic_body(copy.deepcopy(messages), tools=copy.deepcopy(tools))
 
-    body = _anthropic_body(messages, litellm_params={"drop_params": True}, tools=tools)
+    body: Final = _anthropic_body(messages, litellm_params={"drop_params": True}, tools=tools)
     assert [part.get("cache_control") for part in body["messages"][0]["content"]] == [
         {"type": "ephemeral"},
         {"type": "ephemeral"},
@@ -4510,8 +4510,8 @@ def test_transform_request_keeps_the_first_and_last_prompt_cache_breakpoints_wit
 
 
 def test_transform_request_counts_a_cache_marked_tool_search_tool_toward_the_cap():
-    config = AnthropicConfig()
-    optional_params = config.map_openai_params(
+    config: Final = AnthropicConfig()
+    optional_params: Final = config.map_openai_params(
         non_default_params={
             "max_tokens": 100,
             "tools": [
@@ -4526,7 +4526,7 @@ def test_transform_request_counts_a_cache_marked_tool_search_tool_toward_the_cap
         model="claude-sonnet-5-5",
         drop_params=False,
     )
-    marked_turns = [
+    marked_turns: Final = [
         {
             "role": "user",
             "content": [{"type": "text", "text": f"turn {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}],
@@ -4546,9 +4546,9 @@ def test_transform_request_counts_a_cache_marked_tool_search_tool_toward_the_cap
     with pytest.raises(litellm.utils.UnsupportedParamsError, match="this request has 5"):
         body({})
 
-    capped = body({"drop_params": True})
+    capped: Final = body({"drop_params": True})
     assert capped["tools"][0]["cache_control"] == {"type": "ephemeral"}
-    sent_parts = itertools.chain.from_iterable(message["content"] for message in capped["messages"])
+    sent_parts: Final = itertools.chain.from_iterable(message["content"] for message in capped["messages"])
     assert [part.get("cache_control") for part in sent_parts] == [
         {"type": "ephemeral"},
         {"type": "ephemeral"},
@@ -4576,15 +4576,15 @@ def test_transform_request_counts_a_cache_marked_tool_search_tool_toward_the_cap
     ],
 )
 def test_transform_request_caps_only_the_marks_claude_receives(unsent_marks: list[dict]):
-    marked_parts = [
+    marked_parts: Final = [
         {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} for index in range(4)
     ]
 
-    body = _anthropic_body(
+    body: Final = _anthropic_body(
         [{"role": "user", "content": marked_parts}, *unsent_marks, {"role": "user", "content": "Go on."}]
     )
 
-    sent_marks = [
+    sent_marks: Final = [
         part.get("cache_control")
         for part in itertools.chain.from_iterable(message["content"] for message in body["messages"])
         if isinstance(part, dict)

tests/unit/llms/bedrock/chat/test_converse_transformation.py

@@ -10074,7 +10074,7 @@ def test_bedrock_prompt_caching_message(messages, expected_cache_control):
 
 
 def test_converse_carries_prompt_cache_breakpoint_to_cache_point():
-    request = litellm.AmazonConverseConfig()._transform_request(
+    request: Final = litellm.AmazonConverseConfig()._transform_request(
         model="anthropic.claude-sonnet-5-5",
         messages=[
             {
@@ -10099,7 +10099,7 @@ def test_converse_carries_prompt_cache_breakpoint_to_cache_point():
 
 
 def test_converse_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_cap():
-    messages = [
+    messages: Final = [
         {
             "role": "user",
             "content": [
@@ -10120,7 +10120,7 @@ def test_converse_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_c
     with pytest.raises(litellm.utils.UnsupportedParamsError, match="at most 4 cache breakpoints"):
         transform({})
 
-    cache_point = {"cachePoint": {"type": "default"}}
+    cache_point: Final = {"cachePoint": {"type": "default"}}
     assert transform({"drop_params": True})["messages"] == [
         {
             "role": "user",
@@ -10157,7 +10157,7 @@ _CONVERSE_CAP_TOOL: Final[dict[str, object]] = {
     ids=["client_tool_cache_control", "tool_config_injection_point"],
 )
 def test_converse_reserves_the_tool_cache_point_within_the_cap(tool_cache_params: dict[str, object]):
-    messages = [
+    messages: Final = [
         {
             "role": "user",
             "content": [
@@ -10178,8 +10178,8 @@ def test_converse_reserves_the_tool_cache_point_within_the_cap(tool_cache_params
     with pytest.raises(litellm.utils.UnsupportedParamsError, match="this request has 5"):
         transform({})
 
-    request = transform({"drop_params": True})
-    cache_point = {"cachePoint": {"type": "default"}}
+    request: Final = transform({"drop_params": True})
+    cache_point: Final = {"cachePoint": {"type": "default"}}
     assert request["messages"][0]["content"] == [
         {"text": "part 0"},
         cache_point,
@@ -10210,8 +10210,11 @@ def test_converse_reserves_the_tool_cache_point_within_the_cap(tool_cache_params
     ],
     ids=["tool_search_tool", "built_in_tool", "tool_config_point_with_no_tool_sent"],
 )
-def test_converse_leaves_the_cap_to_markers_when_a_marked_tool_is_not_sent(unsent_tool_params: dict[str, object]):
-    request = litellm.AmazonConverseConfig()._transform_request(
+@pytest.mark.parametrize("litellm_params", [{}, {"drop_params": True}], ids=["strict", "drop_params"])
+def test_converse_leaves_the_cap_to_markers_when_a_marked_tool_is_not_sent(
+    unsent_tool_params: dict[str, object], litellm_params: dict[str, object]
+):
+    request: Final = litellm.AmazonConverseConfig()._transform_request(
         model="anthropic.claude-sonnet-5-5",
         messages=[
             {
@@ -10223,10 +10226,10 @@ def test_converse_leaves_the_cap_to_markers_when_a_marked_tool_is_not_sent(unsen
             }
         ],
         optional_params=copy.deepcopy(unsent_tool_params),
-        litellm_params={},
+        litellm_params=dict(litellm_params),
     )
 
-    cache_point = {"cachePoint": {"type": "default"}}
+    cache_point: Final = {"cachePoint": {"type": "default"}}
     assert request["messages"][0]["content"] == [
         {"text": "part 0"},
         cache_point,
@@ -10265,7 +10268,7 @@ def _marked_tool_result_parts() -> list[dict[str, object]]:
 
 
 def test_converse_counts_one_cache_point_for_a_tool_result_with_several_marks():
-    request = litellm.AmazonConverseConfig()._transform_request(
+    request: Final = litellm.AmazonConverseConfig()._transform_request(
         model="anthropic.claude-sonnet-5-5",
         messages=[
             {"role": "user", "content": _marked_text_parts(3)},
@@ -10279,7 +10282,7 @@ def test_converse_counts_one_cache_point_for_a_tool_result_with_several_marks():
 
 
 def test_converse_drops_marks_until_a_merged_tool_result_fits_the_cap():
-    messages = [
+    messages: Final = [
         {"role": "user", "content": _marked_text_parts(4)},
         *_converse_tool_turn(*_marked_tool_result_parts()),
     ]
@@ -10295,8 +10298,8 @@ def test_converse_drops_marks_until_a_merged_tool_result_fits_the_cap():
     with pytest.raises(litellm.UnsupportedParamsError, match="this request has 5"):
         transform({})
 
-    request = transform({"drop_params": True})
-    cache_point = {"cachePoint": {"type": "default"}}
+    request: Final = transform({"drop_params": True})
+    cache_point: Final = {"cachePoint": {"type": "default"}}
     assert request["messages"][0]["content"] == [
         {"text": "part 0"},
         cache_point,
@@ -10310,7 +10313,7 @@ def test_converse_drops_marks_until_a_merged_tool_result_fits_the_cap():
 
 
 def test_converse_drops_prompt_cache_breakpoint_from_a_nova_tool_result():
-    messages = [
+    messages: Final = [
         {
             "role": "system",
             "content": [{"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}}],
@@ -10330,7 +10333,7 @@ def test_converse_drops_prompt_cache_breakpoint_from_a_nova_tool_result():
     with pytest.raises(litellm.UnsupportedParamsError, match="tool result"):
         transform({})
 
-    request = transform({"drop_params": True})
+    request: Final = transform({"drop_params": True})
     assert request["system"] == [{"text": "You are terse."}, {"cachePoint": {"type": "default"}}]
     assert _collect_cache_points(request["messages"]) == []