diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index dbe34e3e0c5..4e473c2f3f0 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -1982,9 +1982,17 @@ def messages_within_cache_control_cap( Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first. """ - reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints( - _validated_object_list(tools), cache_control, request_kwargs + return messages_within_reserved_cache_control_cap( + messages, + AnthropicCacheControlHook.count_external_cache_breakpoints( + _validated_object_list(tools), cache_control, request_kwargs + ), ) + + +def messages_within_reserved_cache_control_cap( + messages: Sequence[_MessageT], reserved_marks: int +) -> Sequence[_MessageT]: kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks) dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept)) if dropped > 0: diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 58ca0821afd..056f34f1bf4 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -2241,22 +2241,27 @@ class AmazonConverseConfig(BaseConfig): return data - @staticmethod def _claude_messages_within_cache_control_cap( + self, messages: Sequence[AllMessageValues], capability_model: str, + deployment_model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object], + model_info: Mapping[str, object] | None, ) -> Sequence[AllMessageValues]: if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower(): return messages from litellm.integrations.anthropic_cache_control_hook import ( - messages_within_cache_control_cap, + messages_within_reserved_cache_control_cap, tool_message_with_single_cache_mark, ) - return messages_within_cache_control_cap( - tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools") + return messages_within_reserved_cache_control_cap( + tuple(map(tool_message_with_single_cache_mark, messages)), + self._tool_cache_points_on_the_wire( + capability_model, deployment_model, optional_params, litellm_params, model_info + ), ) async def _async_transform_request( @@ -2285,7 +2290,7 @@ class AmazonConverseConfig(BaseConfig): messages, system_content_blocks = self._transform_system_message( list( self._claude_messages_within_cache_control_cap( - marked_messages, capability_model, json_optional_params, litellm_params + marked_messages, capability_model, model, json_optional_params, litellm_params, model_info ) ), model=capability_model, @@ -2399,7 +2404,7 @@ class AmazonConverseConfig(BaseConfig): messages, system_content_blocks = self._transform_system_message( list( self._claude_messages_within_cache_control_cap( - marked_messages, capability_model, json_optional_params, litellm_params + marked_messages, capability_model, model, json_optional_params, litellm_params, model_info ) ), model=capability_model, diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 883c3f5502b..54c1b5937bf 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -4434,7 +4434,7 @@ def _anthropic_body(messages: list, litellm_params: dict | None = None, tools: l def test_transform_request_carries_prompt_cache_breakpoint_to_cache_control(): - body = _anthropic_body( + body: Final = _anthropic_body( [ { "role": "system", @@ -4477,28 +4477,28 @@ def test_transform_request_carries_prompt_cache_breakpoint_to_cache_control(): ], ) def test_transform_request_prompt_cache_breakpoint_without_claude_equivalent(marker, expected_block): - messages = [{"role": "user", "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": marker}]}] + messages: Final = [{"role": "user", "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": marker}]}] with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"): _anthropic_body(copy.deepcopy(messages)) - body = _anthropic_body(messages, litellm_params={"drop_params": True}) + body: Final = _anthropic_body(messages, litellm_params={"drop_params": True}) assert body["messages"] == [{"role": "user", "content": [expected_block]}] def test_transform_request_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_cap(): - client_marked_part = {"type": "text", "text": "part 0", "cache_control": {"type": "ephemeral"}} - marked_parts = [ + client_marked_part: Final = {"type": "text", "text": "part 0", "cache_control": {"type": "ephemeral"}} + marked_parts: Final = [ {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} for index in range(1, 5) ] - messages = [{"role": "user", "content": [client_marked_part, *marked_parts]}] - tools = [{"name": "lookup", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}}] + messages: Final = [{"role": "user", "content": [client_marked_part, *marked_parts]}] + tools: Final = [{"name": "lookup", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}}] with pytest.raises(litellm.utils.UnsupportedParamsError, match="at most 4 cache breakpoints"): _anthropic_body(copy.deepcopy(messages), tools=copy.deepcopy(tools)) - body = _anthropic_body(messages, litellm_params={"drop_params": True}, tools=tools) + body: Final = _anthropic_body(messages, litellm_params={"drop_params": True}, tools=tools) assert [part.get("cache_control") for part in body["messages"][0]["content"]] == [ {"type": "ephemeral"}, {"type": "ephemeral"}, @@ -4510,8 +4510,8 @@ def test_transform_request_keeps_the_first_and_last_prompt_cache_breakpoints_wit def test_transform_request_counts_a_cache_marked_tool_search_tool_toward_the_cap(): - config = AnthropicConfig() - optional_params = config.map_openai_params( + config: Final = AnthropicConfig() + optional_params: Final = config.map_openai_params( non_default_params={ "max_tokens": 100, "tools": [ @@ -4526,7 +4526,7 @@ def test_transform_request_counts_a_cache_marked_tool_search_tool_toward_the_cap model="claude-sonnet-5-5", drop_params=False, ) - marked_turns = [ + marked_turns: Final = [ { "role": "user", "content": [{"type": "text", "text": f"turn {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}], @@ -4546,9 +4546,9 @@ def test_transform_request_counts_a_cache_marked_tool_search_tool_toward_the_cap with pytest.raises(litellm.utils.UnsupportedParamsError, match="this request has 5"): body({}) - capped = body({"drop_params": True}) + capped: Final = body({"drop_params": True}) assert capped["tools"][0]["cache_control"] == {"type": "ephemeral"} - sent_parts = itertools.chain.from_iterable(message["content"] for message in capped["messages"]) + sent_parts: Final = itertools.chain.from_iterable(message["content"] for message in capped["messages"]) assert [part.get("cache_control") for part in sent_parts] == [ {"type": "ephemeral"}, {"type": "ephemeral"}, @@ -4576,15 +4576,15 @@ def test_transform_request_counts_a_cache_marked_tool_search_tool_toward_the_cap ], ) def test_transform_request_caps_only_the_marks_claude_receives(unsent_marks: list[dict]): - marked_parts = [ + marked_parts: Final = [ {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} for index in range(4) ] - body = _anthropic_body( + body: Final = _anthropic_body( [{"role": "user", "content": marked_parts}, *unsent_marks, {"role": "user", "content": "Go on."}] ) - sent_marks = [ + sent_marks: Final = [ part.get("cache_control") for part in itertools.chain.from_iterable(message["content"] for message in body["messages"]) if isinstance(part, dict) diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 3722ac8a411..7c19ef5b751 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -10074,7 +10074,7 @@ def test_bedrock_prompt_caching_message(messages, expected_cache_control): def test_converse_carries_prompt_cache_breakpoint_to_cache_point(): - request = litellm.AmazonConverseConfig()._transform_request( + request: Final = litellm.AmazonConverseConfig()._transform_request( model="anthropic.claude-sonnet-5-5", messages=[ { @@ -10099,7 +10099,7 @@ def test_converse_carries_prompt_cache_breakpoint_to_cache_point(): def test_converse_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_cap(): - messages = [ + messages: Final = [ { "role": "user", "content": [ @@ -10120,7 +10120,7 @@ def test_converse_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_c with pytest.raises(litellm.utils.UnsupportedParamsError, match="at most 4 cache breakpoints"): transform({}) - cache_point = {"cachePoint": {"type": "default"}} + cache_point: Final = {"cachePoint": {"type": "default"}} assert transform({"drop_params": True})["messages"] == [ { "role": "user", @@ -10157,7 +10157,7 @@ _CONVERSE_CAP_TOOL: Final[dict[str, object]] = { ids=["client_tool_cache_control", "tool_config_injection_point"], ) def test_converse_reserves_the_tool_cache_point_within_the_cap(tool_cache_params: dict[str, object]): - messages = [ + messages: Final = [ { "role": "user", "content": [ @@ -10178,8 +10178,8 @@ def test_converse_reserves_the_tool_cache_point_within_the_cap(tool_cache_params with pytest.raises(litellm.utils.UnsupportedParamsError, match="this request has 5"): transform({}) - request = transform({"drop_params": True}) - cache_point = {"cachePoint": {"type": "default"}} + request: Final = transform({"drop_params": True}) + cache_point: Final = {"cachePoint": {"type": "default"}} assert request["messages"][0]["content"] == [ {"text": "part 0"}, cache_point, @@ -10210,8 +10210,11 @@ def test_converse_reserves_the_tool_cache_point_within_the_cap(tool_cache_params ], ids=["tool_search_tool", "built_in_tool", "tool_config_point_with_no_tool_sent"], ) -def test_converse_leaves_the_cap_to_markers_when_a_marked_tool_is_not_sent(unsent_tool_params: dict[str, object]): - request = litellm.AmazonConverseConfig()._transform_request( +@pytest.mark.parametrize("litellm_params", [{}, {"drop_params": True}], ids=["strict", "drop_params"]) +def test_converse_leaves_the_cap_to_markers_when_a_marked_tool_is_not_sent( + unsent_tool_params: dict[str, object], litellm_params: dict[str, object] +): + request: Final = litellm.AmazonConverseConfig()._transform_request( model="anthropic.claude-sonnet-5-5", messages=[ { @@ -10223,10 +10226,10 @@ def test_converse_leaves_the_cap_to_markers_when_a_marked_tool_is_not_sent(unsen } ], optional_params=copy.deepcopy(unsent_tool_params), - litellm_params={}, + litellm_params=dict(litellm_params), ) - cache_point = {"cachePoint": {"type": "default"}} + cache_point: Final = {"cachePoint": {"type": "default"}} assert request["messages"][0]["content"] == [ {"text": "part 0"}, cache_point, @@ -10265,7 +10268,7 @@ def _marked_tool_result_parts() -> list[dict[str, object]]: def test_converse_counts_one_cache_point_for_a_tool_result_with_several_marks(): - request = litellm.AmazonConverseConfig()._transform_request( + request: Final = litellm.AmazonConverseConfig()._transform_request( model="anthropic.claude-sonnet-5-5", messages=[ {"role": "user", "content": _marked_text_parts(3)}, @@ -10279,7 +10282,7 @@ def test_converse_counts_one_cache_point_for_a_tool_result_with_several_marks(): def test_converse_drops_marks_until_a_merged_tool_result_fits_the_cap(): - messages = [ + messages: Final = [ {"role": "user", "content": _marked_text_parts(4)}, *_converse_tool_turn(*_marked_tool_result_parts()), ] @@ -10295,8 +10298,8 @@ def test_converse_drops_marks_until_a_merged_tool_result_fits_the_cap(): with pytest.raises(litellm.UnsupportedParamsError, match="this request has 5"): transform({}) - request = transform({"drop_params": True}) - cache_point = {"cachePoint": {"type": "default"}} + request: Final = transform({"drop_params": True}) + cache_point: Final = {"cachePoint": {"type": "default"}} assert request["messages"][0]["content"] == [ {"text": "part 0"}, cache_point, @@ -10310,7 +10313,7 @@ def test_converse_drops_marks_until_a_merged_tool_result_fits_the_cap(): def test_converse_drops_prompt_cache_breakpoint_from_a_nova_tool_result(): - messages = [ + messages: Final = [ { "role": "system", "content": [{"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}}], @@ -10330,7 +10333,7 @@ def test_converse_drops_prompt_cache_breakpoint_from_a_nova_tool_result(): with pytest.raises(litellm.UnsupportedParamsError, match="tool result"): transform({}) - request = transform({"drop_params": True}) + request: Final = transform({"drop_params": True}) assert request["system"] == [{"text": "You are terse."}, {"cachePoint": {"type": "default"}}] assert _collect_cache_points(request["messages"]) == []