From 25de2b4cfb53e9f34103c9744731cc295bc3523c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:03:44 +0000 Subject: [PATCH] fix(anthropic): count only the marks each route sends when placing Claude breakpoints A namespace member the Anthropic transform drops no longer holds back a later 1h breakpoint, and a Converse computer use tool is ordered after the toolConfig tools it is sent after. The cache control hook keeps an operator's system injection point on OpenRouter and other non-Bedrock routes when a past server tool call is marked. A tuple additional_drop_params counts like a list. The router's prompt caching pin reads a breakpoint on a tool call and ignores marks on earlier tool calls, so the pin survives the mark moving to the next call --- .../anthropic_cache_control_hook.py | 23 ++++- .../chat/prompt_cache_breakpoints.py | 42 ++++++--- litellm/main.py | 4 +- litellm/router_utils/prompt_caching_cache.py | 39 ++++++-- .../chat/test_prompt_cache_breakpoints.py | 38 ++++++++ .../router_utils/test_prompt_caching_cache.py | 36 ++++++++ tests/unit/test_main.py | 90 ++++++++++++++++++- 7 files changed, 248 insertions(+), 24 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 1986eedc43f..7df13541828 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -28,7 +28,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( cache_control_as_prompt_cache_breakpoint, with_prompt_cache_breakpoint, ) -from litellm.litellm_core_utils.prompt_templates.factory import tool_call_ids_answered_by_client +from litellm.litellm_core_utils.prompt_templates.factory import ( + tool_call_ids_answered_by_client, + tool_call_ids_answered_in, +) from litellm.llms.anthropic.chat.prompt_cache_breakpoints import ( prompt_cache_breakpoint_is_usable, tool_mark_may_reach_claude, @@ -255,7 +258,11 @@ def _tool_call_ids_sent_as_tool_calls_when_a_server_tool_call_is_marked( if not _marks_a_server_tool_call(messages): return frozenset() if model is not None and not _builds_anthropic_messages(model, custom_llm_provider): - return _tool_call_ids(messages) + return ( + _tool_call_ids(messages) + if _served_by_bedrock(model, custom_llm_provider) + else tool_call_ids_answered_in(messages) + ) return tool_call_ids_answered_by_client(messages, _keeps_mid_conversation_system(model, custom_llm_provider)) @@ -274,6 +281,18 @@ def _marks_a_server_tool_call(messages: Sequence[object]) -> bool: return any(_is_marked_server_tool_call(tool_call) for tool_call in _tool_calls(messages)) +def _served_by_bedrock(model: str, custom_llm_provider: str | None) -> bool: + from litellm.exceptions import BadRequestError + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + from litellm.types.utils import LlmProviders + + try: + _, provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + except (BadRequestError, ValueError): + return False + return provider == LlmProviders.BEDROCK.value + + def _builds_anthropic_messages(model: str, custom_llm_provider: str | None) -> bool: from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, get_llm_provider diff --git a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py index 9765b078eaf..0b4081b8348 100644 --- a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py +++ b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py @@ -157,13 +157,19 @@ def _member_mark(member: _Fields) -> object: return own if own is not None else (_fields(member.get("function")) or {}).get("cache_control") +def _transform_sends_member(member: _Fields) -> bool: + return "input_schema" in member or "function" in member or "name" in member + + def _namespace_member_marks(tools: Sequence[object], route: ClaudeRoute) -> tuple[tuple[_Path, object, _Kind], ...]: if route == "converse": return () return tuple( (path, mark, "written") for path, member in _namespace_members(tools) - if tool_mark_reaches_claude(member, route) and (mark := _member_mark(member)) is not None + if _transform_sends_member(member) + and tool_mark_reaches_claude(member, route) + and (mark := _member_mark(member)) is not None ) @@ -225,11 +231,16 @@ def prompt_cache_breakpoint_is_usable(marker: object) -> bool: return not isinstance(mark, UnsupportedPromptCacheParam) or mark.fallback is not None +def _is_computer_use_tool(tool: object) -> bool: + tool_type: Final = (_fields(tool) or {}).get("type") + return isinstance(tool_type, str) and tool_type.startswith(BEDROCK_COMPUTER_USE_TOOLS) + + def _converse_sends_tool_mark(tool: _Fields) -> bool: tool_type: Final = tool.get("type") if tool_type in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES: return False - if isinstance(tool_type, str) and tool_type.startswith(BEDROCK_COMPUTER_USE_TOOLS): + if isinstance(tool_type, str) and _is_computer_use_tool(tool): return not (tool_type.startswith("computer_") and "function" in tool) return "function" in tool or "input_schema" in tool @@ -450,16 +461,18 @@ def _leading_system_count(messages: Sequence[object]) -> int: return next((index for index, message in enumerate(messages) if _role(message) not in _SYSTEM_ROLES), len(messages)) -def _wire_position(path: _Path, leading_system_count: int) -> tuple[int, ...]: - """Where Claude reads a mark: tool definitions, then the cachePoint a route appends after them, then the leading - system messages it hoists, then every other message in place, with a message-level mark on the message's text, - after its content parts and before its tool calls, and last the request-level mark.""" +def _wire_position(path: _Path, leading_system_count: int, late_tools: frozenset[int]) -> tuple[int, ...]: + """Where Claude reads a mark: tool definitions, then the cachePoint a route appends after them, then the + ``late_tools`` a route sends apart from the others, then the leading system messages it hoists, then every other + message in place, with a message-level mark on the message's text, after its content parts and before its tool + calls, and last the request-level mark.""" if path == _TOOL_CONFIG: return (1,) if path == _REQUEST: return (4,) if path[0] == "tools": - return (0, *(index for index in path if isinstance(index, int))) + indices: Final = tuple(index for index in path if isinstance(index, int)) + return (1, 1, *indices) if indices[0] in late_tools else (0, *indices) index: Final = int(path[1]) group: Final = 2 if index < leading_system_count else 3 block: Final = (1,) if len(path) == 2 else (0 if path[2] == "content" else 2, int(path[3])) @@ -482,12 +495,16 @@ def _run_ends(messages: Sequence[object]) -> tuple[int, ...]: def _last_read( - path: _Path, messages: Sequence[object], leading_system_count: int, run_ends: Sequence[int] + path: _Path, + messages: Sequence[object], + leading_system_count: int, + run_ends: Sequence[int], + late_tools: frozenset[int], ) -> tuple[int, ...]: """The transforms may move a system message after the conversation start behind the user, tool and function turns that follow it, never past the next assistant turn, so Claude may read its marks as late as the end of those turns.""" - position: Final = _wire_position(path, leading_system_count) + position: Final = _wire_position(path, leading_system_count, late_tools) index: Final = int(path[1]) if path[0] == "messages" else -1 if index < leading_system_count or _role(messages[index]) not in _SYSTEM_ROLES: return position @@ -658,6 +675,9 @@ def claude_prompt_cache( translated: Final = _translated(hoisted_messages, hoisted_tools) leading_system_count: Final = _leading_system_count(messages) run_ends: Final = _run_ends(messages) + late_tools: Final = frozenset( + index for index, tool in enumerate(renamed_tools) if route == "converse" and _is_computer_use_tool(tool) + ) unsent_call_ids: Final = frozenset() if pairs_unanswered_tool_calls else unanswered_tool_search_call_ids(messages) request_mark: Final = params.get("cache_control") placements: Final[tuple[tuple[_Path, object, _Kind], ...]] = ( @@ -676,8 +696,8 @@ def claude_prompt_cache( path, _ttl(cache_control), kind, - _wire_position(path, leading_system_count), - _last_read(path, messages, leading_system_count, run_ends), + _wire_position(path, leading_system_count, late_tools), + _last_read(path, messages, leading_system_count, run_ends, late_tools), ) for path, cache_control, kind in placements ), diff --git a/litellm/main.py b/litellm/main.py index fa2f87599fe..f1053dab670 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1134,9 +1134,9 @@ def _claude_route(provider_config: BaseConfig | None, reaches_converse: bool) -> def _listed_drop_params(additional_drop_params: object) -> tuple[str, ...]: - if not isinstance(additional_drop_params, list): + if not isinstance(additional_drop_params, (list, tuple)): return () - listed: Final = cast(list[object], additional_drop_params) # cast-ok: elements checked below + listed: Final = cast(Sequence[object], additional_drop_params) # cast-ok: elements checked below return tuple(param for param in listed if isinstance(param, str)) diff --git a/litellm/router_utils/prompt_caching_cache.py b/litellm/router_utils/prompt_caching_cache.py index a3768abb478..8d83c281bbd 100644 --- a/litellm/router_utils/prompt_caching_cache.py +++ b/litellm/router_utils/prompt_caching_cache.py @@ -44,6 +44,7 @@ _MARK_KEYS: Final = frozenset({"cache_control", "prompt_cache_breakpoint"}) _ENVELOPE_SKIPPED_KEYS: Final = _MARK_KEYS | {"content"} _PREFIX_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...]) _TOOLS_ADAPTER: Final = TypeAdapter(tuple[JsonValue, ...]) +_OBJECTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _PINS_ADAPTER: Final[TypeAdapter[tuple[JsonValue, ...] | None]] = TypeAdapter(tuple[JsonValue, ...] | None) @@ -77,6 +78,19 @@ def _carries_cache_mark(block: Mapping[str, object]) -> bool: return _mark_pin_ttl_seconds(block) is not None +def _carries_marked_tool_call(message: Mapping[str, object]) -> bool: + tool_calls: Final = _OBJECTS_ADAPTER.validate_python(message.get("tool_calls") or ()) + return any(_carries_cache_mark(call) for call in tool_calls) + + +def _unmarked(block: JsonValue) -> JsonValue: + return {key: value for key, value in block.items() if key not in _MARK_KEYS} if isinstance(block, dict) else block + + +def _envelope_value(key: str, value: JsonValue) -> JsonValue: + return [_unmarked(call) for call in value] if key == "tool_calls" and isinstance(value, list) else value + + def _block_unit( envelope: tuple[tuple[str, JsonValue], ...], message_run_type: str | None, block: JsonValue ) -> tuple[bytes, str | None]: @@ -89,7 +103,9 @@ def _block_unit( def _message_units(message: Mapping[str, JsonValue]) -> tuple[tuple[bytes, str | None], ...]: - envelope: Final = _sorted_pairs(item for item in message.items() if item[0] not in _ENVELOPE_SKIPPED_KEYS) + envelope: Final = _sorted_pairs( + (key, _envelope_value(key, value)) for key, value in message.items() if key not in _ENVELOPE_SKIPPED_KEYS + ) message_run_type: Final = "tool_result" if message.get("role") == "tool" else None content: Final = message.get("content") if isinstance(content, list) and content: @@ -142,7 +158,12 @@ def _breakpoint_pin_ttl_seconds(carrier: JsonValue | Mapping[str, JsonValue]) -> def _breakpoint_carriers(message: Mapping[str, JsonValue]) -> tuple[JsonValue | Mapping[str, JsonValue], ...]: content: Final = message.get("content") - return (message, *content) if isinstance(content, list) else (message,) + tool_calls: Final = message.get("tool_calls") + return ( + message, + *(content if isinstance(content, list) else ()), + *(tool_calls if isinstance(tool_calls, list) else ()), + ) def _pinned_value(value: JsonValue) -> PromptCachingCacheValue | None: @@ -199,13 +220,15 @@ class PromptCachingCache: last_cacheable_content_idx = None # Also check for cache_control within content blocks (when content is a list) - if not isinstance(content, list): - continue + if isinstance(content, list): + for content_idx, content_block in enumerate(content): + if isinstance(content_block, dict) and _carries_cache_mark(content_block): + last_cacheable_message_idx = msg_idx + last_cacheable_content_idx = content_idx - for content_idx, content_block in enumerate(content): - if isinstance(content_block, dict) and _carries_cache_mark(content_block): - last_cacheable_message_idx = msg_idx - last_cacheable_content_idx = content_idx + if _carries_marked_tool_call(message): + last_cacheable_message_idx = msg_idx + last_cacheable_content_idx = None # If no cacheable block found, return empty list (no cacheable prefix) if last_cacheable_message_idx is None: diff --git a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py index 24d8c65f1de..8a996f4dcd8 100644 --- a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py +++ b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py @@ -704,6 +704,23 @@ def test_a_malformed_breakpoint_beside_a_valid_one_on_a_tool_result_keeps_the_va ] +def test_a_malformed_breakpoint_beside_one_whose_ttl_drop_params_drops_keeps_that_one_under_drop_params(): + messages: Final = [ + { + "role": "tool", + "tool_call_id": "c1", + "content": [ + _part("r0", prompt_cache_breakpoint={"mode": "implicit"}), + _part("r1", prompt_cache_breakpoint=_THIRTY_MINUTES), + ], + } + ] + + assert list(_resolved(messages, drop_params=True).messages) == [ + {"role": "tool", "tool_call_id": "c1", "content": [_part("r0"), _part("r1")], "cache_control": _EPHEMERAL}, + ] + + @pytest.mark.parametrize("malformed", [5, [], "", {"mode": "implicit"}]) def test_a_malformed_breakpoint_on_a_messages_last_part_keeps_the_messages_own_breakpoint_under_drop_params( malformed: object, @@ -1078,3 +1095,24 @@ def test_a_tool_config_cache_point_settles_after_the_tools_under_drop_params( assert list(result.messages) == [ {"role": "user", "content": [_part("ask") if sent_ask is None else _part("ask", cache_control=sent_ask)]} ] + + +@pytest.mark.parametrize( + ("route", "sent_bash_mark"), + [("anthropic", _ONE_HOUR), ("converse", _EPHEMERAL)], + ids=["route_sends_tools_in_order", "route_sends_bash_after_functions"], +) +def test_an_hour_on_a_bash_tool_listed_before_a_five_minute_function_is_dropped_only_where_bash_is_sent_last( + route: ClaudeRoute, sent_bash_mark: dict[str, object] +): + bash: Final = {"type": "bash_20250124", "name": "bash"} + function: Final = {"type": "function", "function": {"name": "t"}, "cache_control": _FIVE_MINUTES} + + result: Final = _resolved( + [{"role": "user", "content": [_part("ask")]}], + [{**bash, "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"}}, function], + drop_params=True, + route=route, + ) + + assert list(result.tools or ()) == [{**bash, "cache_control": sent_bash_mark}, function] diff --git a/tests/unit/router_utils/test_prompt_caching_cache.py b/tests/unit/router_utils/test_prompt_caching_cache.py index 63b20af4032..994b055ecb7 100644 --- a/tests/unit/router_utils/test_prompt_caching_cache.py +++ b/tests/unit/router_utils/test_prompt_caching_cache.py @@ -1,4 +1,5 @@ import asyncio +from itertools import chain from typing import Final import pytest @@ -285,6 +286,41 @@ def test_sync_one_hour_message_level_pin_outlives_the_five_minute_window() -> No assert cache.get_model_id(messages=one_hour, tools=None) is None + +def _tool_call(call_id: str) -> dict[str, object]: + return {"id": call_id, "type": "function", "function": {"name": "lookup_clause", "arguments": "{}"}} + + +def _answered_tool_call(call_id: str) -> tuple[AllMessageValues, AllMessageValues]: + return ( + {"role": "assistant", "content": None, "tool_calls": [_tool_call(call_id)]}, + {"role": "tool", "tool_call_id": call_id, "content": "Section 9."}, + ) + + +def _tool_call_turn(*answered_call_ids: str, marked_call_id: str) -> list[AllMessageValues]: + marked_call: Final = {**_tool_call(marked_call_id), "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"}} + return [ + {"role": "system", "content": "Here is the full text of a complex legal agreement" * 200}, + {"role": "user", "content": "Find the termination clause."}, + *chain.from_iterable(map(_answered_tool_call, answered_call_ids)), + {"role": "assistant", "content": None, "tool_calls": [marked_call]}, + ] + + +def test_a_one_hour_breakpoint_on_a_tool_call_keeps_the_pin_after_the_mark_moves_to_the_next_call() -> None: + clock: Final = _ManualClock() + cache: Final = _pin_cache(clock) + + cache.add_model_id( + model_id="deployment-1h", messages=_tool_call_turn(marked_call_id="call_1"), tools=None, honors_one_hour_ttl=True + ) + clock.advance(PROMPT_CACHE_PIN_TTL_SECONDS + 1) + + assert cache.get_model_id(messages=_tool_call_turn("call_1", marked_call_id="call_2"), tools=None) == { + "model_id": "deployment-1h" + } + async def _first_non_none(read): while True: value = await read() diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index a7bdfc2ef6a..10575122391 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -8,7 +8,7 @@ import logging import os import threading import urllib.parse -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime @@ -1105,6 +1105,32 @@ def test_a_breakpoint_drop_params_drops_leaves_key_auto_caching_on(respx_mock: r }, id="anthropic-namespace", ), + pytest.param( + "anthropic/claude-opus-5-5", + r"https://api\.anthropic\.com/v1/messages", + _CLAUDE_REPLY, + {"api_key": "fake"}, + { + "type": "namespace", + "name": "agents", + "tools": [{"type": "web_search_20250305", "cache_control": _FIVE_MINUTES}], + }, + id="anthropic-nameless-namespace-member", + ), + pytest.param( + "bedrock/converse/us.anthropic.claude-opus-5-5", + _CONVERSE_ROUTE, + _CONVERSE_REPLY, + _FAKE_AWS, + { + "type": "namespace", + "name": "agents", + "tools": [ + {"name": "spawn", "input_schema": {"type": "object"}, "cache_control": _FIVE_MINUTES} + ], + }, + id="converse-namespace-member", + ), pytest.param( "bedrock/invoke/us.anthropic.claude-opus-5-5", _INVOKE_ROUTE, @@ -1218,6 +1244,68 @@ def test_a_marked_past_server_tool_call_takes_a_slot_from_operator_injection_poi assert _converse_cache_points(json.loads(route.calls.last.request.content)) == ("message",) * 4 +@pytest.mark.parametrize( + ("key", "mark"), + [("prompt_cache_breakpoint", _BREAKPOINT), ("cache_control", {"type": "ephemeral"})], + ids=["breakpoint", "cache_control"], +) +def test_a_marked_past_server_tool_call_leaves_the_operator_system_point_in_place_on_openrouter( + respx_mock: respx.MockRouter, openai_api_response: Mapping[str, object], key: str, mark: Mapping[str, str] +): + route: Final = respx_mock.post("https://openrouter.ai/api/v1/chat/completions").respond(json=openai_api_response) + + litellm.completion( + model="openrouter/anthropic/claude-opus-5-5", + messages=[ + {"role": "system", "content": "sys"}, + {"role": "user", "content": [{"type": "text", "text": str(index), key: mark} for index in range(3)]}, + { + "role": "assistant", + "content": "Searching.", + "tool_calls": [{**_SERVER_SEARCH_CALL, key: mark}], + "provider_specific_fields": { + "web_search_results": [ + {"type": "web_search_tool_result", "tool_use_id": _SERVER_SEARCH_CALL["id"], "content": []} + ] + }, + }, + {"role": "tool", "tool_call_id": _SERVER_SEARCH_CALL["id"], "content": "Sunny"}, + {"role": "user", "content": "Thanks"}, + ], + cache_control_injection_points=[ + {"location": "message", "role": "system"}, + {"location": "message", "index": -1}, + ], + api_key="fake", + ) + + assert "cache_control" in json.dumps(json.loads(route.calls.last.request.content)["messages"][0]) + + +def test_a_tuple_additional_drop_params_places_the_same_breakpoints_as_a_list(respx_mock: respx.MockRouter): + route: Final = respx_mock.post(url__regex=_CONVERSE_ROUTE).respond(json=_CONVERSE_REPLY) + + def sent_with(additional_drop_params: Sequence[str]) -> object: + litellm.completion( + model="bedrock/chat_completions/us.anthropic.claude-opus-5-5", + messages=[ + {"role": "system", "content": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": _BREAKPOINT}]}, + *( + {"role": role, "content": [{"type": "text", "text": role, "prompt_cache_breakpoint": _BREAKPOINT}]} + for role in ("user", "assistant", "user") + ), + ], + tools=[{**_WEATHER_TOOL, "function": {**_WEATHER_TOOL["function"], "cache_control": _FIVE_MINUTES}}], + reasoning_effort="none", + drop_params=True, + additional_drop_params=additional_drop_params, + **_FAKE_AWS, + ) + return json.loads(route.calls.last.request.content) + + assert sent_with(("reasoning_effort",)) == sent_with(["reasoning_effort"]) + + def test_completion_keeps_a_function_call_whose_id_only_looks_like_a_claude_server_tool( respx_mock: respx.MockRouter, openai_api_response ):