diff --git a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py index 4c7945f229d..badbf7e4320 100644 --- a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py +++ b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py @@ -1,8 +1,8 @@ """OpenAI's ``prompt_cache_breakpoint`` on a chat request bound for Claude, rewritten as Claude's ``cache_control``.""" from collections.abc import Callable, Iterator, Mapping, Sequence -from dataclasses import dataclass -from itertools import chain +from dataclasses import dataclass, replace +from itertools import accumulate, chain from types import MappingProxyType from typing import Final, Literal, NoReturn, TypeAlias, cast # noqa: TID251 # isinstance-checked hot-path JSON @@ -18,7 +18,10 @@ _Mark: TypeAlias = Callable[[_Path, _Fields], _Fields] _Kind: TypeAlias = Literal["written", "explicit", "free"] _MESSAGE_BLOCK_KEYS: Final = ("content", "tool_calls") _TOOL_RESULT_ROLES: Final = ("tool", "function") +_SYSTEM_ROLES: Final = ("system", "developer") +_SYSTEM_MOVES_PAST_ROLES: Final = ("user", *_TOOL_RESULT_ROLES) _REQUEST: Final[_Path] = ("cache_control",) +_TOOL_CONFIG: Final[_Path] = ("tool_config",) _CLAUDE_TTL_MARKS: Final = MappingProxyType( { "5m": ChatCompletionCachedContent(type="ephemeral", ttl="5m"), @@ -46,6 +49,7 @@ class _Placed: path: _Path ttl: Literal["5m", "1h"] kind: _Kind + rank: tuple[int, ...] @dataclass(frozen=True, slots=True) @@ -171,27 +175,23 @@ def _joined_breakpoint(kept: object, replaced: Sequence[object]) -> object: return {**fields, "ttl": "1h"} if any(_block_mark(marker, "ttl") == "1h" for marker in replaced) else kept -def _tool_result_hoisted(message: _Fields, one_mark_per_tool_result: bool) -> _Hoisted: - """Claude rejects a tool result marked both on the result and inside its content, so a ``cache_control`` the - client wrote at either level wins over every breakpoint at the other, and a breakpoint on the message replaces the - ones on its parts. A route that sends one mark per tool result gets its part breakpoints as one on the message, - or as none when the client marked a part.""" +def _tool_result_hoisted(message: _Fields) -> _Hoisted: + """Claude rejects a tool result marked both on the result and inside its content, or marked twice inside it, so + a ``cache_control`` the client wrote at either level wins over every breakpoint, and the result's breakpoints + become one on the message.""" parts: Final = _blocks(message.get("content")) if parts is None: return _Hoisted(message, ()) own: Final = message.get("prompt_cache_breakpoint") - client_marked_part: Final = any(_block_mark(part, "cache_control") is not None for part in parts) - if own is not None and client_marked_part: - return _Hoisted(_without_message_breakpoint(message), ()) unmarked_parts: Final = [_without_breakpoint(part) for part in parts] if message.get("cache_control") is not None: return _Hoisted({**message, "content": unmarked_parts}, ()) part_breakpoints: Final = tuple( marker for marker in (_block_mark(part, "prompt_cache_breakpoint") for part in parts) if marker is not None ) - if own is None and client_marked_part and one_mark_per_tool_result: - return _Hoisted({**message, "content": unmarked_parts}, part_breakpoints) - if own is None and (client_marked_part or not one_mark_per_tool_result or not part_breakpoints): + if any(_block_mark(part, "cache_control") is not None for part in parts): + return _Hoisted({**_without_message_breakpoint(message), "content": unmarked_parts}, part_breakpoints) + if own is None and not part_breakpoints: return _Hoisted(message, ()) kept: Final = part_breakpoints[0] if own is None else own replaced: Final = part_breakpoints[1:] if own is None else part_breakpoints @@ -250,12 +250,12 @@ def _message_breakpoint_hoisted(message: _Fields) -> _Hoisted: ) -def _hoisted_message(message: object, one_mark_per_tool_result: bool) -> _Hoisted: +def _hoisted_message(message: object) -> _Hoisted: fields: Final = _fields(message) if fields is None: return _Hoisted(message, ()) return ( - _tool_result_hoisted(fields, one_mark_per_tool_result) + _tool_result_hoisted(fields) if fields.get("role") in _TOOL_RESULT_ROLES else _message_breakpoint_hoisted(fields) ) @@ -362,29 +362,58 @@ def _translated(messages: Sequence[object], tools: Sequence[object]) -> frozense ) +def _role(message: object) -> object: + return (_fields(message) or {}).get("role") + + def _leading_system_count(messages: Sequence[object]) -> int: - return next( - ( - index - for index, message in enumerate(messages) - if (_fields(message) or {}).get("role") not in ("system", "developer") - ), - len(messages), - ) + return next((index for index, message in enumerate(messages) if _role(message) not in _SYSTEM_ROLES), len(messages)) def _wire_position(path: _Path, leading_system_count: int) -> tuple[int, ...]: - """Where Claude reads a mark: tool definitions, then the leading system messages it hoists, then every other - message in place, with a message-level mark on the message's text, after its content parts and before its tool - calls.""" + """Where Claude reads a mark: tool definitions, then the cachePoint a route appends after them, then the leading + system messages it hoists, then every other message in place, with a message-level mark on the message's text, + after its content parts and before its tool calls, and last the request-level mark.""" + if path == _TOOL_CONFIG: + return (1,) + if path == _REQUEST: + return (4,) if path[0] == "tools": return (0, *(index for index in path if isinstance(index, int))) index: Final = int(path[1]) - group: Final = 1 if index < leading_system_count else 2 + group: Final = 2 if index < leading_system_count else 3 block: Final = (1,) if len(path) == 2 else (0 if path[2] == "content" else 2, int(path[3])) return (group, index, *block) +def _open_system_group(opened_at: tuple[int, ...], indexed: tuple[int, object]) -> tuple[int, ...]: + index, message = indexed + role: Final = _role(message) + if role in _SYSTEM_ROLES: + return opened_at or (index,) + return opened_at if role in _SYSTEM_MOVES_PAST_ROLES else () + + +def _system_groups(messages: Sequence[object], leading_system_count: int) -> Mapping[int, int]: + """The transforms move a system message after the conversation start behind the user, tool and function turns + that follow it, never past the next assistant turn, so each such message and those turns map to the message + that opens their group.""" + opened: Final = tuple( + accumulate( + enumerate(messages[leading_system_count:], start=leading_system_count), _open_system_group, initial=() + ) + )[1:] + return MappingProxyType( + {index: opened_at[0] for index, opened_at in enumerate(opened, start=leading_system_count) if opened_at} + ) + + +def _rank(path: _Path, leading_system_count: int, system_groups: Mapping[int, int]) -> tuple[int, ...]: + """Marks in one system group share a rank, since Claude may read them in any order.""" + opened_at: Final = system_groups.get(int(path[1])) if path[0] == "messages" else None + return _wire_position(path, leading_system_count) if opened_at is None else (3, opened_at) + + def _tool_call_id(message: _Fields, index: int) -> object: return (_fields((_blocks(message.get("tool_calls")) or ())[index]) or {}).get("id") @@ -430,19 +459,21 @@ def _chosen(mark: _Placed, ttl: Literal["5m", "1h"]) -> bool: return mark.kind != "free" and mark.ttl == ttl -def _last_chosen_hour(marks: Sequence[_Placed]) -> int: - return max((index for index, mark in enumerate(marks) if _chosen(mark, "1h")), default=-1) +def _last_chosen_hour(marks: Sequence[_Placed]) -> tuple[int, ...]: + return max((mark.rank for mark in marks if _chosen(mark, "1h")), default=(-1,)) def _loosened(marks: Sequence[_Placed], drop_params: bool) -> tuple[_Placed, ...]: """Claude rejects a 1h mark read after a 5m one.""" - first_five: Final = next((index for index, mark in enumerate(marks) if _chosen(mark, "5m")), len(marks)) + first_five: Final = min((mark.rank for mark in marks if _chosen(mark, "5m")), default=(5,)) last_hour: Final = _last_chosen_hour(marks) return tuple( - _Placed(mark.path, "5m", "free") - if drop_params and mark.kind == "explicit" and (index > first_five if mark.ttl == "1h" else index < last_hour) + replace(mark, ttl="5m", kind="free") + if drop_params + and mark.kind == "explicit" + and (mark.rank >= first_five if mark.ttl == "1h" else mark.rank <= last_hour) else mark - for index, mark in enumerate(marks) + for mark in marks ) @@ -452,11 +483,28 @@ def _settled(marks: Sequence[_Placed], drop_params: bool) -> tuple[_Placed, ...] loosened: Final = _loosened(marks, drop_params) last_hour: Final = _last_chosen_hour(loosened) return tuple( - _Placed(mark.path, "1h", "free") if mark.kind == "free" and index < last_hour else mark - for index, mark in enumerate(loosened) + replace(mark, ttl="1h") if mark.kind == "free" and mark.rank <= last_hour else mark for mark in loosened ) +def _route_marks( + tool_config_cache_control: ChatCompletionCachedContent | None, request_mark: object +) -> tuple[tuple[_Path, object, _Kind], ...]: + """The cachePoint a route appends after the tools takes 1h like a breakpoint when it sets no ttl. Neither it nor + the request-level mark is ever dropped.""" + tool_config_kind: Final[_Kind] = ( + "free" if tool_config_cache_control is None or tool_config_cache_control.get("ttl") is None else "written" + ) + return ( + *(() if tool_config_cache_control is None else ((_TOOL_CONFIG, tool_config_cache_control, tool_config_kind),)), + *(() if request_mark is None else ((_REQUEST, request_mark, "written"),)), + ) + + +def _droppable(mark: _Placed) -> bool: + return mark.kind != "written" and mark.path != _TOOL_CONFIG + + def claude_prompt_cache( messages: Sequence[object], tools: Sequence[object] | None, @@ -464,18 +512,18 @@ def claude_prompt_cache( *, drop_params: bool, tool_history_as_text: bool, - one_mark_per_tool_result: bool, + tool_config_cache_control: ChatCompletionCachedContent | None, ) -> ClaudePromptCache | UnsupportedPromptCacheParam: """``prompt_cache_breakpoint`` marks renamed to ``cache_control``. A mark Claude cannot honor is reported unless ``drop_params``, which keeps what Claude can use and, when the request ends up with more than Claude's four - breakpoints, drops the earliest translated ones. A request-level ``cache_control`` in ``params`` takes one of the - four, marks on tool history a ``tool_history_as_text`` route flattens take none, and a - ``one_mark_per_tool_result`` route gets a tool result's breakpoints as one. A request carrying no breakpoint is - returned as written.""" + breakpoints, drops the earliest translated ones. A request-level ``cache_control`` in ``params`` and the + ``tool_config_cache_control`` cachePoint a route appends after the tools each take one of the four, and marks on + tool history a ``tool_history_as_text`` route flattens take none. A request carrying no breakpoint is returned as + written.""" listed_tools: Final = tools or () if not any(map(_message_carries_breakpoint, messages)) and not any(map(_tool_carries_breakpoint, listed_tools)): return ClaudePromptCache(tuple(messages), None if tools is None else tuple(tools)) - hoists: Final = tuple(_hoisted_message(message, one_mark_per_tool_result) for message in messages) + hoists: Final = tuple(map(_hoisted_message, messages)) hoisted_messages: Final = tuple(hoist.message for hoist in hoists) blocks: Final = tuple(block for _, block in _markable(*_marked_request(hoisted_messages, listed_tools, _unchanged))) problem: Final = next( @@ -495,24 +543,28 @@ def claude_prompt_cache( renamed_messages, renamed_tools = _marked_request(hoisted_messages, listed_tools, _renamed) translated: Final = _translated(hoisted_messages, listed_tools) leading_system_count: Final = _leading_system_count(messages) + system_groups: Final = _system_groups(messages, leading_system_count) unsent_call_ids: Final = unanswered_tool_search_call_ids(messages) - block_marks: Final = sorted( - ( - _Placed(path, _ttl(block.get("cache_control")), _kind(path, block.get("cache_control"), translated)) + request_mark: Final = params.get("cache_control") + placements: Final[tuple[tuple[_Path, object, _Kind], ...]] = ( + *( + (path, block.get("cache_control"), _kind(path, block.get("cache_control"), translated)) for path, block in _markable(renamed_messages, renamed_tools) if block.get("cache_control") is not None and _on_wire(path, renamed_messages, renamed_tools, unsent_call_ids, tool_history_as_text) ), - key=lambda mark: _wire_position(mark.path, leading_system_count), + *_route_marks(tool_config_cache_control, request_mark), ) - request_mark: Final = params.get("cache_control") - marks: Final = ( - *block_marks, - *(() if request_mark is None else (_Placed(_REQUEST, _ttl(request_mark), "written"),)), + marks: Final = sorted( + ( + _Placed(path, _ttl(cache_control), kind, _rank(path, leading_system_count, system_groups)) + for path, cache_control, kind in placements + ), + key=lambda mark: _wire_position(mark.path, leading_system_count), ) excess: Final = len(marks) - MAX_CACHE_CONTROL_BLOCKS dropped: Final[frozenset[_Path]] = ( - frozenset(tuple(mark.path for mark in marks if mark.kind != "written")[:excess]) + frozenset(tuple(mark.path for mark in marks if _droppable(mark))[:excess]) if excess > 0 and drop_params else frozenset() ) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 6ba31b606fd..06b56932578 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -255,12 +255,59 @@ class AmazonConverseConfig(BaseConfig): ) @staticmethod - def sends_tool_history_as_text(tools: object, non_default_params: Mapping[str, object]) -> bool: - return ( - litellm.bedrock_neutralize_orphaned_tool_blocks - and not tools - and not any(map(non_default_params.__contains__, ("functions", "response_format", "web_search_options"))) + def may_send_tools( + *, tools: object, functions: object, response_format: object, web_search_options: object + ) -> bool: + """Converse builds tools from ``functions``, a ``response_format`` or ``web_search_options`` too.""" + return bool(tools) or any(value is not None for value in (functions, response_format, web_search_options)) + + @staticmethod + def sends_tool_history_as_text(may_send_tools: bool) -> bool: + return litellm.bedrock_neutralize_orphaned_tool_blocks and not may_send_tools + + @staticmethod + def tool_config_cache_control(injection_points: object) -> ChatCompletionCachedContent | None: + """The ``cache_control`` of the cachePoint a ``tool_config`` injection point appends after the tools.""" + from litellm.integrations.anthropic_cache_control_hook import configured_injection_points + + point: Final = next( + ( + point + for point in configured_injection_points(injection_points) + if point.get("location") == "tool_config" + ), + None, ) + if point is None: + return None + control: Final = point.get("control") + return control if isinstance(control, dict) else ChatCompletionCachedContent(type="ephemeral") + + @staticmethod + def _sends_a_one_hour_cache_point( + system_content_blocks: Sequence[SystemContentBlock], messages: Sequence[AllMessageValues] + ) -> bool: + holders: Final = chain.from_iterable( + (message, *parts_of(message_field(message, "content")), *parts_of(message_field(message, "tool_calls"))) + for message in messages + ) + return any(message_field(block.get("cachePoint"), "ttl") == "1h" for block in system_content_blocks) or any( + message_field(message_field(holder, "cache_control"), "ttl") == "1h" for holder in holders + ) + + @staticmethod + def _tool_config_cache_point_control( + control: ChatCompletionCachedContent, + system_content_blocks: Sequence[SystemContentBlock], + messages: Sequence[AllMessageValues], + ) -> ChatCompletionCachedContent: + """Claude rejects a 1h cachePoint read after a 5m one, so a tool_config cachePoint that sets no ttl takes 1h + when a later one does.""" + if control.get("ttl") is not None or not AmazonConverseConfig._sends_a_one_hour_cache_point( + system_content_blocks, messages + ): + return control + return ChatCompletionCachedContent(type="ephemeral", ttl="1h") @staticmethod def _neutralize_orphaned_tool_blocks( @@ -1999,27 +2046,36 @@ class AmazonConverseConfig(BaseConfig): ) # Append cachePoint to tools if cache_control_injection_points has tool_config - cache_injection_points: Final = additional_request_params.pop("cache_control_injection_points", None) + injection_points: Final = cast( # cast-ok: untyped request params + object, additional_request_params.pop("cache_control_injection_points", None) + ) + tool_config_control: Final = self.tool_config_cache_control(injection_points) if ( - cache_injection_points + tool_config_control is not None and len(bedrock_tools) > 0 and bedrock_model_accepts_cache_points(model, deployment_model) ): - for point in cache_injection_points: - if point.get("location") == "tool_config": - cache_point = self._build_cache_point_block(point.get("control"), model, deployment_model) - bedrock_tools.append(ToolBlock(cachePoint=cache_point)) - # Spend attribution credits the gateway only for breakpoints it placed, and - # this is the one place a tool_config point becomes one. The hook that reads - # the configuration cannot record it: whether a cachePoint lands depends on - # this provider and on the request carrying tools, neither of which it sees. - if litellm_params is not None: - from litellm.integrations.anthropic_cache_control_hook import ( - AnthropicCacheControlHook, - ) + bedrock_tools.append( + ToolBlock( + cachePoint=self._build_cache_point_block( + self._tool_config_cache_point_control( + tool_config_control, system_content_blocks, messages or () + ), + model, + deployment_model, + ) + ) + ) + # Spend attribution credits the gateway only for breakpoints it placed, and + # this is the one place a tool_config point becomes one. The hook that reads + # the configuration cannot record it: whether a cachePoint lands depends on + # this provider and on the request carrying tools, neither of which it sees. + if litellm_params is not None: + from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + ) - AnthropicCacheControlHook.record_gateway_injection(litellm_params, 1) - break + AnthropicCacheControlHook.record_gateway_injection(litellm_params, 1) bedrock_tool_config: ToolConfigBlock | None = None if len(bedrock_tools) > 0: diff --git a/litellm/main.py b/litellm/main.py index 65a06b9d42a..6f3804a5fe1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5594,16 +5594,29 @@ def completion( "drop_params" in kwargs and normalize_drop_params(cast(object, kwargs["drop_params"])) is True # cast-ok: untyped request kwargs ) + request_tools: Final = cast(list[object] | None, tools) # cast-ok: completion types tools as a bare List + converse_may_send_tools: Final = AmazonConverseConfig.may_send_tools( + tools=request_tools, + functions=cast(object, functions), # cast-ok: completion types functions as a bare List + response_format=cast(object, response_format), # cast-ok: completion types response_format as a bare dict + web_search_options=web_search_options, + ) claude_cache: Final = ( resolved_claude_prompt_cache( claude_prompt_cache( routed_messages, - cast(list[object] | None, tools), # cast-ok: completion types tools as a bare List + request_tools, non_default_params, drop_params=claude_drop_params, tool_history_as_text=isinstance(provider_config, AmazonConverseConfig) - and AmazonConverseConfig.sends_tool_history_as_text(tools, non_default_params), - one_mark_per_tool_result=isinstance(provider_config, AmazonConverseConfig), + and AmazonConverseConfig.sends_tool_history_as_text(converse_may_send_tools), + tool_config_cache_control=( + AmazonConverseConfig.tool_config_cache_control( + non_default_params.get("cache_control_injection_points") + ) + if isinstance(provider_config, AmazonConverseConfig) and converse_may_send_tools + else None + ), ), model=model, llm_provider=custom_llm_provider, diff --git a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py index fbf2e10ca68..0d3b5687714 100644 --- a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py +++ b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py @@ -9,6 +9,7 @@ from litellm.llms.anthropic.chat.prompt_cache_breakpoints import ( claude_prompt_cache, resolved_claude_prompt_cache, ) +from litellm.types.llms.openai import ChatCompletionCachedContent _EXPLICIT: Final = {"mode": "explicit"} _EPHEMERAL: Final = {"type": "ephemeral"} @@ -21,7 +22,7 @@ def _cache( params: dict[str, object] | None = None, *, drop_params: bool = False, - one_mark_per_tool_result: bool = False, + tool_config_cache_control: ChatCompletionCachedContent | None = None, ) -> ClaudePromptCache | UnsupportedPromptCacheParam: return claude_prompt_cache( messages, @@ -29,7 +30,7 @@ def _cache( params or {}, drop_params=drop_params, tool_history_as_text=False, - one_mark_per_tool_result=one_mark_per_tool_result, + tool_config_cache_control=tool_config_cache_control, ) @@ -39,10 +40,10 @@ def _resolved( params: dict[str, object] | None = None, *, drop_params: bool = False, - one_mark_per_tool_result: bool = False, + tool_config_cache_control: ChatCompletionCachedContent | None = None, ) -> ClaudePromptCache: outcome: Final = _cache( - messages, tools, params, drop_params=drop_params, one_mark_per_tool_result=one_mark_per_tool_result + messages, tools, params, drop_params=drop_params, tool_config_cache_control=tool_config_cache_control ) assert isinstance(outcome, ClaudePromptCache), outcome return outcome @@ -502,7 +503,7 @@ def test_a_part_breakpoint_a_tool_results_own_breakpoint_replaces_is_still_repor assert outcome.value == {"mode": "explicit", "ttl": "30m"} -def test_a_route_sending_one_mark_per_tool_result_counts_its_marked_parts_as_one_under_drop_params(): +def test_a_tool_results_marked_parts_count_as_one_mark_on_the_result_under_drop_params(): messages: Final = [ {"role": "system", "content": [_part("s", prompt_cache_breakpoint=_EXPLICIT)]}, {"role": "user", "content": [_part("u1", prompt_cache_breakpoint=_EXPLICIT)]}, @@ -514,7 +515,7 @@ def test_a_route_sending_one_mark_per_tool_result_counts_its_marked_parts_as_one {"role": "user", "content": [_part("u2", prompt_cache_breakpoint=_EXPLICIT)]}, ] - result: Final = _resolved(messages, drop_params=True, one_mark_per_tool_result=True) + result: Final = _resolved(messages, drop_params=True) assert list(result.messages) == [ {"role": "system", "content": [_part("s", cache_control=_EPHEMERAL)]}, @@ -524,7 +525,7 @@ def test_a_route_sending_one_mark_per_tool_result_counts_its_marked_parts_as_one ] -def test_a_route_sending_one_mark_per_tool_result_lets_a_client_marked_part_win_under_drop_params(): +def test_a_client_marked_part_of_a_tool_result_wins_over_its_breakpoints_under_drop_params(): messages: Final = [ {"role": "system", "content": [_part("s", prompt_cache_breakpoint=_EXPLICIT)]}, {"role": "user", "content": [_part("u1", prompt_cache_breakpoint=_EXPLICIT)]}, @@ -536,7 +537,7 @@ def test_a_route_sending_one_mark_per_tool_result_lets_a_client_marked_part_win_ {"role": "user", "content": [_part("u2", prompt_cache_breakpoint=_EXPLICIT)]}, ] - result: Final = _resolved(messages, drop_params=True, one_mark_per_tool_result=True) + result: Final = _resolved(messages, drop_params=True) assert list(result.messages) == [ {"role": "system", "content": [_part("s", cache_control=_EPHEMERAL)]}, @@ -546,7 +547,7 @@ def test_a_route_sending_one_mark_per_tool_result_lets_a_client_marked_part_win_ ] -def test_a_part_breakpoint_a_client_marked_part_replaces_on_a_one_mark_route_is_still_reported(): +def test_a_part_breakpoint_a_client_marked_part_replaces_is_still_reported(): messages: Final = [ { "role": "tool", @@ -558,7 +559,7 @@ def test_a_part_breakpoint_a_client_marked_part_replaces_on_a_one_mark_route_is_ } ] - outcome: Final = _cache(messages, one_mark_per_tool_result=True) + outcome: Final = _cache(messages) assert isinstance(outcome, UnsupportedPromptCacheParam) assert outcome.value == {"mode": "explicit", "ttl": "30m"} @@ -576,3 +577,129 @@ def test_a_null_tool_level_breakpoint_leaves_the_function_breakpoint_in_force(): result: Final = _resolved([{"role": "user", "content": "hi"}], tools) assert list(result.tools or ()) == [{"type": "function", "function": {"name": "t"}, "cache_control": _EPHEMERAL}] + + +@pytest.mark.parametrize( + ("between", "ask", "drop_params", "sent_reminder", "sent_ask"), + [ + pytest.param((), _EXPLICIT, False, _ONE_HOUR, _ONE_HOUR, id="free-mark-takes-one-hour"), + pytest.param((), _EXPLICIT, True, _ONE_HOUR, _ONE_HOUR, id="free-mark-takes-one-hour-under-drop-params"), + pytest.param( + ({"role": "user", "content": "more"}, {"role": "system", "content": "note"}), + _EXPLICIT, + False, + _ONE_HOUR, + _ONE_HOUR, + id="past-a-later-system-message", + ), + pytest.param( + ({"role": "user", "content": "more"}, {"role": "assistant", "content": "b"}), + _EXPLICIT, + False, + _ONE_HOUR, + _EPHEMERAL, + id="after-the-next-assistant-turn", + ), + pytest.param( + (), + {"mode": "explicit", "ttl": "5m"}, + True, + _EPHEMERAL, + {"type": "ephemeral", "ttl": "5m"}, + id="set-ttls-loosen-under-drop-params", + ), + ], +) +def test_marks_on_a_mid_conversation_system_message_and_the_turns_it_may_move_past_settle_together( + between: tuple[dict[str, object], ...], + ask: dict[str, object], + drop_params: bool, + sent_reminder: dict[str, object], + sent_ask: dict[str, object], +): + messages: Final = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "q"}, + {"role": "assistant", "content": "a"}, + {"role": "system", "content": [_part("reminder", prompt_cache_breakpoint={"mode": "explicit", "ttl": "1h"})]}, + *between, + {"role": "user", "content": [_part("ask", prompt_cache_breakpoint=ask)]}, + ] + + result: Final = _resolved(messages, drop_params=drop_params) + + assert (result.messages[3], result.messages[-1]) == ( + {"role": "system", "content": [_part("reminder", cache_control=sent_reminder)]}, + {"role": "user", "content": [_part("ask", cache_control=sent_ask)]}, + ) + + +def test_a_tool_config_cache_point_takes_one_of_claudes_four_and_is_never_dropped_under_drop_params(): + messages: Final = [ + {"role": "system", "content": [_part("s", prompt_cache_breakpoint=_EXPLICIT)]}, + {"role": "user", "content": [_part(str(index), prompt_cache_breakpoint=_EXPLICIT) for index in range(3)]}, + ] + + result: Final = _resolved( + messages, + [{"type": "function", "function": {"name": "t"}}], + drop_params=True, + tool_config_cache_control=ChatCompletionCachedContent(type="ephemeral"), + ) + + assert list(result.messages) == [ + {"role": "system", "content": [_part("s")]}, + {"role": "user", "content": [_part(str(index), cache_control=_EPHEMERAL) for index in range(3)]}, + ] + + +@pytest.mark.parametrize( + ("tool_config", "tool_marker", "ask_marker", "sent_tool", "sent_ask"), + [ + pytest.param( + ChatCompletionCachedContent(type="ephemeral"), + None, + {"mode": "explicit", "ttl": "1h"}, + {}, + _ONE_HOUR, + id="free-keeps-a-later-hour", + ), + pytest.param( + ChatCompletionCachedContent(type="ephemeral", ttl="5m"), + None, + {"mode": "explicit", "ttl": "1h"}, + {}, + _EPHEMERAL, + id="five-minutes-loosens-a-later-hour", + ), + pytest.param( + ChatCompletionCachedContent(type="ephemeral", ttl="1h"), + _EXPLICIT, + None, + {"cache_control": _ONE_HOUR}, + None, + id="one-hour-lifts-an-earlier-tool-mark", + ), + ], +) +def test_a_tool_config_cache_point_settles_after_the_tools_under_drop_params( + tool_config: ChatCompletionCachedContent, + tool_marker: dict[str, object] | None, + ask_marker: dict[str, object] | None, + sent_tool: dict[str, object], + sent_ask: dict[str, object] | None, +): + tool: Final = {"type": "function", "function": {"name": "t"}} + ask: Final = _part("ask") if ask_marker is None else _part("ask", prompt_cache_breakpoint=ask_marker) + + result: Final = _resolved( + [{"role": "user", "content": [ask]}], + [tool if tool_marker is None else {**tool, "prompt_cache_breakpoint": tool_marker}], + drop_params=True, + tool_config_cache_control=tool_config, + ) + + assert list(result.tools or ()) == [{**tool, **sent_tool}] + assert list(result.messages) == [ + {"role": "user", "content": [_part("ask") if sent_ask is None else _part("ask", cache_control=sent_ask)]} + ] diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 6c996a7ce75..84264702b43 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -5947,6 +5947,67 @@ def test_cache_control_injection_tool_config_not_added_without_injection_point() assert all("cachePoint" not in tool for tool in tools) +@pytest.mark.parametrize( + ("messages", "sent_cache_point"), + [ + pytest.param( + [ + {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + {"role": "user", "content": "What is the weather?"}, + ], + {"type": "default", "ttl": "1h"}, + id="one-hour-system", + ), + pytest.param( + [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is the weather?", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + } + ], + {"type": "default", "ttl": "1h"}, + id="one-hour-message", + ), + pytest.param( + [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is the weather?", + "cache_control": {"type": "ephemeral", "ttl": "5m"}, + } + ], + } + ], + {"type": "default"}, + id="five-minute-message", + ), + ], +) +def test_a_tool_config_cache_point_without_a_ttl_takes_one_hour_when_a_later_cache_point_does( + messages: list[AllMessageValues], sent_cache_point: dict[str, str] +): + result: Final = AmazonConverseConfig()._transform_request( + model="us.anthropic.claude-opus-5-5", + messages=messages, + optional_params={ + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + "cache_control_injection_points": [{"location": "tool_config"}], + }, + litellm_params={}, + ) + + assert result["toolConfig"]["tools"][-1] == {"cachePoint": sent_cache_point} + + def test_cache_control_injection_tool_config_honors_ttl_for_supported_model(monkeypatch): """ Regression test: cache_control_injection_points with location=tool_config diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 5cfb7b8e370..669b68a7884 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -759,6 +759,96 @@ def test_a_breakpoint_reaches_bedrock_mantle_claude_as_cache_control(respx_mock: ] +_WEATHER_TOOL: Final = {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}} + + +def test_breakpoints_on_converse_tool_history_beside_a_json_schema_response_format_keep_to_four_under_drop_params( + respx_mock: respx.MockRouter, +): + route: Final = respx_mock.post(url__regex=_CONVERSE_ROUTE).respond(json=_CONVERSE_REPLY) + + litellm.completion( + model="bedrock/converse/us.anthropic.claude-opus-5-5", + messages=[ + {"role": "system", "content": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": _BREAKPOINT}]}, + { + "role": "user", + "content": [ + {"type": "text", "text": str(index), "prompt_cache_breakpoint": _BREAKPOINT} for index in range(3) + ], + }, + { + "role": "assistant", + "content": None, + "tool_calls": [{**_LOADED_FUNCTION_CALL, "prompt_cache_breakpoint": _BREAKPOINT}], + }, + {**_WEATHER_RESULT, "prompt_cache_breakpoint": _BREAKPOINT}, + {"role": "user", "content": "And tomorrow?"}, + ], + response_format={"type": "json_schema", "json_schema": {"name": "forecast", "schema": {"type": "object"}}}, + drop_params=True, + **_FAKE_AWS, + ) + + assert _converse_cache_points(json.loads(route.calls.last.request.content)) == ("message",) * 4 + + +def test_breakpoints_on_one_tool_results_parts_reach_claude_as_one_mark_on_the_result(respx_mock: respx.MockRouter): + route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").respond( + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-opus-5-5", + "content": [{"type": "text", "text": "18C"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ) + + litellm.completion( + model="anthropic/claude-opus-5-5", + messages=[ + _USER_TURN, + {"role": "assistant", "content": None, "tool_calls": [_LOADED_FUNCTION_CALL]}, + { + **_WEATHER_RESULT, + "content": [ + {"type": "text", "text": text, "prompt_cache_breakpoint": _BREAKPOINT} for text in ("18C", "sunny") + ], + }, + ], + tools=[_WEATHER_TOOL], + api_key="fake", + ) + + assert json.loads(route.calls.last.request.content)["messages"][2]["content"] == [ + { + "type": "tool_result", + "tool_use_id": "toolu_01weather", + "content": [{"type": "text", "text": "18C"}, {"type": "text", "text": "sunny"}], + "cache_control": {"type": "ephemeral"}, + } + ] + + +def test_a_tool_breakpoint_before_a_one_hour_tool_config_cache_point_reaches_converse_as_one_hour( + respx_mock: respx.MockRouter, +): + route: Final = respx_mock.post(url__regex=_CONVERSE_ROUTE).respond(json=_CONVERSE_REPLY) + + litellm.completion( + model="bedrock/converse/us.anthropic.claude-opus-5-5", + messages=[_USER_TURN], + tools=[{**_WEATHER_TOOL, "prompt_cache_breakpoint": _BREAKPOINT}], + cache_control_injection_points=[{"location": "tool_config", "control": {"type": "ephemeral", "ttl": "1h"}}], + **_FAKE_AWS, + ) + + sent_tools: Final = json.loads(route.calls.last.request.content)["toolConfig"]["tools"] + assert [tool["cachePoint"] for tool in sent_tools if "cachePoint" in tool] == [{"type": "default", "ttl": "1h"}] * 2 + + def test_completion_keeps_a_function_call_whose_id_only_looks_like_a_claude_server_tool( respx_mock: respx.MockRouter, openai_api_response ):