diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 845fd109f79..d35b77b2658 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -309,7 +309,7 @@ def _is_marked_server_tool_call(tool_call: object) -> bool: return ( isinstance(tool_call_id, str) and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) - and _attribute_or_key(tool_call, "cache_control") is not None + and _carries_cache_breakpoint(tool_call) ) diff --git a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py index 7cd44c0e5f7..d9ba0a60688 100644 --- a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py +++ b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py @@ -50,6 +50,7 @@ class _Placed: path: _Path ttl: Literal["5m", "1h"] kind: _Kind + translated: bool first_read: tuple[int, ...] last_read: tuple[int, ...] @@ -447,13 +448,16 @@ def _on_wire( tools: Sequence[object], unsent_call_ids: frozenset[str], tool_history_as_text: bool, + function_marks_on_wire: bool, ) -> bool: """Claude sends a message-level mark on string content, or on list content only for a tool result, and none on - a developer message, which reaches it as a system message without one; a mark under ``function`` only when the - tool carries none of its own; no tool_search call nobody answered; and no mark on tool history the route sends - as text.""" + a developer message, which reaches it as a system message without one; a mark under ``function`` only on a route + that sends one and when the tool carries none of its own; no tool_search call nobody answered; and no mark on + tool history the route sends as text.""" if path[0] == "tools": - return len(path) == 2 or (_fields(tools[int(path[1])]) or {}).get("cache_control") is None + return len(path) == 2 or ( + function_marks_on_wire and (_fields(tools[int(path[1])]) or {}).get("cache_control") is None + ) message: Final = _fields(messages[int(path[1])]) or {} if tool_history_as_text and _is_tool_history(message): return False @@ -467,10 +471,10 @@ def _ttl(cache_control: object) -> Literal["5m", "1h"]: return "1h" if (_fields(cache_control) or {}).get("ttl") == "1h" else "5m" -def _kind(path: _Path, cache_control: object, translated: frozenset[_Path]) -> _Kind: - if path not in translated: - return "written" - return "free" if (_fields(cache_control) or {}).get("ttl") is None else "explicit" +def _kind(cache_control: object, from_breakpoint: bool) -> _Kind: + if (_fields(cache_control) or {}).get("ttl") is None: + return "free" + return "explicit" if from_breakpoint else "written" def _chosen(mark: _Placed, ttl: Literal["5m", "1h"]) -> bool: @@ -514,9 +518,9 @@ def _loosened(marks: Sequence[_Placed], drop_params: bool) -> tuple[_Placed, ... def _settled(marks: Sequence[_Placed], drop_params: bool) -> tuple[_Placed, ...]: - """A breakpoint without a ttl takes 1h when Claude may read a 1h mark after it and no 5m mark before it, so the - request stays valid for Claude; under ``drop_params`` a ttl the client set on a breakpoint that breaks that order - is dropped first.""" + """A mark without a ttl, a breakpoint's or a ``cache_control`` the request already carries, takes 1h when Claude + may read a 1h mark after it and no 5m mark before it, so the request stays valid for Claude; under + ``drop_params`` a ttl the client set on a breakpoint that breaks that order is dropped first.""" loosened: Final = _loosened(marks, drop_params) first_five: Final = _first_five(loosened) last_hour: Final = max((mark.last_read for mark in loosened if _chosen(mark, "1h")), default=(-1,)) @@ -530,22 +534,14 @@ def _settled(marks: Sequence[_Placed], drop_params: bool) -> tuple[_Placed, ...] def _route_marks( tool_config_cache_control: ChatCompletionCachedContent | None, request_mark: object -) -> tuple[tuple[_Path, object, _Kind], ...]: - """The cachePoint a route appends after the tools takes 1h like a breakpoint when it sets no ttl. Neither it nor - the request-level mark is ever dropped.""" - tool_config_kind: Final[_Kind] = ( - "free" if tool_config_cache_control is None or tool_config_cache_control.get("ttl") is None else "written" - ) +) -> tuple[tuple[_Path, object, bool], ...]: + """Neither the cachePoint a route appends after the tools nor the request-level mark is ever dropped.""" return ( - *(() if tool_config_cache_control is None else ((_TOOL_CONFIG, tool_config_cache_control, tool_config_kind),)), - *(() if request_mark is None else ((_REQUEST, request_mark, "written"),)), + *(() if tool_config_cache_control is None else ((_TOOL_CONFIG, tool_config_cache_control, False),)), + *(() if request_mark is None else ((_REQUEST, request_mark, False),)), ) -def _droppable(mark: _Placed) -> bool: - return mark.kind != "written" and mark.path != _TOOL_CONFIG - - def claude_prompt_cache( messages: Sequence[object], tools: Sequence[object] | None, @@ -554,13 +550,17 @@ def claude_prompt_cache( drop_params: bool, tool_history_as_text: bool, tool_config_cache_control: ChatCompletionCachedContent | None, + function_marks_on_wire: bool, + pairs_unanswered_tool_calls: bool, ) -> ClaudePromptCache | UnsupportedPromptCacheParam: """``prompt_cache_breakpoint`` marks renamed to ``cache_control``. A mark Claude cannot honor is reported unless ``drop_params``, which keeps what Claude can use and, when the request ends up with more than Claude's four breakpoints, drops the earliest translated ones. A request-level ``cache_control`` in ``params`` and the ``tool_config_cache_control`` cachePoint a route appends after the tools each take one of the four, and marks on - tool history a ``tool_history_as_text`` route flattens take none. A request carrying no breakpoint is returned as - written.""" + tool history a ``tool_history_as_text`` route flattens, marks under a tool's ``function`` on a route without + ``function_marks_on_wire``, and marks on a tool_search call nobody answered, unless the route + ``pairs_unanswered_tool_calls`` with a placeholder result, take none. A request carrying no breakpoint is returned + as written.""" listed_tools: Final = tools or () if not any(map(_message_carries_breakpoint, messages)) and not any(map(_tool_carries_breakpoint, listed_tools)): return ClaudePromptCache(tuple(messages), None if tools is None else tuple(tools)) @@ -588,14 +588,16 @@ def claude_prompt_cache( translated: Final = _translated(hoisted_messages, hoisted_tools) leading_system_count: Final = _leading_system_count(messages) run_ends: Final = _run_ends(messages) - unsent_call_ids: Final = unanswered_tool_search_call_ids(messages) + unsent_call_ids: Final = frozenset() if pairs_unanswered_tool_calls else unanswered_tool_search_call_ids(messages) request_mark: Final = params.get("cache_control") - placements: Final[tuple[tuple[_Path, object, _Kind], ...]] = ( + placements: Final[tuple[tuple[_Path, object, bool], ...]] = ( *( - (path, block.get("cache_control"), _kind(path, block.get("cache_control"), translated)) + (path, block.get("cache_control"), path in translated) for path, block in _markable(renamed_messages, renamed_tools) if block.get("cache_control") is not None - and _on_wire(path, renamed_messages, renamed_tools, unsent_call_ids, tool_history_as_text) + and _on_wire( + path, renamed_messages, renamed_tools, unsent_call_ids, tool_history_as_text, function_marks_on_wire + ) ), *_route_marks(tool_config_cache_control, request_mark), ) @@ -604,17 +606,18 @@ def claude_prompt_cache( _Placed( path, _ttl(cache_control), - kind, + _kind(cache_control, from_breakpoint), + from_breakpoint, _wire_position(path, leading_system_count), _last_read(path, messages, leading_system_count, run_ends), ) - for path, cache_control, kind in placements + for path, cache_control, from_breakpoint in placements ), key=lambda mark: mark.first_read, ) excess: Final = len(marks) - MAX_CACHE_CONTROL_BLOCKS dropped: Final[frozenset[_Path]] = ( - frozenset(tuple(mark.path for mark in marks if _droppable(mark))[:excess]) + frozenset(tuple(mark.path for mark in marks if mark.translated)[:excess]) if excess > 0 and drop_params else frozenset() ) diff --git a/litellm/main.py b/litellm/main.py index 221a1e73505..184d7b3407a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1110,14 +1110,22 @@ def _reaches_converse( custom_llm_provider: str | None, model: str, route_params: Mapping[str, object], - additional_drop_params: Sequence[str] | None, + additional_drop_params: object, ) -> bool: """A Bedrock model on the native Chat Completions route still reaches Converse when the request needs it.""" if isinstance(provider_config, AmazonConverseConfig): return True + if custom_llm_provider != "bedrock": + return False return ( - custom_llm_provider == "bedrock" - and bedrock_route_for_request(model, route_params, additional_drop_params) == "converse" + bedrock_route_for_request( + model, + route_params, + None + if additional_drop_params is None + else _ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(additional_drop_params), + ) + == "converse" ) @@ -5646,14 +5654,12 @@ def completion( and normalize_drop_params(cast(object, kwargs["drop_params"])) is True # cast-ok: untyped request kwargs ) request_tools: Final = cast(list[object] | None, tools) # cast-ok: completion types tools as a bare List - converse_may_send_tools: Final = AmazonConverseConfig.may_send_tools( - tools=request_tools, - functions=cast(object, functions), # cast-ok: completion types functions as a bare List - response_format=cast(object, response_format), # cast-ok: completion types response_format as a bare dict - web_search_options=web_search_options, - messages=routed_messages, + claude_cache_target: Final = provider_config is not None and _is_claude_prompt_cache_target( + custom_llm_provider=custom_llm_provider, + model=model, + base_model=base_model if isinstance(base_model, str) else None, ) - reaches_converse: Final = _reaches_converse( + reaches_converse: Final = claude_cache_target and _reaches_converse( provider_config, custom_llm_provider, model, @@ -5668,11 +5674,14 @@ def completion( **non_default_params, } ), - ( - _ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(kwargs["additional_drop_params"]) - if kwargs.get("additional_drop_params") is not None - else None - ), + kwargs.get("additional_drop_params"), + ) + converse_may_send_tools: Final = reaches_converse and AmazonConverseConfig.may_send_tools( + tools=request_tools, + functions=cast(object, functions), # cast-ok: completion types functions as a bare List + response_format=cast(object, response_format), # cast-ok: completion types response_format as a bare dict + web_search_options=web_search_options, + messages=routed_messages, ) claude_cache: Final = ( resolved_claude_prompt_cache( @@ -5687,19 +5696,16 @@ def completion( AmazonConverseConfig.tool_config_cache_control( non_default_params.get("cache_control_injection_points") ) - if reaches_converse and converse_may_send_tools + if converse_may_send_tools else None ), + function_marks_on_wire=not reaches_converse, + pairs_unanswered_tool_calls=litellm.modify_params is True, ), model=model, llm_provider=custom_llm_provider, ) - if provider_config is not None - and _is_claude_prompt_cache_target( - custom_llm_provider=custom_llm_provider, - model=model, - base_model=base_model if isinstance(base_model, str) else None, - ) + if claude_cache_target else None ) provider_messages: Final = cast( # cast-ok: completion() only renames marks and drops tool_search calls diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index d9b17529ee6..00436cd87b2 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -28,6 +28,10 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( _convert_to_bedrock_tool_call_invoke, convert_to_anthropic_tool_invoke, ) +from litellm.llms.anthropic.chat.prompt_cache_breakpoints import ( + ClaudePromptCache, + claude_prompt_cache, +) from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantToolCall @@ -1272,6 +1276,53 @@ def test_cache_control_hook_counts_a_server_tool_call_mark_only_when_anthropic_r assert (census, _wire_cache_marks(messages, model)) == (expected_breakpoints, expected_breakpoints) +def _translated_for_claude(messages: list[AllMessageValues], modify_params: bool) -> list[AllMessageValues]: + outcome: Final = claude_prompt_cache( + messages, + None, + {}, + drop_params=False, + tool_history_as_text=False, + tool_config_cache_control=None, + function_marks_on_wire=True, + pairs_unanswered_tool_calls=modify_params, + ) + assert isinstance(outcome, ClaudePromptCache), outcome + return cast(list[AllMessageValues], list(outcome.messages)) + + +@pytest.mark.parametrize( + ("follow_ups", "modify_params", "expected_breakpoints"), + [ + pytest.param((), False, 0, id="trailing_call_rebuilt_from_its_kept_result"), + pytest.param((), True, 1, id="trailing_call_replayed_with_the_modify_params_placeholder"), + pytest.param((_SEARCH_ANSWER,), False, 1, id="replayed_with_the_client_answer"), + pytest.param((_SEARCH_ANSWER, _GO_ON), False, 0, id="client_answer_replaced_by_the_kept_result"), + ], +) +def test_cache_control_hook_counts_a_breakpoint_on_a_server_tool_call_only_when_claude_receives_its_mark( + monkeypatch: pytest.MonkeyPatch, + follow_ups: tuple[AllMessageValues, ...], + modify_params: bool, + expected_breakpoints: int, +): + monkeypatch.setattr(litellm, "modify_params", modify_params) + search_call: Final = { + **{key: value for key, value in _SEARCH_CALL.items() if key != "cache_control"}, + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + messages: Final = [ + cast(AllMessageValues, {"role": "user", "content": "Weather in Paris?"}), + _replayed_search_call(search_call), + *follow_ups, + ] + + census: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages) + received: Final = _wire_cache_marks(_translated_for_claude(messages, modify_params)) + + assert (census, received) == (expected_breakpoints, expected_breakpoints) + + @pytest.mark.parametrize( "after_the_call", [(_SEARCH_ANSWER, _SYSTEM_NOTE), (_SYSTEM_NOTE, _SEARCH_ANSWER)], diff --git a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py index 680f6f49dcb..8088ae4ef15 100644 --- a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py +++ b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py @@ -24,6 +24,8 @@ def _cache( *, drop_params: bool = False, tool_config_cache_control: ChatCompletionCachedContent | None = None, + function_marks_on_wire: bool = True, + pairs_unanswered_tool_calls: bool = False, ) -> ClaudePromptCache | UnsupportedPromptCacheParam: return claude_prompt_cache( messages, @@ -32,6 +34,8 @@ def _cache( drop_params=drop_params, tool_history_as_text=False, tool_config_cache_control=tool_config_cache_control, + function_marks_on_wire=function_marks_on_wire, + pairs_unanswered_tool_calls=pairs_unanswered_tool_calls, ) @@ -42,9 +46,17 @@ def _resolved( *, drop_params: bool = False, tool_config_cache_control: ChatCompletionCachedContent | None = None, + function_marks_on_wire: bool = True, + pairs_unanswered_tool_calls: bool = False, ) -> ClaudePromptCache: outcome: Final = _cache( - messages, tools, params, drop_params=drop_params, tool_config_cache_control=tool_config_cache_control + messages, + tools, + params, + drop_params=drop_params, + tool_config_cache_control=tool_config_cache_control, + function_marks_on_wire=function_marks_on_wire, + pairs_unanswered_tool_calls=pairs_unanswered_tool_calls, ) assert isinstance(outcome, ClaudePromptCache), outcome return outcome @@ -191,7 +203,14 @@ def test_a_breakpoint_a_client_cache_control_wins_over_is_still_reported( assert outcome.value == _THIRTY_MINUTES -def test_a_breakpoint_on_a_tool_search_call_nobody_answered_takes_no_slot_from_the_system_mark_under_drop_params(): +@pytest.mark.parametrize( + ("pairs_unanswered_tool_calls", "system_mark"), + [(False, {"cache_control": _EPHEMERAL}), (True, {})], + ids=["dropped_unanswered", "answered_with_a_placeholder"], +) +def test_a_breakpoint_on_a_tool_search_call_nobody_answered_takes_a_slot_only_when_the_route_answers_it( + pairs_unanswered_tool_calls: bool, system_mark: dict[str, object] +): search_call: Final = { "id": "srvtoolu_01search", "type": "function", @@ -205,9 +224,47 @@ def test_a_breakpoint_on_a_tool_search_call_nobody_answered_takes_no_slot_from_t {"role": "user", "content": "Which section mentions watchlists?"}, ] - result: Final = _resolved(messages, drop_params=True) + result: Final = _resolved(messages, drop_params=True, pairs_unanswered_tool_calls=pairs_unanswered_tool_calls) - assert result.messages[0] == {"role": "system", "content": [_part("sys", cache_control=_EPHEMERAL)]} + assert result.messages[0] == {"role": "system", "content": [_part("sys", **system_mark)]} + + +def test_a_cache_control_without_a_ttl_takes_the_hour_a_later_breakpoint_asks_for(): + result: Final = _resolved( + [ + {"role": "system", "content": [_part("sys", cache_control=_EPHEMERAL)]}, + {"role": "user", "content": [_part("ask", prompt_cache_breakpoint={"mode": "explicit", "ttl": "1h"})]}, + ] + ) + + assert list(result.messages) == [ + {"role": "system", "content": [_part("sys", cache_control=_ONE_HOUR)]}, + {"role": "user", "content": [_part("ask", cache_control=_ONE_HOUR)]}, + ] + + +@pytest.mark.parametrize( + ("function_marks_on_wire", "system_mark"), + [(True, _EPHEMERAL), (False, _ONE_HOUR)], + ids=["route_sends_function_marks", "route_drops_function_marks"], +) +def test_a_five_minute_mark_under_a_tools_function_holds_back_an_hour_only_on_a_route_that_sends_it( + function_marks_on_wire: bool, system_mark: dict[str, object] +): + five_minutes: Final = {"type": "ephemeral", "ttl": "5m"} + tool: Final = {"type": "function", "function": {"name": "t", "cache_control": five_minutes}} + + result: Final = _resolved( + [ + {"role": "system", "content": [_part("sys", prompt_cache_breakpoint=_EXPLICIT)]}, + {"role": "user", "content": [_part("ask", prompt_cache_breakpoint={"mode": "explicit", "ttl": "1h"})]}, + ], + [tool], + function_marks_on_wire=function_marks_on_wire, + ) + + assert result.messages[0] == {"role": "system", "content": [_part("sys", cache_control=system_mark)]} + assert list(result.tools or ()) == [tool] @pytest.mark.parametrize( @@ -341,10 +398,10 @@ def test_a_breakpoint_read_after_the_last_one_hour_mark_keeps_the_default_ttl(): ("system", "ask", "as_written", "under_drop_params"), [ ( - {"cache_control": _EPHEMERAL}, + {"cache_control": {"type": "ephemeral", "ttl": "5m"}}, {"prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"}}, - ({"cache_control": _EPHEMERAL}, {"cache_control": _ONE_HOUR}), - ({"cache_control": _EPHEMERAL}, {"cache_control": _EPHEMERAL}), + ({"cache_control": {"type": "ephemeral", "ttl": "5m"}}, {"cache_control": _ONE_HOUR}), + ({"cache_control": {"type": "ephemeral", "ttl": "5m"}}, {"cache_control": _EPHEMERAL}), ), ( {"prompt_cache_breakpoint": {"mode": "explicit", "ttl": "5m"}}, diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index ec64c97a521..6a1dc3fed37 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -759,6 +759,29 @@ def test_a_breakpoint_reaches_bedrock_mantle_claude_as_cache_control(respx_mock: ] +def test_a_string_additional_drop_params_still_reaches_claude_on_the_anthropic_route(respx_mock: respx.MockRouter): + route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").respond( + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-opus-5-5", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ) + + response: Final = litellm.completion( + model="anthropic/claude-opus-5-5", + messages=[{"role": "user", "content": "Say hi."}], + api_key="fake", + additional_drop_params="temperature", + ) + + assert (route.call_count, response.choices[0].message.content) == (1, "hi") + + _WEATHER_TOOL: Final = {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}