diff --git a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py index 0cd41fc25b5..3eb6d688bea 100644 --- a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py +++ b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py @@ -151,11 +151,25 @@ def _marked_message(index: int, message: object, mark: _Mark) -> object: } +def _with_function_breakpoint_hoisted(tool: _Fields) -> _Fields: + """Converse reads only a tool's top-level mark, so a breakpoint written under ``function`` moves up.""" + function: Final = _fields(tool.get("function")) + marker: Final = None if function is None else function.get("prompt_cache_breakpoint") + if function is None or marker is None: + return tool + keeps_client_mark: Final = function.get("cache_control") is not None or "prompt_cache_breakpoint" in tool + return { + **tool, + "function": {key: value for key, value in function.items() if key != "prompt_cache_breakpoint"}, + **({} if keeps_client_mark else {"prompt_cache_breakpoint": marker}), + } + + def _marked_tool(index: int, tool: object, mark: _Mark) -> object: fields: Final = _fields(tool) if fields is None: return tool - marked: Final = mark(("tools", index), fields) + marked: Final = mark(("tools", index), _with_function_breakpoint_hoisted(fields)) function: Final = _fields(marked.get("function")) return marked if function is None else {**marked, "function": mark(("tools", index, "function"), function)} diff --git a/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_wire.py b/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_wire.py index 445f0c3840d..0ebc93bf1bb 100644 --- a/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_wire.py +++ b/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_wire.py @@ -192,6 +192,21 @@ def test_breakpoint_becomes_a_cache_mark_on_the_same_block(gateway: Gateway, pro assert _labels(provider, request) == [_expected(provider, site, marker)] +@pytest.mark.parametrize("provider", _PROVIDERS) +def test_breakpoint_under_a_tool_function_marks_the_tool(gateway: Gateway, provider: Provider) -> None: + marker: Final = new_marker() + function: Final = TOOL["function"] + assert isinstance(function, dict), TOOL + tool: Final[dict[str, JsonValue]] = {**TOOL, "function": {**function, "prompt_cache_breakpoint": _EXPLICIT}} + with wire_server(lambda request: _respond(provider, request)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(gateway, scenario, wire, provider) + status, _, text = post_chat(gateway, {**_body(model, marker, _marked(())), "tools": [tool]}) + assert status == 200, text + request: Final = _only_request(wire) + assert b"prompt_cache_breakpoint" not in request.body, request.body + assert _labels(provider, request) == [_expected(provider, "tool", marker)] + + def test_breakpoint_ttl_claude_supports_is_kept(gateway: Gateway) -> None: marker: Final = new_marker() marks: Final = _marked(("user_part",), {"prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"}}) diff --git a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py index 39b15201375..b1446573831 100644 --- a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py +++ b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py @@ -78,7 +78,7 @@ def test_breakpoints_become_cache_control_on_every_block_that_carries_one(): ] assert list(result.tools or ()) == [ {"type": "function", "function": {"name": "a"}, "cache_control": _EPHEMERAL}, - {"type": "function", "function": {"name": "b", "cache_control": _EPHEMERAL}}, + {"type": "function", "function": {"name": "b"}, "cache_control": _EPHEMERAL}, ]