diff --git a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py index 9b601933023..81ddfc1d185 100644 --- a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py +++ b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py @@ -674,8 +674,7 @@ def _converse_tool_config_order(tools: Sequence[object], late_tools: frozenset[i for index, tool in enumerate(tools) if index not in late_tools and _in_converse_tool_config(_fields(tool) or {}) ) - system_tools: Final = tuple(index for index in sent if "systemTool" in (_fields(tools[index]) or {})) - return (*(index for index in sent if index not in system_tools), *system_tools) + return tuple(sorted(sent, key=lambda index: "systemTool" in (_fields(tools[index]) or {}))) def _sets_a_ttl(cache_control: object) -> bool: @@ -711,14 +710,28 @@ def _route_plan( drop_params: bool, ) -> _Plan: """Converse leaves out a ``tool_config`` cachePoint with a ttl that would join the ``tail`` of its tool config when - a cachePoint there sets a ttl too. That request is planned again with the tail's ttl kept in its place.""" + a cachePoint there sets a ttl too. That request is planned again with the tail's ttl kept in its place. A passed + through cachePoint with a ttl takes the place of the translated mark on the tool just before it, which is not + sent, and of the ``tool_config`` cachePoint, which Bedrock joins to it, when that mark keeps its ttl or sets none, + or under ``drop_params``.""" + passed_through: Final = next(filter(_sets_a_ttl, tail.cache_points), None) + covered: Final = next((mark for mark in marks if mark.path == tail.tool and mark.kind != "written"), None) + if ( + passed_through is not None + and covered is not None + and (drop_params or covered.kind == "free" or covered.ttl == _ttl(passed_through)) + ): + stand_in: Final = replace(covered, ttl=_ttl(passed_through), kind="written") + covered_plan: Final = _planned( + tuple(stand_in if mark is covered else mark for mark in marks if mark.path != _TOOL_CONFIG), drop_params + ) + return replace(covered_plan, dropped=covered_plan.dropped | {covered.path}) plan: Final = _planned(marks, drop_params) tool_mark: Final = ( None if tail.tool is None else _planned_cache_control(plan, tail.tool, (_fields(tools[int(tail.tool[1])]) or {}).get("cache_control")) ) - passed_through: Final = next(filter(_sets_a_ttl, tail.cache_points), None) tail_cache_control: Final = tool_mark if passed_through is None else passed_through if not ( _sets_a_ttl(tail_cache_control) @@ -759,13 +772,15 @@ def claude_prompt_cache( tool_history_as_text: bool, tool_config_cache_control: ChatCompletionCachedContent | None, route: ClaudeRoute, + adds_a_tool: bool, pairs_unanswered_tool_calls: bool, ) -> ClaudePromptCache | UnsupportedPromptCacheParam: """``prompt_cache_breakpoint`` marks renamed to ``cache_control``. A mark Claude cannot honor is reported unless ``drop_params``, which keeps what Claude can use and, when the request ends up with more than Claude's four breakpoints, drops the earliest translated ones. A request-level ``cache_control`` in ``params`` and the ``tool_config_cache_control`` cachePoint a route appends after the tools each take one of the four, unless Converse - leaves that cachePoint out, and marks on tool history a ``tool_history_as_text`` route flattens, marks on a tool the + leaves that cachePoint out or sends no tool in its tool config, counting the tool it adds itself when it + ``adds_a_tool``, and marks on tool history a ``tool_history_as_text`` route flattens, marks on a tool the ``route`` sends without one or under a tool's ``function`` on Converse, and marks on a tool_search call nobody answered, unless the route ``pairs_unanswered_tool_calls`` with a placeholder result, take none. A ``cache_control`` on a ``namespace`` tool's member takes one off Converse and is never changed. A request carrying no breakpoint is @@ -803,7 +818,7 @@ def claude_prompt_cache( unsent_call_ids: Final = frozenset() if pairs_unanswered_tool_calls else unanswered_tool_search_call_ids(messages) request_mark: Final = params.get("cache_control") tool_config_order: Final = _converse_tool_config_order(renamed_tools, late_tools) - sent_tool_config_cache_control: Final = tool_config_cache_control if tool_config_order or not listed_tools else None + sent_tool_config_cache_control: Final = tool_config_cache_control if tool_config_order or adds_a_tool else None placements: Final[tuple[tuple[_Path, object, _Kind], ...]] = ( *( (path, block.get("cache_control"), _kind(path, block.get("cache_control"), translated)) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index fd0175de22d..8d44882a1cc 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -264,6 +264,13 @@ class AmazonConverseConfig(BaseConfig): "tools" not in optional_params and AmazonConverseConfig._adds_a_dummy_tool(messages) ) + @staticmethod + def adds_a_tool(optional_params: Mapping[str, object], messages: Sequence[object]) -> bool: + return any( + message_field(message_field(tool, "function"), "name") == RESPONSE_FORMAT_TOOL_NAME + for tool in parts_of(optional_params.get("tools")) + ) or ("tools" not in optional_params and AmazonConverseConfig._adds_a_dummy_tool(messages)) + @staticmethod def _adds_a_dummy_tool(messages: Sequence[object]) -> bool: return ( diff --git a/litellm/main.py b/litellm/main.py index df6d3df43eb..be3b7d5397c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1136,6 +1136,16 @@ def _claude_route(provider_config: BaseConfig | None, reaches_converse: bool) -> return "anthropic" +def _converse_tools(converse_params: Mapping[str, object] | None, messages: Sequence[object]) -> tuple[bool, bool]: + """Whether Converse sends a tool in its tool config, and whether it adds one of its own there.""" + if converse_params is None: + return False, False + return ( + AmazonConverseConfig.sends_tools(converse_params, messages), + AmazonConverseConfig.adds_a_tool(converse_params, messages), + ) + + def _listed_drop_params(additional_drop_params: object) -> tuple[str, ...]: if not isinstance(additional_drop_params, (list, tuple)): return () @@ -5763,18 +5773,16 @@ def completion( accepts_summary=provider_config is not None and provider_config.accepts_reasoning_summary_in_reasoning_effort, ) - converse_sends_tools: Final = ( - reaches_converse - and carries_prompt_cache_breakpoint(routed_messages, request_tools) - and AmazonConverseConfig.sends_tools( - _map_request_params( - **{**request_param_args, "reasoning_effort": provider_reasoning_effort}, - model_info=validated_model_info, - **non_default_params, - ), - routed_messages, + converse_params: Final = ( + _map_request_params( + **{**request_param_args, "reasoning_effort": provider_reasoning_effort}, + model_info=validated_model_info, + **non_default_params, ) + if reaches_converse and carries_prompt_cache_breakpoint(routed_messages, request_tools) + else None ) + converse_sends_tools, converse_adds_a_tool = _converse_tools(converse_params, routed_messages) claude_cache: Final = ( resolved_claude_prompt_cache( claude_prompt_cache( @@ -5792,6 +5800,7 @@ def completion( else None ), route=_claude_route(provider_config, reaches_converse), + adds_a_tool=converse_adds_a_tool, pairs_unanswered_tool_calls=litellm.modify_params is True, ), model=model, diff --git a/tests/benchmarks/test_prompt_cache_breakpoint_benchmarks.py b/tests/benchmarks/test_prompt_cache_breakpoint_benchmarks.py index 948ae0d2709..6645711a8aa 100644 --- a/tests/benchmarks/test_prompt_cache_breakpoint_benchmarks.py +++ b/tests/benchmarks/test_prompt_cache_breakpoint_benchmarks.py @@ -3,7 +3,7 @@ from typing import Final import pytest -from litellm.llms.anthropic.chat.prompt_cache_breakpoints import ClaudePromptCache, claude_prompt_cache +from litellm.llms.anthropic.chat.prompt_cache_breakpoints import ClaudePromptCache, ClaudeRoute, claude_prompt_cache _EXPLICIT: Final = {"mode": "explicit"} @@ -25,16 +25,24 @@ _ONE_HOUR_PARTS_BESIDE_A_BREAKPOINT: Final = ( }, ) +_SYSTEM_TOOLS_BESIDE_A_MARKED_TOOL: Final = ( + *({"systemTool": {"name": f"s{index}"}} for index in range(20000)), + {"type": "function", "function": {"name": "t"}, "prompt_cache_breakpoint": _EXPLICIT}, +) -def _settled_under_drop_params(messages: Sequence[object]) -> None: + +def _settled_under_drop_params( + messages: Sequence[object], tools: Sequence[object] | None = None, route: ClaudeRoute = "anthropic" +) -> None: outcome: Final = claude_prompt_cache( messages, - None, + tools, {}, drop_params=True, tool_history_as_text=False, tool_config_cache_control=None, - route="anthropic", + route=route, + adds_a_tool=False, pairs_unanswered_tool_calls=False, ) assert isinstance(outcome, ClaudePromptCache) @@ -48,3 +56,8 @@ def test_settling_thousands_of_marked_system_turns_under_drop_params(): @pytest.mark.benchmark def test_settling_a_breakpoint_beside_thousands_of_one_hour_parts_under_drop_params(): _settled_under_drop_params(_ONE_HOUR_PARTS_BESIDE_A_BREAKPOINT) + + +@pytest.mark.benchmark +def test_settling_a_tool_breakpoint_beside_thousands_of_bedrock_system_tools_under_drop_params(): + _settled_under_drop_params(({"role": "user", "content": "hi"},), _SYSTEM_TOOLS_BESIDE_A_MARKED_TOOL, "converse") diff --git a/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_placement_wire.py b/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_placement_wire.py index f0e64faf856..8ebd7118a71 100644 --- a/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_placement_wire.py +++ b/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_placement_wire.py @@ -867,6 +867,20 @@ _MARKED_TOOL: Final[dict[str, JsonValue]] = {**TOOL, "prompt_cache_breakpoint": None, id="cache-point-with-five-minutes-listed-last", ), + pytest.param( + [_MARKED_TOOL, {"cachePoint": {"type": "default", "ttl": "1h"}}], + True, + (Mark(_TOOL_LABEL, "1h"),), + "1h", + id="cache-point-with-an-hour-right-after-the-marked-tool", + ), + pytest.param( + [_MARKED_TOOL, {"cachePoint": {"type": "default", "ttl": "5m"}}], + True, + (Mark(_TOOL_LABEL, "5m"),), + None, + id="cache-point-with-five-minutes-right-after-the-marked-tool", + ), pytest.param( [_MARKED_TOOL, {"cachePoint": {"type": "default"}}], True, @@ -901,6 +915,60 @@ def test_a_five_minute_tool_config_cache_point_shortens_later_breakpoints_only_w ) +@pytest.mark.parametrize( + ("tool_breakpoint", "cache_point_ttl"), + [ + pytest.param(_ONE_HOUR_BREAKPOINT, "5m", id="hour-breakpoint-before-a-five-minute-cache-point"), + pytest.param(_FIVE_MINUTE_BREAKPOINT, "1h", id="five-minute-breakpoint-before-an-hour-cache-point"), + ], +) +def test_a_tool_breakpoint_before_a_cache_point_with_another_ttl_gets_bedrocks_400_without_drop_params( + gateway: Gateway, tool_breakpoint: dict[str, JsonValue], cache_point_ttl: str +) -> None: + marker: Final = new_marker() + tools: Final[list[JsonValue]] = [ + {**TOOL, "prompt_cache_breakpoint": tool_breakpoint}, + {"cachePoint": {"type": "default", "ttl": cache_point_ttl}}, + ] + with wire_server(lambda request: respond("bedrock_converse", request)) as wire, gateway.scenario() as scenario: + model: Final = deployment(gateway, scenario, wire, "bedrock_converse") + status, _, text = post_chat(gateway, chat_body(model, [_final(marker)], tools=tools)) + request: Final = only_request(wire) + assert status == 400, text + assert "conflicts with the existing field ttl" in text, text + assert sent_marks("bedrock_converse", request) == ( + Mark(_TOOL_LABEL, tool_breakpoint["ttl"]), + Mark("other", cache_point_ttl), + ) + + +def test_a_five_minute_tool_config_cache_point_after_the_response_format_tool_shortens_later_breakpoints( + gateway: Gateway, +) -> None: + marker: Final = new_marker() + turns: Final[list[JsonValue]] = [ + {"role": "system", "content": [_text(SYSTEM, prompt_cache_breakpoint=_EXPLICIT)]}, + _final(marker, prompt_cache_breakpoint=_ONE_HOUR_BREAKPOINT), + ] + marks: Final = _marks_sent( + gateway, + "bedrock_converse", + lambda model: chat_body( + model, + turns, + tools=[{"type": "web_fetch_20250910", "name": "web_fetch", "cache_control": _ONE_HOUR}], + drop_params=True, + response_format=_JSON_SCHEMA, + ), + cache_control_injection_points=[{"location": "tool_config", "control": {"type": "ephemeral", "ttl": "5m"}}], + ) + assert marks == ( + Mark("tool:json_tool_call", "5m"), + Mark(_label(SYSTEM), None), + Mark(_label(final_text(marker)), None), + ) + + @pytest.mark.parametrize( ("deployment_drops", "expected"), ( diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 88e7ab508a6..cc73c6d068a 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -1286,6 +1286,7 @@ def _translated_for_claude(messages: list[AllMessageValues], modify_params: bool tool_history_as_text=False, tool_config_cache_control=None, route="anthropic", + adds_a_tool=False, pairs_unanswered_tool_calls=modify_params, ) assert isinstance(outcome, ClaudePromptCache), outcome diff --git a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py index 406afb7b1b4..80b0659df26 100644 --- a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py +++ b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py @@ -25,6 +25,7 @@ def _cache( drop_params: bool = False, tool_config_cache_control: ChatCompletionCachedContent | None = None, route: ClaudeRoute = "anthropic", + adds_a_tool: bool = False, pairs_unanswered_tool_calls: bool = False, ) -> ClaudePromptCache | UnsupportedPromptCacheParam: return claude_prompt_cache( @@ -35,6 +36,7 @@ def _cache( tool_history_as_text=False, tool_config_cache_control=tool_config_cache_control, route=route, + adds_a_tool=adds_a_tool, pairs_unanswered_tool_calls=pairs_unanswered_tool_calls, ) @@ -47,6 +49,7 @@ def _resolved( drop_params: bool = False, tool_config_cache_control: ChatCompletionCachedContent | None = None, route: ClaudeRoute = "anthropic", + adds_a_tool: bool = False, pairs_unanswered_tool_calls: bool = False, ) -> ClaudePromptCache: outcome: Final = _cache( @@ -56,6 +59,7 @@ def _resolved( drop_params=drop_params, tool_config_cache_control=tool_config_cache_control, route=route, + adds_a_tool=adds_a_tool, pairs_unanswered_tool_calls=pairs_unanswered_tool_calls, ) assert isinstance(outcome, ClaudePromptCache), outcome @@ -1308,6 +1312,152 @@ def test_later_one_hour_breakpoints_are_shortened_only_when_converses_tool_confi ] +@pytest.mark.parametrize( + ("tools", "sent_tools"), + [ + pytest.param(None, None, id="no-listed-tool"), + pytest.param( + [{"type": "web_fetch_20250910", "name": "web_fetch", "cache_control": _ONE_HOUR}], + [{"type": "web_fetch_20250910", "name": "web_fetch", "cache_control": _ONE_HOUR}], + id="a-listed-tool-converse-drops", + ), + pytest.param( + [{"type": "bash_20250124", "name": "bash", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"}}], + [{"type": "bash_20250124", "name": "bash", "cache_control": _EPHEMERAL}], + id="a-computer-use-tool-converse-sends-outside-the-tool-config", + ), + ], +) +def test_later_one_hour_breakpoints_are_shortened_after_the_five_minute_cache_point_beside_the_tool_converse_adds( + tools: list[object] | None, sent_tools: list[object] | None +): + result: Final = _resolved( + [ + {"role": "system", "content": [_part("s", prompt_cache_breakpoint=_EXPLICIT)]}, + {"role": "user", "content": [_part("ask", prompt_cache_breakpoint={"mode": "explicit", "ttl": "1h"})]}, + ], + tools, + drop_params=True, + tool_config_cache_control=ChatCompletionCachedContent(type="ephemeral", ttl="5m"), + route="converse", + adds_a_tool=True, + ) + + assert result.tools == (None if sent_tools is None else tuple(sent_tools)) + assert list(result.messages) == [ + {"role": "system", "content": [_part("s", cache_control=_EPHEMERAL)]}, + {"role": "user", "content": [_part("ask", cache_control=_EPHEMERAL)]}, + ] + + +@pytest.mark.parametrize( + "tool_config", + [None, ChatCompletionCachedContent(type="ephemeral", ttl="5m")], + ids=["no-tool-config-cache-point", "five-minute-tool-config-cache-point"], +) +@pytest.mark.parametrize( + ("tool_breakpoint", "cache_point_ttl", "drop_params"), + [ + pytest.param({"mode": "explicit", "ttl": "1h"}, "1h", False, id="same-ttl"), + pytest.param({"mode": "explicit", "ttl": "1h"}, "1h", True, id="same-ttl-with-drop-params"), + pytest.param(_EXPLICIT, "1h", False, id="breakpoint-without-a-ttl-before-an-hour"), + pytest.param(_EXPLICIT, "5m", False, id="breakpoint-without-a-ttl-before-five-minutes"), + pytest.param(_EXPLICIT, "5m", True, id="breakpoint-without-a-ttl-with-drop-params"), + pytest.param({"mode": "explicit", "ttl": "1h"}, "5m", True, id="another-ttl-with-drop-params"), + ], +) +def test_a_bedrock_cache_point_with_a_ttl_right_after_a_tool_breakpoint_is_the_only_one_sent_there( + tool_breakpoint: dict[str, object], + cache_point_ttl: str, + tool_config: ChatCompletionCachedContent | None, + drop_params: bool, +): + cache_point: Final = {"cachePoint": {"type": "default", "ttl": cache_point_ttl}} + + result: Final = _resolved( + [{"role": "user", "content": "hi"}], + [{"type": "function", "function": {"name": "t"}, "prompt_cache_breakpoint": tool_breakpoint}, cache_point], + drop_params=drop_params, + tool_config_cache_control=tool_config, + route="converse", + ) + + assert list(result.tools or ()) == [{"type": "function", "function": {"name": "t"}}, cache_point] + + +@pytest.mark.parametrize( + "tool_config", + [None, ChatCompletionCachedContent(type="ephemeral", ttl="5m")], + ids=["no-tool-config-cache-point", "five-minute-tool-config-cache-point"], +) +@pytest.mark.parametrize( + ("tool_breakpoint", "sent_tool_mark", "cache_point_ttl"), + [ + pytest.param({"mode": "explicit", "ttl": "1h"}, _ONE_HOUR, "5m", id="hour-before-five-minutes"), + pytest.param({"mode": "explicit", "ttl": "5m"}, _FIVE_MINUTES, "1h", id="five-minutes-before-an-hour"), + ], +) +def test_a_tool_breakpoint_and_a_bedrock_cache_point_right_after_it_with_another_ttl_are_both_sent_without_drop_params( + tool_breakpoint: dict[str, object], + sent_tool_mark: dict[str, object], + cache_point_ttl: str, + tool_config: ChatCompletionCachedContent | None, +): + cache_point: Final = {"cachePoint": {"type": "default", "ttl": cache_point_ttl}} + + result: Final = _resolved( + [{"role": "user", "content": "hi"}], + [{"type": "function", "function": {"name": "t"}, "prompt_cache_breakpoint": tool_breakpoint}, cache_point], + drop_params=False, + tool_config_cache_control=tool_config, + route="converse", + ) + + assert list(result.tools or ()) == [ + {"type": "function", "function": {"name": "t"}, "cache_control": sent_tool_mark}, + cache_point, + ] + + +def test_a_bedrock_cache_point_with_a_ttl_after_a_tool_breakpoint_keeps_that_breakpoints_slot_of_the_four(): + result: Final = _resolved( + [{"role": "user", "content": [_part(str(index), prompt_cache_breakpoint=_EXPLICIT) for index in range(4)]}], + [ + { + "type": "function", + "function": {"name": "t"}, + "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"}, + }, + {"cachePoint": {"type": "default", "ttl": "1h"}}, + ], + drop_params=True, + route="converse", + ) + + assert list(result.messages) == [ + { + "role": "user", + "content": [_part("0"), *(_part(str(index), cache_control=_EPHEMERAL) for index in range(1, 4))], + } + ] + + +def test_a_client_cache_control_on_a_tool_before_a_bedrock_cache_point_with_a_ttl_is_sent_as_written(): + tools: Final = [ + {"type": "function", "function": {"name": "t"}, "cache_control": _ONE_HOUR}, + {"cachePoint": {"type": "default", "ttl": "1h"}}, + ] + + result: Final = _resolved( + [{"role": "user", "content": [_part("ask", prompt_cache_breakpoint=_EXPLICIT)]}], + tools, + drop_params=True, + route="converse", + ) + + assert list(result.tools or ()) == tools + + @pytest.mark.parametrize( ("route", "sent_bash_mark"), [("anthropic", _ONE_HOUR), ("converse", _EPHEMERAL)], diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 07233b897e5..cd91b946c3f 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -1302,6 +1302,39 @@ def test_a_five_minute_tool_config_cache_point_does_not_shorten_later_breakpoint ) +@pytest.mark.parametrize("response_format_args", _RESPONSE_FORMATS_CONVERSE_SENDS_AS_A_TOOL) +def test_a_five_minute_tool_config_cache_point_after_the_response_format_tool_shortens_breakpoints_when_converse_drops_every_listed_tool( + respx_mock: respx.MockRouter, response_format_args: Mapping[str, object] +): + route: Final = respx_mock.post(url__regex=_CONVERSE_ROUTE).respond(json=_CONVERSE_REPLY) + + litellm.completion( + **{ + "model": "bedrock/converse/us.anthropic.claude-opus-5-5", + "drop_params": True, + **_FAKE_AWS, + **response_format_args, + }, + messages=[ + {"role": "system", "content": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": _BREAKPOINT}]}, + { + "role": "user", + "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": _ONE_HOUR_BREAKPOINT}], + }, + ], + tools=[ + {"type": "web_fetch_20250910", "name": "web_fetch", "cache_control": {"type": "ephemeral", "ttl": "1h"}} + ], + cache_control_injection_points=_FIVE_MINUTE_TOOL_CONFIG, + ) + + assert _converse_cache_point_places(json.loads(route.calls.last.request.content)) == ( + ("tool", "5m"), + ("system", None), + ("message", None), + ) + + @pytest.mark.parametrize( "system_tool_first", [True, False], ids=["system-tool-listed-first", "system-tool-listed-last"] ) @@ -1363,6 +1396,39 @@ def test_a_five_minute_tool_config_cache_point_is_not_sent_after_a_passed_throug ) +@pytest.mark.parametrize( + ("cache_point_ttl", "sent_later_ttl"), [("1h", "1h"), ("5m", None)], ids=["one-hour", "five-minutes"] +) +def test_a_passed_through_cache_point_with_a_ttl_behind_a_one_hour_tool_breakpoint_is_the_only_tool_cache_point_sent( + respx_mock: respx.MockRouter, cache_point_ttl: str, sent_later_ttl: str | None +): + route: Final = respx_mock.post(url__regex=_CONVERSE_ROUTE).respond(json=_CONVERSE_REPLY) + + litellm.completion( + model="bedrock/converse/us.anthropic.claude-opus-5-5", + messages=[ + {"role": "system", "content": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": _BREAKPOINT}]}, + { + "role": "user", + "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": _ONE_HOUR_BREAKPOINT}], + }, + ], + tools=[ + {**_WEATHER_TOOL, "prompt_cache_breakpoint": _ONE_HOUR_BREAKPOINT}, + {"cachePoint": {"type": "default", "ttl": cache_point_ttl}}, + ], + cache_control_injection_points=_FIVE_MINUTE_TOOL_CONFIG, + drop_params=True, + **_FAKE_AWS, + ) + + assert _converse_cache_point_places(json.loads(route.calls.last.request.content)) == ( + ("tool", cache_point_ttl), + ("system", sent_later_ttl), + ("message", sent_later_ttl), + ) + + def test_a_tool_config_cache_point_after_a_tool_mark_without_a_ttl_still_shortens_a_later_one_hour_breakpoint( respx_mock: respx.MockRouter, ): @@ -1416,6 +1482,37 @@ def _sent_cache_ttls(body: Mapping[str, object]) -> tuple[str | None, ...]: return tuple(mark.get("ttl") for mark in marks if mark is not None) +@pytest.mark.parametrize( + ("model", "url", "credentials"), + [ + pytest.param("anthropic/claude-opus-5-5", r"https://api\.anthropic\.com/v1/messages", {"api_key": "fake"}), + pytest.param("bedrock/invoke/us.anthropic.claude-opus-5-5", _INVOKE_ROUTE, _FAKE_AWS), + ], + ids=["anthropic", "bedrock-invoke"], +) +def test_a_five_minute_tool_config_cache_point_does_not_shorten_a_one_hour_breakpoint_on_a_route_that_never_sends_it( + respx_mock: respx.MockRouter, model: str, url: str, credentials: Mapping[str, str] +): + route: Final = respx_mock.post(url__regex=url).respond(json=_CLAUDE_REPLY) + + litellm.completion( + model=model, + messages=[ + {"role": "system", "content": [{"type": "text", "text": "sys"}]}, + { + "role": "user", + "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": _ONE_HOUR_BREAKPOINT}], + }, + ], + tools=[_WEATHER_TOOL], + cache_control_injection_points=_FIVE_MINUTE_TOOL_CONFIG, + drop_params=True, + **credentials, + ) + + assert _sent_cache_ttls(json.loads(route.calls.last.request.content)) == ("1h",) + + def test_a_string_additional_drop_params_on_bedrock_invoke_still_answers_unsupported_params_error( respx_mock: respx.MockRouter, ):