mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-28 06:23:07 +00:00
fix(anthropic): count Converse's own tools and let a raw cachePoint stand in for a tool breakpoint
Converse adds a json_tool_call tool for response_format, or a dummy tool for tool history, and appends the operator's tool_config cachePoint after it even when it drops every listed tool. The cache step now counts that point whenever Converse sends a tool in its tool config, so under drop_params a later 1h breakpoint goes out as 5m instead of getting Claude's ttl order 400. A raw Bedrock cachePoint with a ttl right after a tool breakpoint formed one run with the breakpoint's cachePoint, and Bedrock rejects a run where two points set a ttl. The raw cachePoint now stands in for that breakpoint, which is not sent, and keeps its slot of Claude's four at the raw cachePoint's ttl, the same way a client's cache_control on a block replaces its breakpoint. It does so only when the breakpoint has the same ttl or none, or under drop_params, so without drop_params a breakpoint with another ttl is still sent and gets Bedrock's 400. The step also orders the tools Converse sends with one stable sort instead of a scan per tool. The Converse tool checks move into a helper, since completion() grew past what basedpyright analyzes
This commit is contained in:
mateo-berri
parent
3501903737
commit
f27f48ff0f
8 changed files with 380 additions and 20 deletions
|
|
@ -674,8 +674,7 @@ def _converse_tool_config_order(tools: Sequence[object], late_tools: frozenset[i
|
|||
for index, tool in enumerate(tools)
|
||||
if index not in late_tools and _in_converse_tool_config(_fields(tool) or {})
|
||||
)
|
||||
system_tools: Final = tuple(index for index in sent if "systemTool" in (_fields(tools[index]) or {}))
|
||||
return (*(index for index in sent if index not in system_tools), *system_tools)
|
||||
return tuple(sorted(sent, key=lambda index: "systemTool" in (_fields(tools[index]) or {})))
|
||||
|
||||
|
||||
def _sets_a_ttl(cache_control: object) -> bool:
|
||||
|
|
@ -711,14 +710,28 @@ def _route_plan(
|
|||
drop_params: bool,
|
||||
) -> _Plan:
|
||||
"""Converse leaves out a ``tool_config`` cachePoint with a ttl that would join the ``tail`` of its tool config when
|
||||
a cachePoint there sets a ttl too. That request is planned again with the tail's ttl kept in its place."""
|
||||
a cachePoint there sets a ttl too. That request is planned again with the tail's ttl kept in its place. A passed
|
||||
through cachePoint with a ttl takes the place of the translated mark on the tool just before it, which is not
|
||||
sent, and of the ``tool_config`` cachePoint, which Bedrock joins to it, when that mark keeps its ttl or sets none,
|
||||
or under ``drop_params``."""
|
||||
passed_through: Final = next(filter(_sets_a_ttl, tail.cache_points), None)
|
||||
covered: Final = next((mark for mark in marks if mark.path == tail.tool and mark.kind != "written"), None)
|
||||
if (
|
||||
passed_through is not None
|
||||
and covered is not None
|
||||
and (drop_params or covered.kind == "free" or covered.ttl == _ttl(passed_through))
|
||||
):
|
||||
stand_in: Final = replace(covered, ttl=_ttl(passed_through), kind="written")
|
||||
covered_plan: Final = _planned(
|
||||
tuple(stand_in if mark is covered else mark for mark in marks if mark.path != _TOOL_CONFIG), drop_params
|
||||
)
|
||||
return replace(covered_plan, dropped=covered_plan.dropped | {covered.path})
|
||||
plan: Final = _planned(marks, drop_params)
|
||||
tool_mark: Final = (
|
||||
None
|
||||
if tail.tool is None
|
||||
else _planned_cache_control(plan, tail.tool, (_fields(tools[int(tail.tool[1])]) or {}).get("cache_control"))
|
||||
)
|
||||
passed_through: Final = next(filter(_sets_a_ttl, tail.cache_points), None)
|
||||
tail_cache_control: Final = tool_mark if passed_through is None else passed_through
|
||||
if not (
|
||||
_sets_a_ttl(tail_cache_control)
|
||||
|
|
@ -759,13 +772,15 @@ def claude_prompt_cache(
|
|||
tool_history_as_text: bool,
|
||||
tool_config_cache_control: ChatCompletionCachedContent | None,
|
||||
route: ClaudeRoute,
|
||||
adds_a_tool: bool,
|
||||
pairs_unanswered_tool_calls: bool,
|
||||
) -> ClaudePromptCache | UnsupportedPromptCacheParam:
|
||||
"""``prompt_cache_breakpoint`` marks renamed to ``cache_control``. A mark Claude cannot honor is reported unless
|
||||
``drop_params``, which keeps what Claude can use and, when the request ends up with more than Claude's four
|
||||
breakpoints, drops the earliest translated ones. A request-level ``cache_control`` in ``params`` and the
|
||||
``tool_config_cache_control`` cachePoint a route appends after the tools each take one of the four, unless Converse
|
||||
leaves that cachePoint out, and marks on tool history a ``tool_history_as_text`` route flattens, marks on a tool the
|
||||
leaves that cachePoint out or sends no tool in its tool config, counting the tool it adds itself when it
|
||||
``adds_a_tool``, and marks on tool history a ``tool_history_as_text`` route flattens, marks on a tool the
|
||||
``route`` sends without one or under a tool's ``function`` on Converse, and marks on a tool_search call nobody
|
||||
answered, unless the route ``pairs_unanswered_tool_calls`` with a placeholder result, take none. A ``cache_control``
|
||||
on a ``namespace`` tool's member takes one off Converse and is never changed. A request carrying no breakpoint is
|
||||
|
|
@ -803,7 +818,7 @@ def claude_prompt_cache(
|
|||
unsent_call_ids: Final = frozenset() if pairs_unanswered_tool_calls else unanswered_tool_search_call_ids(messages)
|
||||
request_mark: Final = params.get("cache_control")
|
||||
tool_config_order: Final = _converse_tool_config_order(renamed_tools, late_tools)
|
||||
sent_tool_config_cache_control: Final = tool_config_cache_control if tool_config_order or not listed_tools else None
|
||||
sent_tool_config_cache_control: Final = tool_config_cache_control if tool_config_order or adds_a_tool else None
|
||||
placements: Final[tuple[tuple[_Path, object, _Kind], ...]] = (
|
||||
*(
|
||||
(path, block.get("cache_control"), _kind(path, block.get("cache_control"), translated))
|
||||
|
|
|
|||
|
|
@ -264,6 +264,13 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"tools" not in optional_params and AmazonConverseConfig._adds_a_dummy_tool(messages)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def adds_a_tool(optional_params: Mapping[str, object], messages: Sequence[object]) -> bool:
|
||||
return any(
|
||||
message_field(message_field(tool, "function"), "name") == RESPONSE_FORMAT_TOOL_NAME
|
||||
for tool in parts_of(optional_params.get("tools"))
|
||||
) or ("tools" not in optional_params and AmazonConverseConfig._adds_a_dummy_tool(messages))
|
||||
|
||||
@staticmethod
|
||||
def _adds_a_dummy_tool(messages: Sequence[object]) -> bool:
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -1136,6 +1136,16 @@ def _claude_route(provider_config: BaseConfig | None, reaches_converse: bool) ->
|
|||
return "anthropic"
|
||||
|
||||
|
||||
def _converse_tools(converse_params: Mapping[str, object] | None, messages: Sequence[object]) -> tuple[bool, bool]:
|
||||
"""Whether Converse sends a tool in its tool config, and whether it adds one of its own there."""
|
||||
if converse_params is None:
|
||||
return False, False
|
||||
return (
|
||||
AmazonConverseConfig.sends_tools(converse_params, messages),
|
||||
AmazonConverseConfig.adds_a_tool(converse_params, messages),
|
||||
)
|
||||
|
||||
|
||||
def _listed_drop_params(additional_drop_params: object) -> tuple[str, ...]:
|
||||
if not isinstance(additional_drop_params, (list, tuple)):
|
||||
return ()
|
||||
|
|
@ -5763,18 +5773,16 @@ def completion(
|
|||
accepts_summary=provider_config is not None
|
||||
and provider_config.accepts_reasoning_summary_in_reasoning_effort,
|
||||
)
|
||||
converse_sends_tools: Final = (
|
||||
reaches_converse
|
||||
and carries_prompt_cache_breakpoint(routed_messages, request_tools)
|
||||
and AmazonConverseConfig.sends_tools(
|
||||
_map_request_params(
|
||||
**{**request_param_args, "reasoning_effort": provider_reasoning_effort},
|
||||
model_info=validated_model_info,
|
||||
**non_default_params,
|
||||
),
|
||||
routed_messages,
|
||||
converse_params: Final = (
|
||||
_map_request_params(
|
||||
**{**request_param_args, "reasoning_effort": provider_reasoning_effort},
|
||||
model_info=validated_model_info,
|
||||
**non_default_params,
|
||||
)
|
||||
if reaches_converse and carries_prompt_cache_breakpoint(routed_messages, request_tools)
|
||||
else None
|
||||
)
|
||||
converse_sends_tools, converse_adds_a_tool = _converse_tools(converse_params, routed_messages)
|
||||
claude_cache: Final = (
|
||||
resolved_claude_prompt_cache(
|
||||
claude_prompt_cache(
|
||||
|
|
@ -5792,6 +5800,7 @@ def completion(
|
|||
else None
|
||||
),
|
||||
route=_claude_route(provider_config, reaches_converse),
|
||||
adds_a_tool=converse_adds_a_tool,
|
||||
pairs_unanswered_tool_calls=litellm.modify_params is True,
|
||||
),
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from typing import Final
|
|||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.anthropic.chat.prompt_cache_breakpoints import ClaudePromptCache, claude_prompt_cache
|
||||
from litellm.llms.anthropic.chat.prompt_cache_breakpoints import ClaudePromptCache, ClaudeRoute, claude_prompt_cache
|
||||
|
||||
_EXPLICIT: Final = {"mode": "explicit"}
|
||||
|
||||
|
|
@ -25,16 +25,24 @@ _ONE_HOUR_PARTS_BESIDE_A_BREAKPOINT: Final = (
|
|||
},
|
||||
)
|
||||
|
||||
_SYSTEM_TOOLS_BESIDE_A_MARKED_TOOL: Final = (
|
||||
*({"systemTool": {"name": f"s{index}"}} for index in range(20000)),
|
||||
{"type": "function", "function": {"name": "t"}, "prompt_cache_breakpoint": _EXPLICIT},
|
||||
)
|
||||
|
||||
def _settled_under_drop_params(messages: Sequence[object]) -> None:
|
||||
|
||||
def _settled_under_drop_params(
|
||||
messages: Sequence[object], tools: Sequence[object] | None = None, route: ClaudeRoute = "anthropic"
|
||||
) -> None:
|
||||
outcome: Final = claude_prompt_cache(
|
||||
messages,
|
||||
None,
|
||||
tools,
|
||||
{},
|
||||
drop_params=True,
|
||||
tool_history_as_text=False,
|
||||
tool_config_cache_control=None,
|
||||
route="anthropic",
|
||||
route=route,
|
||||
adds_a_tool=False,
|
||||
pairs_unanswered_tool_calls=False,
|
||||
)
|
||||
assert isinstance(outcome, ClaudePromptCache)
|
||||
|
|
@ -48,3 +56,8 @@ def test_settling_thousands_of_marked_system_turns_under_drop_params():
|
|||
@pytest.mark.benchmark
|
||||
def test_settling_a_breakpoint_beside_thousands_of_one_hour_parts_under_drop_params():
|
||||
_settled_under_drop_params(_ONE_HOUR_PARTS_BESIDE_A_BREAKPOINT)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_settling_a_tool_breakpoint_beside_thousands_of_bedrock_system_tools_under_drop_params():
|
||||
_settled_under_drop_params(({"role": "user", "content": "hi"},), _SYSTEM_TOOLS_BESIDE_A_MARKED_TOOL, "converse")
|
||||
|
|
|
|||
|
|
@ -867,6 +867,20 @@ _MARKED_TOOL: Final[dict[str, JsonValue]] = {**TOOL, "prompt_cache_breakpoint":
|
|||
None,
|
||||
id="cache-point-with-five-minutes-listed-last",
|
||||
),
|
||||
pytest.param(
|
||||
[_MARKED_TOOL, {"cachePoint": {"type": "default", "ttl": "1h"}}],
|
||||
True,
|
||||
(Mark(_TOOL_LABEL, "1h"),),
|
||||
"1h",
|
||||
id="cache-point-with-an-hour-right-after-the-marked-tool",
|
||||
),
|
||||
pytest.param(
|
||||
[_MARKED_TOOL, {"cachePoint": {"type": "default", "ttl": "5m"}}],
|
||||
True,
|
||||
(Mark(_TOOL_LABEL, "5m"),),
|
||||
None,
|
||||
id="cache-point-with-five-minutes-right-after-the-marked-tool",
|
||||
),
|
||||
pytest.param(
|
||||
[_MARKED_TOOL, {"cachePoint": {"type": "default"}}],
|
||||
True,
|
||||
|
|
@ -901,6 +915,60 @@ def test_a_five_minute_tool_config_cache_point_shortens_later_breakpoints_only_w
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_breakpoint", "cache_point_ttl"),
|
||||
[
|
||||
pytest.param(_ONE_HOUR_BREAKPOINT, "5m", id="hour-breakpoint-before-a-five-minute-cache-point"),
|
||||
pytest.param(_FIVE_MINUTE_BREAKPOINT, "1h", id="five-minute-breakpoint-before-an-hour-cache-point"),
|
||||
],
|
||||
)
|
||||
def test_a_tool_breakpoint_before_a_cache_point_with_another_ttl_gets_bedrocks_400_without_drop_params(
|
||||
gateway: Gateway, tool_breakpoint: dict[str, JsonValue], cache_point_ttl: str
|
||||
) -> None:
|
||||
marker: Final = new_marker()
|
||||
tools: Final[list[JsonValue]] = [
|
||||
{**TOOL, "prompt_cache_breakpoint": tool_breakpoint},
|
||||
{"cachePoint": {"type": "default", "ttl": cache_point_ttl}},
|
||||
]
|
||||
with wire_server(lambda request: respond("bedrock_converse", request)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = deployment(gateway, scenario, wire, "bedrock_converse")
|
||||
status, _, text = post_chat(gateway, chat_body(model, [_final(marker)], tools=tools))
|
||||
request: Final = only_request(wire)
|
||||
assert status == 400, text
|
||||
assert "conflicts with the existing field ttl" in text, text
|
||||
assert sent_marks("bedrock_converse", request) == (
|
||||
Mark(_TOOL_LABEL, tool_breakpoint["ttl"]),
|
||||
Mark("other", cache_point_ttl),
|
||||
)
|
||||
|
||||
|
||||
def test_a_five_minute_tool_config_cache_point_after_the_response_format_tool_shortens_later_breakpoints(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
marker: Final = new_marker()
|
||||
turns: Final[list[JsonValue]] = [
|
||||
{"role": "system", "content": [_text(SYSTEM, prompt_cache_breakpoint=_EXPLICIT)]},
|
||||
_final(marker, prompt_cache_breakpoint=_ONE_HOUR_BREAKPOINT),
|
||||
]
|
||||
marks: Final = _marks_sent(
|
||||
gateway,
|
||||
"bedrock_converse",
|
||||
lambda model: chat_body(
|
||||
model,
|
||||
turns,
|
||||
tools=[{"type": "web_fetch_20250910", "name": "web_fetch", "cache_control": _ONE_HOUR}],
|
||||
drop_params=True,
|
||||
response_format=_JSON_SCHEMA,
|
||||
),
|
||||
cache_control_injection_points=[{"location": "tool_config", "control": {"type": "ephemeral", "ttl": "5m"}}],
|
||||
)
|
||||
assert marks == (
|
||||
Mark("tool:json_tool_call", "5m"),
|
||||
Mark(_label(SYSTEM), None),
|
||||
Mark(_label(final_text(marker)), None),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_drops", "expected"),
|
||||
(
|
||||
|
|
|
|||
|
|
@ -1286,6 +1286,7 @@ def _translated_for_claude(messages: list[AllMessageValues], modify_params: bool
|
|||
tool_history_as_text=False,
|
||||
tool_config_cache_control=None,
|
||||
route="anthropic",
|
||||
adds_a_tool=False,
|
||||
pairs_unanswered_tool_calls=modify_params,
|
||||
)
|
||||
assert isinstance(outcome, ClaudePromptCache), outcome
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ def _cache(
|
|||
drop_params: bool = False,
|
||||
tool_config_cache_control: ChatCompletionCachedContent | None = None,
|
||||
route: ClaudeRoute = "anthropic",
|
||||
adds_a_tool: bool = False,
|
||||
pairs_unanswered_tool_calls: bool = False,
|
||||
) -> ClaudePromptCache | UnsupportedPromptCacheParam:
|
||||
return claude_prompt_cache(
|
||||
|
|
@ -35,6 +36,7 @@ def _cache(
|
|||
tool_history_as_text=False,
|
||||
tool_config_cache_control=tool_config_cache_control,
|
||||
route=route,
|
||||
adds_a_tool=adds_a_tool,
|
||||
pairs_unanswered_tool_calls=pairs_unanswered_tool_calls,
|
||||
)
|
||||
|
||||
|
|
@ -47,6 +49,7 @@ def _resolved(
|
|||
drop_params: bool = False,
|
||||
tool_config_cache_control: ChatCompletionCachedContent | None = None,
|
||||
route: ClaudeRoute = "anthropic",
|
||||
adds_a_tool: bool = False,
|
||||
pairs_unanswered_tool_calls: bool = False,
|
||||
) -> ClaudePromptCache:
|
||||
outcome: Final = _cache(
|
||||
|
|
@ -56,6 +59,7 @@ def _resolved(
|
|||
drop_params=drop_params,
|
||||
tool_config_cache_control=tool_config_cache_control,
|
||||
route=route,
|
||||
adds_a_tool=adds_a_tool,
|
||||
pairs_unanswered_tool_calls=pairs_unanswered_tool_calls,
|
||||
)
|
||||
assert isinstance(outcome, ClaudePromptCache), outcome
|
||||
|
|
@ -1308,6 +1312,152 @@ def test_later_one_hour_breakpoints_are_shortened_only_when_converses_tool_confi
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tools", "sent_tools"),
|
||||
[
|
||||
pytest.param(None, None, id="no-listed-tool"),
|
||||
pytest.param(
|
||||
[{"type": "web_fetch_20250910", "name": "web_fetch", "cache_control": _ONE_HOUR}],
|
||||
[{"type": "web_fetch_20250910", "name": "web_fetch", "cache_control": _ONE_HOUR}],
|
||||
id="a-listed-tool-converse-drops",
|
||||
),
|
||||
pytest.param(
|
||||
[{"type": "bash_20250124", "name": "bash", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"}}],
|
||||
[{"type": "bash_20250124", "name": "bash", "cache_control": _EPHEMERAL}],
|
||||
id="a-computer-use-tool-converse-sends-outside-the-tool-config",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_later_one_hour_breakpoints_are_shortened_after_the_five_minute_cache_point_beside_the_tool_converse_adds(
|
||||
tools: list[object] | None, sent_tools: list[object] | None
|
||||
):
|
||||
result: Final = _resolved(
|
||||
[
|
||||
{"role": "system", "content": [_part("s", prompt_cache_breakpoint=_EXPLICIT)]},
|
||||
{"role": "user", "content": [_part("ask", prompt_cache_breakpoint={"mode": "explicit", "ttl": "1h"})]},
|
||||
],
|
||||
tools,
|
||||
drop_params=True,
|
||||
tool_config_cache_control=ChatCompletionCachedContent(type="ephemeral", ttl="5m"),
|
||||
route="converse",
|
||||
adds_a_tool=True,
|
||||
)
|
||||
|
||||
assert result.tools == (None if sent_tools is None else tuple(sent_tools))
|
||||
assert list(result.messages) == [
|
||||
{"role": "system", "content": [_part("s", cache_control=_EPHEMERAL)]},
|
||||
{"role": "user", "content": [_part("ask", cache_control=_EPHEMERAL)]},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool_config",
|
||||
[None, ChatCompletionCachedContent(type="ephemeral", ttl="5m")],
|
||||
ids=["no-tool-config-cache-point", "five-minute-tool-config-cache-point"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("tool_breakpoint", "cache_point_ttl", "drop_params"),
|
||||
[
|
||||
pytest.param({"mode": "explicit", "ttl": "1h"}, "1h", False, id="same-ttl"),
|
||||
pytest.param({"mode": "explicit", "ttl": "1h"}, "1h", True, id="same-ttl-with-drop-params"),
|
||||
pytest.param(_EXPLICIT, "1h", False, id="breakpoint-without-a-ttl-before-an-hour"),
|
||||
pytest.param(_EXPLICIT, "5m", False, id="breakpoint-without-a-ttl-before-five-minutes"),
|
||||
pytest.param(_EXPLICIT, "5m", True, id="breakpoint-without-a-ttl-with-drop-params"),
|
||||
pytest.param({"mode": "explicit", "ttl": "1h"}, "5m", True, id="another-ttl-with-drop-params"),
|
||||
],
|
||||
)
|
||||
def test_a_bedrock_cache_point_with_a_ttl_right_after_a_tool_breakpoint_is_the_only_one_sent_there(
|
||||
tool_breakpoint: dict[str, object],
|
||||
cache_point_ttl: str,
|
||||
tool_config: ChatCompletionCachedContent | None,
|
||||
drop_params: bool,
|
||||
):
|
||||
cache_point: Final = {"cachePoint": {"type": "default", "ttl": cache_point_ttl}}
|
||||
|
||||
result: Final = _resolved(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
[{"type": "function", "function": {"name": "t"}, "prompt_cache_breakpoint": tool_breakpoint}, cache_point],
|
||||
drop_params=drop_params,
|
||||
tool_config_cache_control=tool_config,
|
||||
route="converse",
|
||||
)
|
||||
|
||||
assert list(result.tools or ()) == [{"type": "function", "function": {"name": "t"}}, cache_point]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool_config",
|
||||
[None, ChatCompletionCachedContent(type="ephemeral", ttl="5m")],
|
||||
ids=["no-tool-config-cache-point", "five-minute-tool-config-cache-point"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("tool_breakpoint", "sent_tool_mark", "cache_point_ttl"),
|
||||
[
|
||||
pytest.param({"mode": "explicit", "ttl": "1h"}, _ONE_HOUR, "5m", id="hour-before-five-minutes"),
|
||||
pytest.param({"mode": "explicit", "ttl": "5m"}, _FIVE_MINUTES, "1h", id="five-minutes-before-an-hour"),
|
||||
],
|
||||
)
|
||||
def test_a_tool_breakpoint_and_a_bedrock_cache_point_right_after_it_with_another_ttl_are_both_sent_without_drop_params(
|
||||
tool_breakpoint: dict[str, object],
|
||||
sent_tool_mark: dict[str, object],
|
||||
cache_point_ttl: str,
|
||||
tool_config: ChatCompletionCachedContent | None,
|
||||
):
|
||||
cache_point: Final = {"cachePoint": {"type": "default", "ttl": cache_point_ttl}}
|
||||
|
||||
result: Final = _resolved(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
[{"type": "function", "function": {"name": "t"}, "prompt_cache_breakpoint": tool_breakpoint}, cache_point],
|
||||
drop_params=False,
|
||||
tool_config_cache_control=tool_config,
|
||||
route="converse",
|
||||
)
|
||||
|
||||
assert list(result.tools or ()) == [
|
||||
{"type": "function", "function": {"name": "t"}, "cache_control": sent_tool_mark},
|
||||
cache_point,
|
||||
]
|
||||
|
||||
|
||||
def test_a_bedrock_cache_point_with_a_ttl_after_a_tool_breakpoint_keeps_that_breakpoints_slot_of_the_four():
|
||||
result: Final = _resolved(
|
||||
[{"role": "user", "content": [_part(str(index), prompt_cache_breakpoint=_EXPLICIT) for index in range(4)]}],
|
||||
[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "t"},
|
||||
"prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"},
|
||||
},
|
||||
{"cachePoint": {"type": "default", "ttl": "1h"}},
|
||||
],
|
||||
drop_params=True,
|
||||
route="converse",
|
||||
)
|
||||
|
||||
assert list(result.messages) == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [_part("0"), *(_part(str(index), cache_control=_EPHEMERAL) for index in range(1, 4))],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_a_client_cache_control_on_a_tool_before_a_bedrock_cache_point_with_a_ttl_is_sent_as_written():
|
||||
tools: Final = [
|
||||
{"type": "function", "function": {"name": "t"}, "cache_control": _ONE_HOUR},
|
||||
{"cachePoint": {"type": "default", "ttl": "1h"}},
|
||||
]
|
||||
|
||||
result: Final = _resolved(
|
||||
[{"role": "user", "content": [_part("ask", prompt_cache_breakpoint=_EXPLICIT)]}],
|
||||
tools,
|
||||
drop_params=True,
|
||||
route="converse",
|
||||
)
|
||||
|
||||
assert list(result.tools or ()) == tools
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("route", "sent_bash_mark"),
|
||||
[("anthropic", _ONE_HOUR), ("converse", _EPHEMERAL)],
|
||||
|
|
|
|||
|
|
@ -1302,6 +1302,39 @@ def test_a_five_minute_tool_config_cache_point_does_not_shorten_later_breakpoint
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response_format_args", _RESPONSE_FORMATS_CONVERSE_SENDS_AS_A_TOOL)
|
||||
def test_a_five_minute_tool_config_cache_point_after_the_response_format_tool_shortens_breakpoints_when_converse_drops_every_listed_tool(
|
||||
respx_mock: respx.MockRouter, response_format_args: Mapping[str, object]
|
||||
):
|
||||
route: Final = respx_mock.post(url__regex=_CONVERSE_ROUTE).respond(json=_CONVERSE_REPLY)
|
||||
|
||||
litellm.completion(
|
||||
**{
|
||||
"model": "bedrock/converse/us.anthropic.claude-opus-5-5",
|
||||
"drop_params": True,
|
||||
**_FAKE_AWS,
|
||||
**response_format_args,
|
||||
},
|
||||
messages=[
|
||||
{"role": "system", "content": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": _BREAKPOINT}]},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": _ONE_HOUR_BREAKPOINT}],
|
||||
},
|
||||
],
|
||||
tools=[
|
||||
{"type": "web_fetch_20250910", "name": "web_fetch", "cache_control": {"type": "ephemeral", "ttl": "1h"}}
|
||||
],
|
||||
cache_control_injection_points=_FIVE_MINUTE_TOOL_CONFIG,
|
||||
)
|
||||
|
||||
assert _converse_cache_point_places(json.loads(route.calls.last.request.content)) == (
|
||||
("tool", "5m"),
|
||||
("system", None),
|
||||
("message", None),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"system_tool_first", [True, False], ids=["system-tool-listed-first", "system-tool-listed-last"]
|
||||
)
|
||||
|
|
@ -1363,6 +1396,39 @@ def test_a_five_minute_tool_config_cache_point_is_not_sent_after_a_passed_throug
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cache_point_ttl", "sent_later_ttl"), [("1h", "1h"), ("5m", None)], ids=["one-hour", "five-minutes"]
|
||||
)
|
||||
def test_a_passed_through_cache_point_with_a_ttl_behind_a_one_hour_tool_breakpoint_is_the_only_tool_cache_point_sent(
|
||||
respx_mock: respx.MockRouter, cache_point_ttl: str, sent_later_ttl: str | None
|
||||
):
|
||||
route: Final = respx_mock.post(url__regex=_CONVERSE_ROUTE).respond(json=_CONVERSE_REPLY)
|
||||
|
||||
litellm.completion(
|
||||
model="bedrock/converse/us.anthropic.claude-opus-5-5",
|
||||
messages=[
|
||||
{"role": "system", "content": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": _BREAKPOINT}]},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": _ONE_HOUR_BREAKPOINT}],
|
||||
},
|
||||
],
|
||||
tools=[
|
||||
{**_WEATHER_TOOL, "prompt_cache_breakpoint": _ONE_HOUR_BREAKPOINT},
|
||||
{"cachePoint": {"type": "default", "ttl": cache_point_ttl}},
|
||||
],
|
||||
cache_control_injection_points=_FIVE_MINUTE_TOOL_CONFIG,
|
||||
drop_params=True,
|
||||
**_FAKE_AWS,
|
||||
)
|
||||
|
||||
assert _converse_cache_point_places(json.loads(route.calls.last.request.content)) == (
|
||||
("tool", cache_point_ttl),
|
||||
("system", sent_later_ttl),
|
||||
("message", sent_later_ttl),
|
||||
)
|
||||
|
||||
|
||||
def test_a_tool_config_cache_point_after_a_tool_mark_without_a_ttl_still_shortens_a_later_one_hour_breakpoint(
|
||||
respx_mock: respx.MockRouter,
|
||||
):
|
||||
|
|
@ -1416,6 +1482,37 @@ def _sent_cache_ttls(body: Mapping[str, object]) -> tuple[str | None, ...]:
|
|||
return tuple(mark.get("ttl") for mark in marks if mark is not None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "url", "credentials"),
|
||||
[
|
||||
pytest.param("anthropic/claude-opus-5-5", r"https://api\.anthropic\.com/v1/messages", {"api_key": "fake"}),
|
||||
pytest.param("bedrock/invoke/us.anthropic.claude-opus-5-5", _INVOKE_ROUTE, _FAKE_AWS),
|
||||
],
|
||||
ids=["anthropic", "bedrock-invoke"],
|
||||
)
|
||||
def test_a_five_minute_tool_config_cache_point_does_not_shorten_a_one_hour_breakpoint_on_a_route_that_never_sends_it(
|
||||
respx_mock: respx.MockRouter, model: str, url: str, credentials: Mapping[str, str]
|
||||
):
|
||||
route: Final = respx_mock.post(url__regex=url).respond(json=_CLAUDE_REPLY)
|
||||
|
||||
litellm.completion(
|
||||
model=model,
|
||||
messages=[
|
||||
{"role": "system", "content": [{"type": "text", "text": "sys"}]},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": _ONE_HOUR_BREAKPOINT}],
|
||||
},
|
||||
],
|
||||
tools=[_WEATHER_TOOL],
|
||||
cache_control_injection_points=_FIVE_MINUTE_TOOL_CONFIG,
|
||||
drop_params=True,
|
||||
**credentials,
|
||||
)
|
||||
|
||||
assert _sent_cache_ttls(json.loads(route.calls.last.request.content)) == ("1h",)
|
||||
|
||||
|
||||
def test_a_string_additional_drop_params_on_bedrock_invoke_still_answers_unsupported_params_error(
|
||||
respx_mock: respx.MockRouter,
|
||||
):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue