mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-30 06:43:27 +00:00
fix(anthropic): keep breakpoint requests valid for Claude when transforms move or add marks
One mark per tool result on every Claude route, a mid-conversation system message settles its ttl with the turns it can move past, Converse counts tool history marks when response_format, functions or web_search_options add tools, and a Converse tool_config cachePoint takes 1h after a 1h mark and counts as one of the four
This commit is contained in:
mateo-berri
parent
09fc34b91d
commit
b97387b50b
6 changed files with 483 additions and 84 deletions
|
|
@ -1,8 +1,8 @@
|
|||
"""OpenAI's ``prompt_cache_breakpoint`` on a chat request bound for Claude, rewritten as Claude's ``cache_control``."""
|
||||
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from dataclasses import dataclass, replace
|
||||
from itertools import accumulate, chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, NoReturn, TypeAlias, cast # noqa: TID251 # isinstance-checked hot-path JSON
|
||||
|
||||
|
|
@ -18,7 +18,10 @@ _Mark: TypeAlias = Callable[[_Path, _Fields], _Fields]
|
|||
_Kind: TypeAlias = Literal["written", "explicit", "free"]
|
||||
_MESSAGE_BLOCK_KEYS: Final = ("content", "tool_calls")
|
||||
_TOOL_RESULT_ROLES: Final = ("tool", "function")
|
||||
_SYSTEM_ROLES: Final = ("system", "developer")
|
||||
_SYSTEM_MOVES_PAST_ROLES: Final = ("user", *_TOOL_RESULT_ROLES)
|
||||
_REQUEST: Final[_Path] = ("cache_control",)
|
||||
_TOOL_CONFIG: Final[_Path] = ("tool_config",)
|
||||
_CLAUDE_TTL_MARKS: Final = MappingProxyType(
|
||||
{
|
||||
"5m": ChatCompletionCachedContent(type="ephemeral", ttl="5m"),
|
||||
|
|
@ -46,6 +49,7 @@ class _Placed:
|
|||
path: _Path
|
||||
ttl: Literal["5m", "1h"]
|
||||
kind: _Kind
|
||||
rank: tuple[int, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -171,27 +175,23 @@ def _joined_breakpoint(kept: object, replaced: Sequence[object]) -> object:
|
|||
return {**fields, "ttl": "1h"} if any(_block_mark(marker, "ttl") == "1h" for marker in replaced) else kept
|
||||
|
||||
|
||||
def _tool_result_hoisted(message: _Fields, one_mark_per_tool_result: bool) -> _Hoisted:
|
||||
"""Claude rejects a tool result marked both on the result and inside its content, so a ``cache_control`` the
|
||||
client wrote at either level wins over every breakpoint at the other, and a breakpoint on the message replaces the
|
||||
ones on its parts. A route that sends one mark per tool result gets its part breakpoints as one on the message,
|
||||
or as none when the client marked a part."""
|
||||
def _tool_result_hoisted(message: _Fields) -> _Hoisted:
|
||||
"""Claude rejects a tool result marked both on the result and inside its content, or marked twice inside it, so
|
||||
a ``cache_control`` the client wrote at either level wins over every breakpoint, and the result's breakpoints
|
||||
become one on the message."""
|
||||
parts: Final = _blocks(message.get("content"))
|
||||
if parts is None:
|
||||
return _Hoisted(message, ())
|
||||
own: Final = message.get("prompt_cache_breakpoint")
|
||||
client_marked_part: Final = any(_block_mark(part, "cache_control") is not None for part in parts)
|
||||
if own is not None and client_marked_part:
|
||||
return _Hoisted(_without_message_breakpoint(message), ())
|
||||
unmarked_parts: Final = [_without_breakpoint(part) for part in parts]
|
||||
if message.get("cache_control") is not None:
|
||||
return _Hoisted({**message, "content": unmarked_parts}, ())
|
||||
part_breakpoints: Final = tuple(
|
||||
marker for marker in (_block_mark(part, "prompt_cache_breakpoint") for part in parts) if marker is not None
|
||||
)
|
||||
if own is None and client_marked_part and one_mark_per_tool_result:
|
||||
return _Hoisted({**message, "content": unmarked_parts}, part_breakpoints)
|
||||
if own is None and (client_marked_part or not one_mark_per_tool_result or not part_breakpoints):
|
||||
if any(_block_mark(part, "cache_control") is not None for part in parts):
|
||||
return _Hoisted({**_without_message_breakpoint(message), "content": unmarked_parts}, part_breakpoints)
|
||||
if own is None and not part_breakpoints:
|
||||
return _Hoisted(message, ())
|
||||
kept: Final = part_breakpoints[0] if own is None else own
|
||||
replaced: Final = part_breakpoints[1:] if own is None else part_breakpoints
|
||||
|
|
@ -250,12 +250,12 @@ def _message_breakpoint_hoisted(message: _Fields) -> _Hoisted:
|
|||
)
|
||||
|
||||
|
||||
def _hoisted_message(message: object, one_mark_per_tool_result: bool) -> _Hoisted:
|
||||
def _hoisted_message(message: object) -> _Hoisted:
|
||||
fields: Final = _fields(message)
|
||||
if fields is None:
|
||||
return _Hoisted(message, ())
|
||||
return (
|
||||
_tool_result_hoisted(fields, one_mark_per_tool_result)
|
||||
_tool_result_hoisted(fields)
|
||||
if fields.get("role") in _TOOL_RESULT_ROLES
|
||||
else _message_breakpoint_hoisted(fields)
|
||||
)
|
||||
|
|
@ -362,29 +362,58 @@ def _translated(messages: Sequence[object], tools: Sequence[object]) -> frozense
|
|||
)
|
||||
|
||||
|
||||
def _role(message: object) -> object:
|
||||
return (_fields(message) or {}).get("role")
|
||||
|
||||
|
||||
def _leading_system_count(messages: Sequence[object]) -> int:
|
||||
return next(
|
||||
(
|
||||
index
|
||||
for index, message in enumerate(messages)
|
||||
if (_fields(message) or {}).get("role") not in ("system", "developer")
|
||||
),
|
||||
len(messages),
|
||||
)
|
||||
return next((index for index, message in enumerate(messages) if _role(message) not in _SYSTEM_ROLES), len(messages))
|
||||
|
||||
|
||||
def _wire_position(path: _Path, leading_system_count: int) -> tuple[int, ...]:
|
||||
"""Where Claude reads a mark: tool definitions, then the leading system messages it hoists, then every other
|
||||
message in place, with a message-level mark on the message's text, after its content parts and before its tool
|
||||
calls."""
|
||||
"""Where Claude reads a mark: tool definitions, then the cachePoint a route appends after them, then the leading
|
||||
system messages it hoists, then every other message in place, with a message-level mark on the message's text,
|
||||
after its content parts and before its tool calls, and last the request-level mark."""
|
||||
if path == _TOOL_CONFIG:
|
||||
return (1,)
|
||||
if path == _REQUEST:
|
||||
return (4,)
|
||||
if path[0] == "tools":
|
||||
return (0, *(index for index in path if isinstance(index, int)))
|
||||
index: Final = int(path[1])
|
||||
group: Final = 1 if index < leading_system_count else 2
|
||||
group: Final = 2 if index < leading_system_count else 3
|
||||
block: Final = (1,) if len(path) == 2 else (0 if path[2] == "content" else 2, int(path[3]))
|
||||
return (group, index, *block)
|
||||
|
||||
|
||||
def _open_system_group(opened_at: tuple[int, ...], indexed: tuple[int, object]) -> tuple[int, ...]:
|
||||
index, message = indexed
|
||||
role: Final = _role(message)
|
||||
if role in _SYSTEM_ROLES:
|
||||
return opened_at or (index,)
|
||||
return opened_at if role in _SYSTEM_MOVES_PAST_ROLES else ()
|
||||
|
||||
|
||||
def _system_groups(messages: Sequence[object], leading_system_count: int) -> Mapping[int, int]:
|
||||
"""The transforms move a system message after the conversation start behind the user, tool and function turns
|
||||
that follow it, never past the next assistant turn, so each such message and those turns map to the message
|
||||
that opens their group."""
|
||||
opened: Final = tuple(
|
||||
accumulate(
|
||||
enumerate(messages[leading_system_count:], start=leading_system_count), _open_system_group, initial=()
|
||||
)
|
||||
)[1:]
|
||||
return MappingProxyType(
|
||||
{index: opened_at[0] for index, opened_at in enumerate(opened, start=leading_system_count) if opened_at}
|
||||
)
|
||||
|
||||
|
||||
def _rank(path: _Path, leading_system_count: int, system_groups: Mapping[int, int]) -> tuple[int, ...]:
|
||||
"""Marks in one system group share a rank, since Claude may read them in any order."""
|
||||
opened_at: Final = system_groups.get(int(path[1])) if path[0] == "messages" else None
|
||||
return _wire_position(path, leading_system_count) if opened_at is None else (3, opened_at)
|
||||
|
||||
|
||||
def _tool_call_id(message: _Fields, index: int) -> object:
|
||||
return (_fields((_blocks(message.get("tool_calls")) or ())[index]) or {}).get("id")
|
||||
|
||||
|
|
@ -430,19 +459,21 @@ def _chosen(mark: _Placed, ttl: Literal["5m", "1h"]) -> bool:
|
|||
return mark.kind != "free" and mark.ttl == ttl
|
||||
|
||||
|
||||
def _last_chosen_hour(marks: Sequence[_Placed]) -> int:
|
||||
return max((index for index, mark in enumerate(marks) if _chosen(mark, "1h")), default=-1)
|
||||
def _last_chosen_hour(marks: Sequence[_Placed]) -> tuple[int, ...]:
|
||||
return max((mark.rank for mark in marks if _chosen(mark, "1h")), default=(-1,))
|
||||
|
||||
|
||||
def _loosened(marks: Sequence[_Placed], drop_params: bool) -> tuple[_Placed, ...]:
|
||||
"""Claude rejects a 1h mark read after a 5m one."""
|
||||
first_five: Final = next((index for index, mark in enumerate(marks) if _chosen(mark, "5m")), len(marks))
|
||||
first_five: Final = min((mark.rank for mark in marks if _chosen(mark, "5m")), default=(5,))
|
||||
last_hour: Final = _last_chosen_hour(marks)
|
||||
return tuple(
|
||||
_Placed(mark.path, "5m", "free")
|
||||
if drop_params and mark.kind == "explicit" and (index > first_five if mark.ttl == "1h" else index < last_hour)
|
||||
replace(mark, ttl="5m", kind="free")
|
||||
if drop_params
|
||||
and mark.kind == "explicit"
|
||||
and (mark.rank >= first_five if mark.ttl == "1h" else mark.rank <= last_hour)
|
||||
else mark
|
||||
for index, mark in enumerate(marks)
|
||||
for mark in marks
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -452,11 +483,28 @@ def _settled(marks: Sequence[_Placed], drop_params: bool) -> tuple[_Placed, ...]
|
|||
loosened: Final = _loosened(marks, drop_params)
|
||||
last_hour: Final = _last_chosen_hour(loosened)
|
||||
return tuple(
|
||||
_Placed(mark.path, "1h", "free") if mark.kind == "free" and index < last_hour else mark
|
||||
for index, mark in enumerate(loosened)
|
||||
replace(mark, ttl="1h") if mark.kind == "free" and mark.rank <= last_hour else mark for mark in loosened
|
||||
)
|
||||
|
||||
|
||||
def _route_marks(
|
||||
tool_config_cache_control: ChatCompletionCachedContent | None, request_mark: object
|
||||
) -> tuple[tuple[_Path, object, _Kind], ...]:
|
||||
"""The cachePoint a route appends after the tools takes 1h like a breakpoint when it sets no ttl. Neither it nor
|
||||
the request-level mark is ever dropped."""
|
||||
tool_config_kind: Final[_Kind] = (
|
||||
"free" if tool_config_cache_control is None or tool_config_cache_control.get("ttl") is None else "written"
|
||||
)
|
||||
return (
|
||||
*(() if tool_config_cache_control is None else ((_TOOL_CONFIG, tool_config_cache_control, tool_config_kind),)),
|
||||
*(() if request_mark is None else ((_REQUEST, request_mark, "written"),)),
|
||||
)
|
||||
|
||||
|
||||
def _droppable(mark: _Placed) -> bool:
|
||||
return mark.kind != "written" and mark.path != _TOOL_CONFIG
|
||||
|
||||
|
||||
def claude_prompt_cache(
|
||||
messages: Sequence[object],
|
||||
tools: Sequence[object] | None,
|
||||
|
|
@ -464,18 +512,18 @@ def claude_prompt_cache(
|
|||
*,
|
||||
drop_params: bool,
|
||||
tool_history_as_text: bool,
|
||||
one_mark_per_tool_result: bool,
|
||||
tool_config_cache_control: ChatCompletionCachedContent | None,
|
||||
) -> ClaudePromptCache | UnsupportedPromptCacheParam:
|
||||
"""``prompt_cache_breakpoint`` marks renamed to ``cache_control``. A mark Claude cannot honor is reported unless
|
||||
``drop_params``, which keeps what Claude can use and, when the request ends up with more than Claude's four
|
||||
breakpoints, drops the earliest translated ones. A request-level ``cache_control`` in ``params`` takes one of the
|
||||
four, marks on tool history a ``tool_history_as_text`` route flattens take none, and a
|
||||
``one_mark_per_tool_result`` route gets a tool result's breakpoints as one. A request carrying no breakpoint is
|
||||
returned as written."""
|
||||
breakpoints, drops the earliest translated ones. A request-level ``cache_control`` in ``params`` and the
|
||||
``tool_config_cache_control`` cachePoint a route appends after the tools each take one of the four, and marks on
|
||||
tool history a ``tool_history_as_text`` route flattens take none. A request carrying no breakpoint is returned as
|
||||
written."""
|
||||
listed_tools: Final = tools or ()
|
||||
if not any(map(_message_carries_breakpoint, messages)) and not any(map(_tool_carries_breakpoint, listed_tools)):
|
||||
return ClaudePromptCache(tuple(messages), None if tools is None else tuple(tools))
|
||||
hoists: Final = tuple(_hoisted_message(message, one_mark_per_tool_result) for message in messages)
|
||||
hoists: Final = tuple(map(_hoisted_message, messages))
|
||||
hoisted_messages: Final = tuple(hoist.message for hoist in hoists)
|
||||
blocks: Final = tuple(block for _, block in _markable(*_marked_request(hoisted_messages, listed_tools, _unchanged)))
|
||||
problem: Final = next(
|
||||
|
|
@ -495,24 +543,28 @@ def claude_prompt_cache(
|
|||
renamed_messages, renamed_tools = _marked_request(hoisted_messages, listed_tools, _renamed)
|
||||
translated: Final = _translated(hoisted_messages, listed_tools)
|
||||
leading_system_count: Final = _leading_system_count(messages)
|
||||
system_groups: Final = _system_groups(messages, leading_system_count)
|
||||
unsent_call_ids: Final = unanswered_tool_search_call_ids(messages)
|
||||
block_marks: Final = sorted(
|
||||
(
|
||||
_Placed(path, _ttl(block.get("cache_control")), _kind(path, block.get("cache_control"), translated))
|
||||
request_mark: Final = params.get("cache_control")
|
||||
placements: Final[tuple[tuple[_Path, object, _Kind], ...]] = (
|
||||
*(
|
||||
(path, block.get("cache_control"), _kind(path, block.get("cache_control"), translated))
|
||||
for path, block in _markable(renamed_messages, renamed_tools)
|
||||
if block.get("cache_control") is not None
|
||||
and _on_wire(path, renamed_messages, renamed_tools, unsent_call_ids, tool_history_as_text)
|
||||
),
|
||||
key=lambda mark: _wire_position(mark.path, leading_system_count),
|
||||
*_route_marks(tool_config_cache_control, request_mark),
|
||||
)
|
||||
request_mark: Final = params.get("cache_control")
|
||||
marks: Final = (
|
||||
*block_marks,
|
||||
*(() if request_mark is None else (_Placed(_REQUEST, _ttl(request_mark), "written"),)),
|
||||
marks: Final = sorted(
|
||||
(
|
||||
_Placed(path, _ttl(cache_control), kind, _rank(path, leading_system_count, system_groups))
|
||||
for path, cache_control, kind in placements
|
||||
),
|
||||
key=lambda mark: _wire_position(mark.path, leading_system_count),
|
||||
)
|
||||
excess: Final = len(marks) - MAX_CACHE_CONTROL_BLOCKS
|
||||
dropped: Final[frozenset[_Path]] = (
|
||||
frozenset(tuple(mark.path for mark in marks if mark.kind != "written")[:excess])
|
||||
frozenset(tuple(mark.path for mark in marks if _droppable(mark))[:excess])
|
||||
if excess > 0 and drop_params
|
||||
else frozenset()
|
||||
)
|
||||
|
|
|
|||
|
|
@ -255,12 +255,59 @@ class AmazonConverseConfig(BaseConfig):
|
|||
)
|
||||
|
||||
@staticmethod
|
||||
def sends_tool_history_as_text(tools: object, non_default_params: Mapping[str, object]) -> bool:
|
||||
return (
|
||||
litellm.bedrock_neutralize_orphaned_tool_blocks
|
||||
and not tools
|
||||
and not any(map(non_default_params.__contains__, ("functions", "response_format", "web_search_options")))
|
||||
def may_send_tools(
|
||||
*, tools: object, functions: object, response_format: object, web_search_options: object
|
||||
) -> bool:
|
||||
"""Converse builds tools from ``functions``, a ``response_format`` or ``web_search_options`` too."""
|
||||
return bool(tools) or any(value is not None for value in (functions, response_format, web_search_options))
|
||||
|
||||
@staticmethod
|
||||
def sends_tool_history_as_text(may_send_tools: bool) -> bool:
|
||||
return litellm.bedrock_neutralize_orphaned_tool_blocks and not may_send_tools
|
||||
|
||||
@staticmethod
|
||||
def tool_config_cache_control(injection_points: object) -> ChatCompletionCachedContent | None:
|
||||
"""The ``cache_control`` of the cachePoint a ``tool_config`` injection point appends after the tools."""
|
||||
from litellm.integrations.anthropic_cache_control_hook import configured_injection_points
|
||||
|
||||
point: Final = next(
|
||||
(
|
||||
point
|
||||
for point in configured_injection_points(injection_points)
|
||||
if point.get("location") == "tool_config"
|
||||
),
|
||||
None,
|
||||
)
|
||||
if point is None:
|
||||
return None
|
||||
control: Final = point.get("control")
|
||||
return control if isinstance(control, dict) else ChatCompletionCachedContent(type="ephemeral")
|
||||
|
||||
@staticmethod
|
||||
def _sends_a_one_hour_cache_point(
|
||||
system_content_blocks: Sequence[SystemContentBlock], messages: Sequence[AllMessageValues]
|
||||
) -> bool:
|
||||
holders: Final = chain.from_iterable(
|
||||
(message, *parts_of(message_field(message, "content")), *parts_of(message_field(message, "tool_calls")))
|
||||
for message in messages
|
||||
)
|
||||
return any(message_field(block.get("cachePoint"), "ttl") == "1h" for block in system_content_blocks) or any(
|
||||
message_field(message_field(holder, "cache_control"), "ttl") == "1h" for holder in holders
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _tool_config_cache_point_control(
|
||||
control: ChatCompletionCachedContent,
|
||||
system_content_blocks: Sequence[SystemContentBlock],
|
||||
messages: Sequence[AllMessageValues],
|
||||
) -> ChatCompletionCachedContent:
|
||||
"""Claude rejects a 1h cachePoint read after a 5m one, so a tool_config cachePoint that sets no ttl takes 1h
|
||||
when a later one does."""
|
||||
if control.get("ttl") is not None or not AmazonConverseConfig._sends_a_one_hour_cache_point(
|
||||
system_content_blocks, messages
|
||||
):
|
||||
return control
|
||||
return ChatCompletionCachedContent(type="ephemeral", ttl="1h")
|
||||
|
||||
@staticmethod
|
||||
def _neutralize_orphaned_tool_blocks(
|
||||
|
|
@ -1999,27 +2046,36 @@ class AmazonConverseConfig(BaseConfig):
|
|||
)
|
||||
|
||||
# Append cachePoint to tools if cache_control_injection_points has tool_config
|
||||
cache_injection_points: Final = additional_request_params.pop("cache_control_injection_points", None)
|
||||
injection_points: Final = cast( # cast-ok: untyped request params
|
||||
object, additional_request_params.pop("cache_control_injection_points", None)
|
||||
)
|
||||
tool_config_control: Final = self.tool_config_cache_control(injection_points)
|
||||
if (
|
||||
cache_injection_points
|
||||
tool_config_control is not None
|
||||
and len(bedrock_tools) > 0
|
||||
and bedrock_model_accepts_cache_points(model, deployment_model)
|
||||
):
|
||||
for point in cache_injection_points:
|
||||
if point.get("location") == "tool_config":
|
||||
cache_point = self._build_cache_point_block(point.get("control"), model, deployment_model)
|
||||
bedrock_tools.append(ToolBlock(cachePoint=cache_point))
|
||||
# Spend attribution credits the gateway only for breakpoints it placed, and
|
||||
# this is the one place a tool_config point becomes one. The hook that reads
|
||||
# the configuration cannot record it: whether a cachePoint lands depends on
|
||||
# this provider and on the request carrying tools, neither of which it sees.
|
||||
if litellm_params is not None:
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
)
|
||||
bedrock_tools.append(
|
||||
ToolBlock(
|
||||
cachePoint=self._build_cache_point_block(
|
||||
self._tool_config_cache_point_control(
|
||||
tool_config_control, system_content_blocks, messages or ()
|
||||
),
|
||||
model,
|
||||
deployment_model,
|
||||
)
|
||||
)
|
||||
)
|
||||
# Spend attribution credits the gateway only for breakpoints it placed, and
|
||||
# this is the one place a tool_config point becomes one. The hook that reads
|
||||
# the configuration cannot record it: whether a cachePoint lands depends on
|
||||
# this provider and on the request carrying tools, neither of which it sees.
|
||||
if litellm_params is not None:
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
)
|
||||
|
||||
AnthropicCacheControlHook.record_gateway_injection(litellm_params, 1)
|
||||
break
|
||||
AnthropicCacheControlHook.record_gateway_injection(litellm_params, 1)
|
||||
|
||||
bedrock_tool_config: ToolConfigBlock | None = None
|
||||
if len(bedrock_tools) > 0:
|
||||
|
|
|
|||
|
|
@ -5594,16 +5594,29 @@ def completion(
|
|||
"drop_params" in kwargs
|
||||
and normalize_drop_params(cast(object, kwargs["drop_params"])) is True # cast-ok: untyped request kwargs
|
||||
)
|
||||
request_tools: Final = cast(list[object] | None, tools) # cast-ok: completion types tools as a bare List
|
||||
converse_may_send_tools: Final = AmazonConverseConfig.may_send_tools(
|
||||
tools=request_tools,
|
||||
functions=cast(object, functions), # cast-ok: completion types functions as a bare List
|
||||
response_format=cast(object, response_format), # cast-ok: completion types response_format as a bare dict
|
||||
web_search_options=web_search_options,
|
||||
)
|
||||
claude_cache: Final = (
|
||||
resolved_claude_prompt_cache(
|
||||
claude_prompt_cache(
|
||||
routed_messages,
|
||||
cast(list[object] | None, tools), # cast-ok: completion types tools as a bare List
|
||||
request_tools,
|
||||
non_default_params,
|
||||
drop_params=claude_drop_params,
|
||||
tool_history_as_text=isinstance(provider_config, AmazonConverseConfig)
|
||||
and AmazonConverseConfig.sends_tool_history_as_text(tools, non_default_params),
|
||||
one_mark_per_tool_result=isinstance(provider_config, AmazonConverseConfig),
|
||||
and AmazonConverseConfig.sends_tool_history_as_text(converse_may_send_tools),
|
||||
tool_config_cache_control=(
|
||||
AmazonConverseConfig.tool_config_cache_control(
|
||||
non_default_params.get("cache_control_injection_points")
|
||||
)
|
||||
if isinstance(provider_config, AmazonConverseConfig) and converse_may_send_tools
|
||||
else None
|
||||
),
|
||||
),
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from litellm.llms.anthropic.chat.prompt_cache_breakpoints import (
|
|||
claude_prompt_cache,
|
||||
resolved_claude_prompt_cache,
|
||||
)
|
||||
from litellm.types.llms.openai import ChatCompletionCachedContent
|
||||
|
||||
_EXPLICIT: Final = {"mode": "explicit"}
|
||||
_EPHEMERAL: Final = {"type": "ephemeral"}
|
||||
|
|
@ -21,7 +22,7 @@ def _cache(
|
|||
params: dict[str, object] | None = None,
|
||||
*,
|
||||
drop_params: bool = False,
|
||||
one_mark_per_tool_result: bool = False,
|
||||
tool_config_cache_control: ChatCompletionCachedContent | None = None,
|
||||
) -> ClaudePromptCache | UnsupportedPromptCacheParam:
|
||||
return claude_prompt_cache(
|
||||
messages,
|
||||
|
|
@ -29,7 +30,7 @@ def _cache(
|
|||
params or {},
|
||||
drop_params=drop_params,
|
||||
tool_history_as_text=False,
|
||||
one_mark_per_tool_result=one_mark_per_tool_result,
|
||||
tool_config_cache_control=tool_config_cache_control,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -39,10 +40,10 @@ def _resolved(
|
|||
params: dict[str, object] | None = None,
|
||||
*,
|
||||
drop_params: bool = False,
|
||||
one_mark_per_tool_result: bool = False,
|
||||
tool_config_cache_control: ChatCompletionCachedContent | None = None,
|
||||
) -> ClaudePromptCache:
|
||||
outcome: Final = _cache(
|
||||
messages, tools, params, drop_params=drop_params, one_mark_per_tool_result=one_mark_per_tool_result
|
||||
messages, tools, params, drop_params=drop_params, tool_config_cache_control=tool_config_cache_control
|
||||
)
|
||||
assert isinstance(outcome, ClaudePromptCache), outcome
|
||||
return outcome
|
||||
|
|
@ -502,7 +503,7 @@ def test_a_part_breakpoint_a_tool_results_own_breakpoint_replaces_is_still_repor
|
|||
assert outcome.value == {"mode": "explicit", "ttl": "30m"}
|
||||
|
||||
|
||||
def test_a_route_sending_one_mark_per_tool_result_counts_its_marked_parts_as_one_under_drop_params():
|
||||
def test_a_tool_results_marked_parts_count_as_one_mark_on_the_result_under_drop_params():
|
||||
messages: Final = [
|
||||
{"role": "system", "content": [_part("s", prompt_cache_breakpoint=_EXPLICIT)]},
|
||||
{"role": "user", "content": [_part("u1", prompt_cache_breakpoint=_EXPLICIT)]},
|
||||
|
|
@ -514,7 +515,7 @@ def test_a_route_sending_one_mark_per_tool_result_counts_its_marked_parts_as_one
|
|||
{"role": "user", "content": [_part("u2", prompt_cache_breakpoint=_EXPLICIT)]},
|
||||
]
|
||||
|
||||
result: Final = _resolved(messages, drop_params=True, one_mark_per_tool_result=True)
|
||||
result: Final = _resolved(messages, drop_params=True)
|
||||
|
||||
assert list(result.messages) == [
|
||||
{"role": "system", "content": [_part("s", cache_control=_EPHEMERAL)]},
|
||||
|
|
@ -524,7 +525,7 @@ def test_a_route_sending_one_mark_per_tool_result_counts_its_marked_parts_as_one
|
|||
]
|
||||
|
||||
|
||||
def test_a_route_sending_one_mark_per_tool_result_lets_a_client_marked_part_win_under_drop_params():
|
||||
def test_a_client_marked_part_of_a_tool_result_wins_over_its_breakpoints_under_drop_params():
|
||||
messages: Final = [
|
||||
{"role": "system", "content": [_part("s", prompt_cache_breakpoint=_EXPLICIT)]},
|
||||
{"role": "user", "content": [_part("u1", prompt_cache_breakpoint=_EXPLICIT)]},
|
||||
|
|
@ -536,7 +537,7 @@ def test_a_route_sending_one_mark_per_tool_result_lets_a_client_marked_part_win_
|
|||
{"role": "user", "content": [_part("u2", prompt_cache_breakpoint=_EXPLICIT)]},
|
||||
]
|
||||
|
||||
result: Final = _resolved(messages, drop_params=True, one_mark_per_tool_result=True)
|
||||
result: Final = _resolved(messages, drop_params=True)
|
||||
|
||||
assert list(result.messages) == [
|
||||
{"role": "system", "content": [_part("s", cache_control=_EPHEMERAL)]},
|
||||
|
|
@ -546,7 +547,7 @@ def test_a_route_sending_one_mark_per_tool_result_lets_a_client_marked_part_win_
|
|||
]
|
||||
|
||||
|
||||
def test_a_part_breakpoint_a_client_marked_part_replaces_on_a_one_mark_route_is_still_reported():
|
||||
def test_a_part_breakpoint_a_client_marked_part_replaces_is_still_reported():
|
||||
messages: Final = [
|
||||
{
|
||||
"role": "tool",
|
||||
|
|
@ -558,7 +559,7 @@ def test_a_part_breakpoint_a_client_marked_part_replaces_on_a_one_mark_route_is_
|
|||
}
|
||||
]
|
||||
|
||||
outcome: Final = _cache(messages, one_mark_per_tool_result=True)
|
||||
outcome: Final = _cache(messages)
|
||||
|
||||
assert isinstance(outcome, UnsupportedPromptCacheParam)
|
||||
assert outcome.value == {"mode": "explicit", "ttl": "30m"}
|
||||
|
|
@ -576,3 +577,129 @@ def test_a_null_tool_level_breakpoint_leaves_the_function_breakpoint_in_force():
|
|||
result: Final = _resolved([{"role": "user", "content": "hi"}], tools)
|
||||
|
||||
assert list(result.tools or ()) == [{"type": "function", "function": {"name": "t"}, "cache_control": _EPHEMERAL}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("between", "ask", "drop_params", "sent_reminder", "sent_ask"),
|
||||
[
|
||||
pytest.param((), _EXPLICIT, False, _ONE_HOUR, _ONE_HOUR, id="free-mark-takes-one-hour"),
|
||||
pytest.param((), _EXPLICIT, True, _ONE_HOUR, _ONE_HOUR, id="free-mark-takes-one-hour-under-drop-params"),
|
||||
pytest.param(
|
||||
({"role": "user", "content": "more"}, {"role": "system", "content": "note"}),
|
||||
_EXPLICIT,
|
||||
False,
|
||||
_ONE_HOUR,
|
||||
_ONE_HOUR,
|
||||
id="past-a-later-system-message",
|
||||
),
|
||||
pytest.param(
|
||||
({"role": "user", "content": "more"}, {"role": "assistant", "content": "b"}),
|
||||
_EXPLICIT,
|
||||
False,
|
||||
_ONE_HOUR,
|
||||
_EPHEMERAL,
|
||||
id="after-the-next-assistant-turn",
|
||||
),
|
||||
pytest.param(
|
||||
(),
|
||||
{"mode": "explicit", "ttl": "5m"},
|
||||
True,
|
||||
_EPHEMERAL,
|
||||
{"type": "ephemeral", "ttl": "5m"},
|
||||
id="set-ttls-loosen-under-drop-params",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_marks_on_a_mid_conversation_system_message_and_the_turns_it_may_move_past_settle_together(
|
||||
between: tuple[dict[str, object], ...],
|
||||
ask: dict[str, object],
|
||||
drop_params: bool,
|
||||
sent_reminder: dict[str, object],
|
||||
sent_ask: dict[str, object],
|
||||
):
|
||||
messages: Final = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "q"},
|
||||
{"role": "assistant", "content": "a"},
|
||||
{"role": "system", "content": [_part("reminder", prompt_cache_breakpoint={"mode": "explicit", "ttl": "1h"})]},
|
||||
*between,
|
||||
{"role": "user", "content": [_part("ask", prompt_cache_breakpoint=ask)]},
|
||||
]
|
||||
|
||||
result: Final = _resolved(messages, drop_params=drop_params)
|
||||
|
||||
assert (result.messages[3], result.messages[-1]) == (
|
||||
{"role": "system", "content": [_part("reminder", cache_control=sent_reminder)]},
|
||||
{"role": "user", "content": [_part("ask", cache_control=sent_ask)]},
|
||||
)
|
||||
|
||||
|
||||
def test_a_tool_config_cache_point_takes_one_of_claudes_four_and_is_never_dropped_under_drop_params():
|
||||
messages: Final = [
|
||||
{"role": "system", "content": [_part("s", prompt_cache_breakpoint=_EXPLICIT)]},
|
||||
{"role": "user", "content": [_part(str(index), prompt_cache_breakpoint=_EXPLICIT) for index in range(3)]},
|
||||
]
|
||||
|
||||
result: Final = _resolved(
|
||||
messages,
|
||||
[{"type": "function", "function": {"name": "t"}}],
|
||||
drop_params=True,
|
||||
tool_config_cache_control=ChatCompletionCachedContent(type="ephemeral"),
|
||||
)
|
||||
|
||||
assert list(result.messages) == [
|
||||
{"role": "system", "content": [_part("s")]},
|
||||
{"role": "user", "content": [_part(str(index), cache_control=_EPHEMERAL) for index in range(3)]},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_config", "tool_marker", "ask_marker", "sent_tool", "sent_ask"),
|
||||
[
|
||||
pytest.param(
|
||||
ChatCompletionCachedContent(type="ephemeral"),
|
||||
None,
|
||||
{"mode": "explicit", "ttl": "1h"},
|
||||
{},
|
||||
_ONE_HOUR,
|
||||
id="free-keeps-a-later-hour",
|
||||
),
|
||||
pytest.param(
|
||||
ChatCompletionCachedContent(type="ephemeral", ttl="5m"),
|
||||
None,
|
||||
{"mode": "explicit", "ttl": "1h"},
|
||||
{},
|
||||
_EPHEMERAL,
|
||||
id="five-minutes-loosens-a-later-hour",
|
||||
),
|
||||
pytest.param(
|
||||
ChatCompletionCachedContent(type="ephemeral", ttl="1h"),
|
||||
_EXPLICIT,
|
||||
None,
|
||||
{"cache_control": _ONE_HOUR},
|
||||
None,
|
||||
id="one-hour-lifts-an-earlier-tool-mark",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_tool_config_cache_point_settles_after_the_tools_under_drop_params(
|
||||
tool_config: ChatCompletionCachedContent,
|
||||
tool_marker: dict[str, object] | None,
|
||||
ask_marker: dict[str, object] | None,
|
||||
sent_tool: dict[str, object],
|
||||
sent_ask: dict[str, object] | None,
|
||||
):
|
||||
tool: Final = {"type": "function", "function": {"name": "t"}}
|
||||
ask: Final = _part("ask") if ask_marker is None else _part("ask", prompt_cache_breakpoint=ask_marker)
|
||||
|
||||
result: Final = _resolved(
|
||||
[{"role": "user", "content": [ask]}],
|
||||
[tool if tool_marker is None else {**tool, "prompt_cache_breakpoint": tool_marker}],
|
||||
drop_params=True,
|
||||
tool_config_cache_control=tool_config,
|
||||
)
|
||||
|
||||
assert list(result.tools or ()) == [{**tool, **sent_tool}]
|
||||
assert list(result.messages) == [
|
||||
{"role": "user", "content": [_part("ask") if sent_ask is None else _part("ask", cache_control=sent_ask)]}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -5947,6 +5947,67 @@ def test_cache_control_injection_tool_config_not_added_without_injection_point()
|
|||
assert all("cachePoint" not in tool for tool in tools)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("messages", "sent_cache_point"),
|
||||
[
|
||||
pytest.param(
|
||||
[
|
||||
{"role": "system", "content": "sys", "cache_control": {"type": "ephemeral", "ttl": "1h"}},
|
||||
{"role": "user", "content": "What is the weather?"},
|
||||
],
|
||||
{"type": "default", "ttl": "1h"},
|
||||
id="one-hour-system",
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What is the weather?",
|
||||
"cache_control": {"type": "ephemeral", "ttl": "1h"},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
{"type": "default", "ttl": "1h"},
|
||||
id="one-hour-message",
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What is the weather?",
|
||||
"cache_control": {"type": "ephemeral", "ttl": "5m"},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
{"type": "default"},
|
||||
id="five-minute-message",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_tool_config_cache_point_without_a_ttl_takes_one_hour_when_a_later_cache_point_does(
|
||||
messages: list[AllMessageValues], sent_cache_point: dict[str, str]
|
||||
):
|
||||
result: Final = AmazonConverseConfig()._transform_request(
|
||||
model="us.anthropic.claude-opus-5-5",
|
||||
messages=messages,
|
||||
optional_params={
|
||||
"tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}],
|
||||
"cache_control_injection_points": [{"location": "tool_config"}],
|
||||
},
|
||||
litellm_params={},
|
||||
)
|
||||
|
||||
assert result["toolConfig"]["tools"][-1] == {"cachePoint": sent_cache_point}
|
||||
|
||||
|
||||
def test_cache_control_injection_tool_config_honors_ttl_for_supported_model(monkeypatch):
|
||||
"""
|
||||
Regression test: cache_control_injection_points with location=tool_config
|
||||
|
|
|
|||
|
|
@ -759,6 +759,96 @@ def test_a_breakpoint_reaches_bedrock_mantle_claude_as_cache_control(respx_mock:
|
|||
]
|
||||
|
||||
|
||||
_WEATHER_TOOL: Final = {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}
|
||||
|
||||
|
||||
def test_breakpoints_on_converse_tool_history_beside_a_json_schema_response_format_keep_to_four_under_drop_params(
|
||||
respx_mock: respx.MockRouter,
|
||||
):
|
||||
route: Final = respx_mock.post(url__regex=_CONVERSE_ROUTE).respond(json=_CONVERSE_REPLY)
|
||||
|
||||
litellm.completion(
|
||||
model="bedrock/converse/us.anthropic.claude-opus-5-5",
|
||||
messages=[
|
||||
{"role": "system", "content": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": _BREAKPOINT}]},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": str(index), "prompt_cache_breakpoint": _BREAKPOINT} for index in range(3)
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{**_LOADED_FUNCTION_CALL, "prompt_cache_breakpoint": _BREAKPOINT}],
|
||||
},
|
||||
{**_WEATHER_RESULT, "prompt_cache_breakpoint": _BREAKPOINT},
|
||||
{"role": "user", "content": "And tomorrow?"},
|
||||
],
|
||||
response_format={"type": "json_schema", "json_schema": {"name": "forecast", "schema": {"type": "object"}}},
|
||||
drop_params=True,
|
||||
**_FAKE_AWS,
|
||||
)
|
||||
|
||||
assert _converse_cache_points(json.loads(route.calls.last.request.content)) == ("message",) * 4
|
||||
|
||||
|
||||
def test_breakpoints_on_one_tool_results_parts_reach_claude_as_one_mark_on_the_result(respx_mock: respx.MockRouter):
|
||||
route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").respond(
|
||||
json={
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-opus-5-5",
|
||||
"content": [{"type": "text", "text": "18C"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1},
|
||||
}
|
||||
)
|
||||
|
||||
litellm.completion(
|
||||
model="anthropic/claude-opus-5-5",
|
||||
messages=[
|
||||
_USER_TURN,
|
||||
{"role": "assistant", "content": None, "tool_calls": [_LOADED_FUNCTION_CALL]},
|
||||
{
|
||||
**_WEATHER_RESULT,
|
||||
"content": [
|
||||
{"type": "text", "text": text, "prompt_cache_breakpoint": _BREAKPOINT} for text in ("18C", "sunny")
|
||||
],
|
||||
},
|
||||
],
|
||||
tools=[_WEATHER_TOOL],
|
||||
api_key="fake",
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["messages"][2]["content"] == [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_01weather",
|
||||
"content": [{"type": "text", "text": "18C"}, {"type": "text", "text": "sunny"}],
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_a_tool_breakpoint_before_a_one_hour_tool_config_cache_point_reaches_converse_as_one_hour(
|
||||
respx_mock: respx.MockRouter,
|
||||
):
|
||||
route: Final = respx_mock.post(url__regex=_CONVERSE_ROUTE).respond(json=_CONVERSE_REPLY)
|
||||
|
||||
litellm.completion(
|
||||
model="bedrock/converse/us.anthropic.claude-opus-5-5",
|
||||
messages=[_USER_TURN],
|
||||
tools=[{**_WEATHER_TOOL, "prompt_cache_breakpoint": _BREAKPOINT}],
|
||||
cache_control_injection_points=[{"location": "tool_config", "control": {"type": "ephemeral", "ttl": "1h"}}],
|
||||
**_FAKE_AWS,
|
||||
)
|
||||
|
||||
sent_tools: Final = json.loads(route.calls.last.request.content)["toolConfig"]["tools"]
|
||||
assert [tool["cachePoint"] for tool in sent_tools if "cachePoint" in tool] == [{"type": "default", "ttl": "1h"}] * 2
|
||||
|
||||
|
||||
def test_completion_keeps_a_function_call_whose_id_only_looks_like_a_claude_server_tool(
|
||||
respx_mock: respx.MockRouter, openai_api_response
|
||||
):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue