scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 200

feat(anthropic): map prompt_cache_breakpoint to cache_control for Claude

Browse source 
A prompt_cache_breakpoint on a content part, message, tool definition or tool call bound for Claude becomes a cache_control mark on the same block, on the Anthropic, Vertex AI, Bedrock invoke, Bedrock Converse and Azure AI routes. A client cache_control on the same block wins and the breakpoint ttl is not forwarded. Requests that carried a breakpoint are held to Claude's four-breakpoint cap: with drop_params the latest four survive in wire order, without it the request is a 400 before any provider call. prompt_cache_options is rejected as unsupported unless drop_params drops it. is_cache_control_set and the cache-control hook count a breakpoint like a cache_control. MAX_CACHE_CONTROL_BLOCKS moves to litellm.constants
This commit is contained in:
ryan-crabbe-berri 2026-10-13 18:53:20 -07:00
parent c581c85559
commit 432c8b5aa5
7 changed files with 414 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
litellm/constants.py
View file

@ -586,6 +586,9 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60
BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = int(os.getenv("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024))
# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024
# Anthropic (and Bedrock Claude) reject requests with more than 4 cache_control
# breakpoints: "A maximum of 4 blocks with cache_control may be provided."
MAX_CACHE_CONTROL_BLOCKS: Final = 4
REPLICATE_POLLING_DELAY_SECONDS: Final = float(os.getenv("REPLICATE_POLLING_DELAY_SECONDS", 0.5))
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = int(os.getenv("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 4096))
DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096

8
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -19,6 +19,7 @@ from urllib.parse import urlparse
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.constants import MAX_CACHE_CONTROL_BLOCKS
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
@ -58,11 +59,6 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
# Anthropic (and Bedrock Claude) reject requests with more than 4 cache_control
# breakpoints: "A maximum of 4 blocks with cache_control may be provided."
MAX_CACHE_CONTROL_BLOCKS: Final = 4
CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6)
_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?")
@ -228,7 +224,7 @@ def _as_object_list(value: object | None) -> list[object] | None:
def _tool_call_carries_cache_breakpoint(tool_call: object, message: object) -> bool:
if _attribute_or_key(tool_call, "cache_control") is None:
if not _carries_cache_breakpoint(tool_call):
return False
return not tool_call_is_rebuilt_as_server_tool_use(

129
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -7,6 +7,7 @@ import json
import mimetypes
import re
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from collections.abc import Set as AbstractSet
from itertools import groupby, islice
from os import PathLike
from pathlib import Path
@ -19,6 +20,7 @@ from litellm.router_utils.batch_utils import InMemoryFile
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
ChatCompletionCachedContent,
ChatCompletionFileObject,
ChatCompletionImageObject,
ChatCompletionReasoningItem,
@ -1805,6 +1807,133 @@ def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
)
def prompt_cache_breakpoint_as_cache_control(block: _MarkedT) -> _MarkedT:
"""OpenAI's ``prompt_cache_breakpoint`` rewritten as Anthropic's ``cache_control`` on the same block.
A ``cache_control`` the client already wrote wins. OpenAI's 30m TTL has no Anthropic equivalent and is dropped."""
if block.get("prompt_cache_breakpoint") is None:
return block
unmarked: Final = {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"}
marked: Final = (
unmarked
if unmarked.get("cache_control") is not None
else {**unmarked, "cache_control": ChatCompletionCachedContent(type="ephemeral")}
)
return cast(_MarkedT, marked) # cast-ok: same block shape as the input with the marker key renamed
def _as_mapping(value: object) -> Mapping[str, object] | None:
return value if isinstance(value, Mapping) else None # pyright: ignore[reportUnknownVariableType] # keys are read one by one
def _as_list(value: object) -> Sequence[object] | None:
return value if isinstance(value, list) else None # pyright: ignore[reportUnknownVariableType] # items are parsed one by one
def message_fields(message: AllMessageValues) -> Mapping[str, object]:
"""The message as a plain mapping, for code that reads keys the role-specific TypedDicts do not all declare."""
return message
def _without_key(block: Mapping[str, object], key: str) -> Mapping[str, object]:
return {name: value for name, value in block.items() if name != key}
def _block_with_prompt_cache_breakpoint_as_cache_control(block: object) -> object:
fields: Final = _as_mapping(block)
return prompt_cache_breakpoint_as_cache_control(fields) if fields is not None else block
def message_with_prompt_cache_breakpoints_as_cache_control(message: AllMessageValues) -> AllMessageValues:
"""The message-level mark, every content part's mark and every tool call's mark renamed to ``cache_control``."""
renamed: Final = prompt_cache_breakpoint_as_cache_control(message)
fields: Final = message_fields(renamed)
content: Final = _as_list(fields.get("content"))
tool_calls: Final = _as_list(fields.get("tool_calls"))
if content is None and tool_calls is None:
return renamed
return cast( # cast-ok: same message shape as the input with the marker key renamed on its blocks
AllMessageValues,
{
**fields,
**(
{"content": [_block_with_prompt_cache_breakpoint_as_cache_control(block) for block in content]}
if content is not None
else {}
),
**(
{"tool_calls": [_block_with_prompt_cache_breakpoint_as_cache_control(call) for call in tool_calls]}
if tool_calls is not None
else {}
),
},
)
def messages_with_prompt_cache_breakpoints_as_cache_control(
messages: Sequence[AllMessageValues],
) -> tuple[AllMessageValues, ...]:
return tuple(message_with_prompt_cache_breakpoints_as_cache_control(message) for message in messages)
def tool_with_prompt_cache_breakpoint_as_cache_control(tool: Mapping[str, object]) -> Mapping[str, object]:
"""A tool definition's mark, written at the top level (Anthropic shape) or under ``function`` (OpenAI shape),
renamed to a top-level ``cache_control``, which every Claude transform reads."""
function: Final = _as_mapping(tool.get("function"))
function_marked: Final = function is not None and function.get("prompt_cache_breakpoint") is not None
if tool.get("prompt_cache_breakpoint") is None and not function_marked:
return tool
unmarked_function: Final = _without_key(function, "prompt_cache_breakpoint") if function is not None else None
unmarked: Final = {
**_without_key(tool, "prompt_cache_breakpoint"),
**({"function": unmarked_function} if unmarked_function is not None else {}),
}
already_marked: Final = unmarked.get("cache_control") is not None or (
unmarked_function is not None and unmarked_function.get("cache_control") is not None
)
return unmarked if already_marked else {**unmarked, "cache_control": ChatCompletionCachedContent(type="ephemeral")}
def _block_without_cache_control(block: object) -> object:
fields: Final = _as_mapping(block)
return _without_key(fields, "cache_control") if fields is not None else block
def _blocks_without_cache_control(blocks: Sequence[object], dropped: AbstractSet[int]) -> Sequence[object]:
return [
_block_without_cache_control(block) if position in dropped else block for position, block in enumerate(blocks)
]
def message_without_cache_control_marks(
message: AllMessageValues, *, message_mark: bool, parts: AbstractSet[int], tool_calls: AbstractSet[int]
) -> AllMessageValues:
"""``message`` minus its message-level ``cache_control`` (when ``message_mark``) and the marks on the content
parts and tool calls at the given positions."""
fields: Final = message_fields(message)
kept: Final = _without_key(fields, "cache_control") if message_mark else fields
content: Final = _as_list(kept.get("content"))
calls: Final = _as_list(kept.get("tool_calls"))
return cast( # cast-ok: same message shape as the input minus the dropped marks
AllMessageValues,
{
**kept,
**({"content": _blocks_without_cache_control(content, parts)} if content is not None else {}),
**({"tool_calls": _blocks_without_cache_control(calls, tool_calls)} if calls is not None else {}),
},
)
def _tool_block_with_prompt_cache_breakpoint_as_cache_control(tool: object) -> object:
fields: Final = _as_mapping(tool)
return tool_with_prompt_cache_breakpoint_as_cache_control(fields) if fields is not None else tool
def tools_with_prompt_cache_breakpoints_as_cache_control(tools: object) -> tuple[object, ...]:
"""Every tool definition in ``tools`` (anything but a list is treated as none) with its mark renamed."""
return tuple(_tool_block_with_prompt_cache_breakpoint_as_cache_control(tool) for tool in _as_list(tools) or ())
LITELLM_INTERNAL_MESSAGE_FIELDS: Final = frozenset({"thinking_blocks", "reasoning_content", "provider_specific_fields"})

189
litellm/llms/anthropic/chat/prompt_cache_breakpoints.py Normal file
View file

@ -0,0 +1,189 @@
"""OpenAI's ``prompt_cache_breakpoint`` on a request bound for Claude.
The marks become Anthropic's ``cache_control`` on the same blocks, and a request that carried at least one is held
to Claude's cap of four breakpoints, which a client writing the OpenAI dialect cannot see. ``prompt_cache_options``
has no Claude equivalent and is rejected or dropped like any other unsupported param."""
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from itertools import chain
from typing import Final, Literal, TypeAlias, assert_never
from litellm._logging import verbose_logger
from litellm.constants import MAX_CACHE_CONTROL_BLOCKS
from litellm.exceptions import BadRequestError, UnsupportedParamsError
from litellm.litellm_core_utils.prompt_templates.common_utils import (
message_without_cache_control_marks,
messages_with_prompt_cache_breakpoints_as_cache_control,
tools_with_prompt_cache_breakpoints_as_cache_control,
)
from litellm.types.llms.openai import AllMessageValues
PROMPT_CACHE_BREAKPOINT_ON_TOOLS: Final = "_litellm_prompt_cache_breakpoint_on_tools"
_Site: TypeAlias = tuple[Literal["tool", "message", "part", "tool_call"], int, int]
@dataclass(frozen=True, slots=True)
class ClaudeCacheMarks:
messages: tuple[AllMessageValues, ...]
tools: tuple[object, ...]
@dataclass(frozen=True, slots=True)
class CacheBreakpointCapExceeded:
found: int
def _as_mapping(value: object) -> Mapping[str, object] | None:
return value if isinstance(value, Mapping) else None # pyright: ignore[reportUnknownVariableType] # keys are read one by one
def _as_blocks(value: object) -> Sequence[object] | None:
return value if isinstance(value, list) else None # pyright: ignore[reportUnknownVariableType] # blocks are parsed one by one
def _carries(block: object, key: str) -> bool:
fields: Final = _as_mapping(block)
return fields is not None and fields.get(key) is not None
def _tool_carries(tool: object, key: str) -> bool:
fields: Final = _as_mapping(tool)
return fields is not None and (fields.get(key) is not None or _carries(fields.get("function"), key))
def tools_carry_prompt_cache_breakpoint(tools: object) -> bool:
return any(_tool_carries(tool, "prompt_cache_breakpoint") for tool in _as_blocks(tools) or ())
def _message_sites(index: int, message: object, key: str) -> Iterator[_Site]:
fields: Final = _as_mapping(message)
if fields is None:
return
if fields.get(key) is not None:
yield ("message", index, -1)
for position, part in enumerate(_as_blocks(fields.get("content")) or ()):
if _carries(part, key):
yield ("part", index, position)
for position, call in enumerate(_as_blocks(fields.get("tool_calls")) or ()):
if _carries(call, key):
yield ("tool_call", index, position)
def _sites(messages: Sequence[object], tools: Sequence[object], key: str) -> tuple[_Site, ...]:
"""Every block carrying ``key`` in wire order: tool definitions first, then the messages as listed."""
return (
*(("tool", index, -1) for index, tool in enumerate(tools) if _tool_carries(tool, key)),
*chain.from_iterable(_message_sites(index, message, key) for index, message in enumerate(messages)),
)
def _without_cache_control(block: Mapping[str, object]) -> Mapping[str, object]:
return {key: value for key, value in block.items() if key != "cache_control"}
def _tool_without_cache_control(tool: object) -> object:
fields: Final = _as_mapping(tool)
if fields is None:
return tool
function: Final = _as_mapping(fields.get("function"))
return {
**_without_cache_control(fields),
**({"function": _without_cache_control(function)} if function is not None else {}),
}
def _dropped_positions(dropped: frozenset[_Site], kind: str, index: int) -> frozenset[int]:
return frozenset(
position for site_kind, site_index, position in dropped if (site_kind, site_index) == (kind, index)
)
def _message_without(index: int, message: AllMessageValues, dropped: frozenset[_Site]) -> AllMessageValues:
return message_without_cache_control_marks(
message,
message_mark=("message", index, -1) in dropped,
parts=_dropped_positions(dropped, "part", index),
tool_calls=_dropped_positions(dropped, "tool_call", index),
)
def claude_cache_marks(
messages: Sequence[AllMessageValues],
tools: object,
*,
automatic_breakpoint: bool,
drop_params: bool,
tools_carried_breakpoint: bool = False,
) -> ClaudeCacheMarks | CacheBreakpointCapExceeded:
"""The request's ``prompt_cache_breakpoint`` marks renamed to ``cache_control`` and, when it carried any, held to
Claude's cap: ``drop_params`` keeps the latest four in wire order, otherwise the excess is reported. A request
that carried none is forwarded as written so the provider's own answer reaches the client.
``automatic_breakpoint`` is the request-level ``cache_control`` (Anthropic's automatic caching), which takes one
slot of its own and is never dropped. ``tools_carried_breakpoint`` covers tools already mapped to Anthropic's
shape, where the mark was renamed before this census ran."""
listed_tools: Final = _as_blocks(tools) or ()
if not tools_carried_breakpoint and not _sites(messages, listed_tools, "prompt_cache_breakpoint"):
return ClaudeCacheMarks(tuple(messages), tuple(listed_tools))
renamed_messages: Final = messages_with_prompt_cache_breakpoints_as_cache_control(messages)
renamed_tools: Final = tools_with_prompt_cache_breakpoints_as_cache_control(listed_tools)
sites: Final = _sites(renamed_messages, renamed_tools, "cache_control")
found: Final = len(sites) + int(automatic_breakpoint)
if found <= MAX_CACHE_CONTROL_BLOCKS:
return ClaudeCacheMarks(renamed_messages, renamed_tools)
if not drop_params:
return CacheBreakpointCapExceeded(found)
dropped: Final = frozenset(sites[: found - MAX_CACHE_CONTROL_BLOCKS])
verbose_logger.debug(
"Dropping the earliest %d of %d cache breakpoints: Claude accepts at most %d.",
len(dropped),
found,
MAX_CACHE_CONTROL_BLOCKS,
)
return ClaudeCacheMarks(
messages=tuple(_message_without(index, message, dropped) for index, message in enumerate(renamed_messages)),
tools=tuple(
_tool_without_cache_control(tool) if ("tool", index, -1) in dropped else tool
for index, tool in enumerate(renamed_tools)
),
)
def resolved_claude_cache_marks(
outcome: ClaudeCacheMarks | CacheBreakpointCapExceeded, model: str, llm_provider: str
) -> ClaudeCacheMarks:
match outcome:
case ClaudeCacheMarks():
return outcome
case CacheBreakpointCapExceeded(found=found):
raise BadRequestError(
message=(
f"Claude accepts at most {MAX_CACHE_CONTROL_BLOCKS} cache breakpoints, found {found}. "
"Set drop_params=True (proxy: litellm_settings: drop_params: true) to keep the latest four."
),
model=model,
llm_provider=llm_provider,
)
case _:
assert_never(outcome)
def reject_prompt_cache_options(model: str, options: object, drop_params: bool) -> None:
"""``prompt_cache_options`` has already been popped from the request; this decides whether that was allowed."""
if options is None:
return
if drop_params:
verbose_logger.debug(
"Dropping prompt_cache_options=%r for %s: Claude caches through cache_control.", options, model
)
return
raise UnsupportedParamsError(
message=(
f"{model} does not support prompt_cache_options={options!r}. Claude caches through cache_control "
"marks (or prompt_cache_breakpoint on the OpenAI dialect). To drop unsupported params, set "
"`litellm.drop_params = True`."
),
model=model,
status_code=400,
)

47
litellm/llms/anthropic/chat/transformation.py
View file

@ -25,6 +25,7 @@ from litellm.constants import (
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.prompt_templates.common_utils import (
sanitize_input_schema_for_anthropic,
tools_with_prompt_cache_breakpoints_as_cache_control,
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
RemoteMedia,
@ -110,6 +111,13 @@ from ..common_utils import (
requires_native_compaction_beta,
strip_advisor_blocks_from_messages,
)
from .prompt_cache_breakpoints import (
PROMPT_CACHE_BREAKPOINT_ON_TOOLS,
claude_cache_marks,
reject_prompt_cache_options,
resolved_claude_cache_marks,
tools_carry_prompt_cache_breakpoint,
)
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -1568,12 +1576,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if param == "max_tokens" or param == "max_completion_tokens":
optional_params["max_tokens"] = value if isinstance(value, int) else max(1, int(round(value)))
elif param == "tools":
anthropic_tools, mcp_servers = self._map_tools(value)
requested_tools = cast(object, value) # cast-ok: the loop yields untyped values
anthropic_tools, mcp_servers = self._map_tools(
list(tools_with_prompt_cache_breakpoints_as_cache_control(requested_tools))
)
optional_params = self._add_tools_to_optional_params(
optional_params=optional_params, tools=anthropic_tools
)
if mcp_servers:
optional_params["mcp_servers"] = mcp_servers
if tools_carry_prompt_cache_breakpoint(requested_tools):
optional_params[PROMPT_CACHE_BREAKPOINT_ON_TOOLS] = True # rebind-ok: out-param store
elif param == "tool_choice" or param == "parallel_tool_calls":
_tool_choice: AnthropicMessagesToolChoice | None = self._map_tool_choice(
tool_choice=non_default_params.get("tool_choice"),
@ -2010,10 +2023,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"has no thinking_blocks. The model won't use extended thinking for this turn."
)
drop_params: Final = litellm.drop_params or litellm_params.get("drop_params") is True
AnthropicConfig._maybe_drop_speed_param(
model=model,
optional_params=optional_params,
drop_params=litellm.drop_params or litellm_params.get("drop_params") is True,
drop_params=drop_params,
custom_llm_provider=self.custom_llm_provider,
)
@ -2023,6 +2037,24 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
custom_llm_provider=self._resolved_provider,
)
body: Final = cast(dict[str, object], optional_params) # cast-ok: BaseConfig passes an untyped dict
reject_prompt_cache_options(
model=model, options=body.pop("prompt_cache_options", None), drop_params=drop_params
)
cache_marks: Final = resolved_claude_cache_marks(
claude_cache_marks(
messages=messages,
tools=body.get("tools"),
automatic_breakpoint=body.get("cache_control") is not None,
drop_params=drop_params,
tools_carried_breakpoint=bool(body.pop(PROMPT_CACHE_BREAKPOINT_ON_TOOLS, False)),
),
model=model,
llm_provider=self._resolved_provider,
)
if "tools" in body:
optional_params["tools"] = list(cache_marks.tools) # rebind-ok: out-param store
# === Tool-name sanitization (single chokepoint) ===
# Anthropic enforces ^[a-zA-Z0-9_-]{1,128}$ on every tool name. We
# sanitize *here* -- not in map_openai_params -- because:
@ -2046,15 +2078,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
optional_params=optional_params,
)
if _name_forward_map:
messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map)
marked_messages: Final = (
self._rewrite_tool_names_in_messages(list(cache_marks.messages), _name_forward_map)
if _name_forward_map
else list(cache_marks.messages)
)
if _name_reverse_map and isinstance(litellm_params, dict):
litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
# Only the leading system run becomes the top-level system prompt. A later
# system message stays in the conversation: hoisting it rewrites the cached
# prefix and re-bills the whole history at cache-write pricing (#36559).
leading_system_run, later_messages = split_leading_system_run(messages)
leading_system_run, later_messages = split_leading_system_run(marked_messages)
anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run))
# Handling anthropic API Prompt Caching
if len(anthropic_system_message_list) > 0:
@ -2075,7 +2110,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
except Exception as e:
raise AnthropicError(
status_code=400,
message=f"{e}\nReceived Messages={messages}",
message=f"{e}\nReceived Messages={marked_messages}",
) # don't use verbose_logger.exception, if exception is raised
self.update_headers_with_optional_anthropic_beta(

12
litellm/llms/anthropic/common_utils.py
View file

@ -106,6 +106,7 @@ _INFERENCE_PROFILE_MINOR_RE: Final = re.compile(r":\d+$")
_DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$")
_DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)")
_CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
CACHE_MARK_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
_CLAUDE_CODE_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
_CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
@ -461,18 +462,15 @@ class AnthropicModelInfo(BaseLLMModelInfo):
_workload_identity_eligible: ClassVar[bool] = True
def is_cache_control_set(self, messages: list[AllMessageValues]) -> bool:
"""
Return if {"cache_control": ..} in message content block
Used to check if anthropic prompt caching headers need to be set.
"""
"""Whether a message or one of its content blocks carries a cache mark, Anthropic's ``cache_control`` or
OpenAI's ``prompt_cache_breakpoint`` (which the chat transform renames on the way to Claude)."""
for message in messages:
if message.get("cache_control", None) is not None:
if any(message.get(key) is not None for key in CACHE_MARK_KEYS):
return True
_message_content = message.get("content")
if _message_content is not None and isinstance(_message_content, list):
for content in _message_content:
if isinstance(content, dict) and "cache_control" in content:
if isinstance(content, dict) and any(key in content for key in CACHE_MARK_KEYS):
return True
return False

47
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -44,6 +44,11 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
message_field,
parts_of,
)
from litellm.llms.anthropic.chat.prompt_cache_breakpoints import (
claude_cache_marks,
reject_prompt_cache_options,
resolved_claude_cache_marks,
)
from litellm.llms.anthropic.chat.transformation import (
DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING,
DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,
@ -2012,6 +2017,36 @@ class AmazonConverseConfig(BaseConfig):
return data
def _messages_with_claude_cache_marks(
self,
model: str,
messages: Sequence[AllMessageValues],
optional_params: dict[str, object], # mutable-ok: the request body is built from this out-param
litellm_params: Mapping[str, object],
) -> tuple[AllMessageValues, ...]:
"""OpenAI ``prompt_cache_breakpoint`` marks as ``cache_control`` so the factory emits ``cachePoint`` blocks,
held to the provider's cap, with ``prompt_cache_options`` rejected or dropped. Models that never emit
``cachePoint`` blocks are left as written."""
if not bedrock_model_accepts_cache_points(model):
return tuple(messages)
drop_params: Final = litellm.drop_params or litellm_params.get("drop_params") is True
reject_prompt_cache_options(
model=model, options=optional_params.pop("prompt_cache_options", None), drop_params=drop_params
)
cache_marks: Final = resolved_claude_cache_marks(
claude_cache_marks(
messages=messages,
tools=optional_params.get("tools"),
automatic_breakpoint=optional_params.get("cache_control") is not None,
drop_params=drop_params,
),
model=model,
llm_provider=self.custom_llm_provider or "bedrock",
)
if "tools" in optional_params:
optional_params["tools"] = list(cache_marks.tools) # rebind-ok: out-param store
return cache_marks.messages
async def _async_transform_request(
self,
model: str,
@ -2020,7 +2055,11 @@ class AmazonConverseConfig(BaseConfig):
litellm_params: dict,
headers: dict | None = None,
) -> RequestObject:
messages, system_content_blocks = self._transform_system_message(messages, model=model)
body: Final = cast(dict[str, object], optional_params) # cast-ok: BaseConfig passes an untyped dict
params: Final = cast(Mapping[str, object], litellm_params) # cast-ok: BaseConfig passes an untyped dict
messages, system_content_blocks = self._transform_system_message(
list(self._messages_with_claude_cache_marks(model, messages, body, params)), model=model
)
# Convert last user message to guarded_text if guardrailConfig is present
messages = self._convert_consecutive_user_messages_to_guarded_text(
@ -2088,7 +2127,11 @@ class AmazonConverseConfig(BaseConfig):
litellm_params: dict,
headers: dict | None = None,
) -> RequestObject:
messages, system_content_blocks = self._transform_system_message(messages, model=model)
body: Final = cast(dict[str, object], optional_params) # cast-ok: BaseConfig passes an untyped dict
params: Final = cast(Mapping[str, object], litellm_params) # cast-ok: BaseConfig passes an untyped dict
messages, system_content_blocks = self._transform_system_message(
list(self._messages_with_claude_cache_marks(model, messages, body, params)), model=model
)
# Convert last user message to guarded_text if guardrailConfig is present
messages = self._convert_consecutive_user_messages_to_guarded_text(