From 432c8b5aa5d9c7831966f91fe8a85bd4dd4b2866 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 13 Oct 2026 18:53:20 -0700 Subject: [PATCH] feat(anthropic): map prompt_cache_breakpoint to cache_control for Claude A prompt_cache_breakpoint on a content part, message, tool definition or tool call bound for Claude becomes a cache_control mark on the same block, on the Anthropic, Vertex AI, Bedrock invoke, Bedrock Converse and Azure AI routes. A client cache_control on the same block wins and the breakpoint ttl is not forwarded. Requests that carried a breakpoint are held to Claude's four-breakpoint cap: with drop_params the latest four survive in wire order, without it the request is a 400 before any provider call. prompt_cache_options is rejected as unsupported unless drop_params drops it. is_cache_control_set and the cache-control hook count a breakpoint like a cache_control. MAX_CACHE_CONTROL_BLOCKS moves to litellm.constants --- litellm/constants.py | 3 + .../anthropic_cache_control_hook.py | 8 +- .../prompt_templates/common_utils.py | 129 ++++++++++++ .../chat/prompt_cache_breakpoints.py | 189 ++++++++++++++++++ litellm/llms/anthropic/chat/transformation.py | 47 ++++- litellm/llms/anthropic/common_utils.py | 12 +- .../bedrock/chat/converse_transformation.py | 47 ++++- 7 files changed, 414 insertions(+), 21 deletions(-) create mode 100644 litellm/llms/anthropic/chat/prompt_cache_breakpoints.py diff --git a/litellm/constants.py b/litellm/constants.py index a7ea4a2975b..a544e21972e 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -586,6 +586,9 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = int(os.getenv("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)) # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 +# Anthropic (and Bedrock Claude) reject requests with more than 4 cache_control +# breakpoints: "A maximum of 4 blocks with cache_control may be provided." +MAX_CACHE_CONTROL_BLOCKS: Final = 4 REPLICATE_POLLING_DELAY_SECONDS: Final = float(os.getenv("REPLICATE_POLLING_DELAY_SECONDS", 0.5)) DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = int(os.getenv("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 4096)) DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096 diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index ed7593c4961..a7bf9e985f8 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -19,6 +19,7 @@ from urllib.parse import urlparse from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.constants import MAX_CACHE_CONTROL_BLOCKS from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.integrations.prompt_management_base import PromptManagementClient @@ -58,11 +59,6 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any - -# Anthropic (and Bedrock Claude) reject requests with more than 4 cache_control -# breakpoints: "A maximum of 4 blocks with cache_control may be provided." -MAX_CACHE_CONTROL_BLOCKS: Final = 4 - CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint") OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6) _GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?") @@ -228,7 +224,7 @@ def _as_object_list(value: object | None) -> list[object] | None: def _tool_call_carries_cache_breakpoint(tool_call: object, message: object) -> bool: - if _attribute_or_key(tool_call, "cache_control") is None: + if not _carries_cache_breakpoint(tool_call): return False return not tool_call_is_rebuilt_as_server_tool_use( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 3c1b3cf964b..31733cf61f3 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -7,6 +7,7 @@ import json import mimetypes import re from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from collections.abc import Set as AbstractSet from itertools import groupby, islice from os import PathLike from pathlib import Path @@ -19,6 +20,7 @@ from litellm.router_utils.batch_utils import InMemoryFile from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, + ChatCompletionCachedContent, ChatCompletionFileObject, ChatCompletionImageObject, ChatCompletionReasoningItem, @@ -1805,6 +1807,133 @@ def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: ) +def prompt_cache_breakpoint_as_cache_control(block: _MarkedT) -> _MarkedT: + """OpenAI's ``prompt_cache_breakpoint`` rewritten as Anthropic's ``cache_control`` on the same block. + + A ``cache_control`` the client already wrote wins. OpenAI's 30m TTL has no Anthropic equivalent and is dropped.""" + if block.get("prompt_cache_breakpoint") is None: + return block + unmarked: Final = {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"} + marked: Final = ( + unmarked + if unmarked.get("cache_control") is not None + else {**unmarked, "cache_control": ChatCompletionCachedContent(type="ephemeral")} + ) + return cast(_MarkedT, marked) # cast-ok: same block shape as the input with the marker key renamed + + +def _as_mapping(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, Mapping) else None # pyright: ignore[reportUnknownVariableType] # keys are read one by one + + +def _as_list(value: object) -> Sequence[object] | None: + return value if isinstance(value, list) else None # pyright: ignore[reportUnknownVariableType] # items are parsed one by one + + +def message_fields(message: AllMessageValues) -> Mapping[str, object]: + """The message as a plain mapping, for code that reads keys the role-specific TypedDicts do not all declare.""" + return message + + +def _without_key(block: Mapping[str, object], key: str) -> Mapping[str, object]: + return {name: value for name, value in block.items() if name != key} + + +def _block_with_prompt_cache_breakpoint_as_cache_control(block: object) -> object: + fields: Final = _as_mapping(block) + return prompt_cache_breakpoint_as_cache_control(fields) if fields is not None else block + + +def message_with_prompt_cache_breakpoints_as_cache_control(message: AllMessageValues) -> AllMessageValues: + """The message-level mark, every content part's mark and every tool call's mark renamed to ``cache_control``.""" + renamed: Final = prompt_cache_breakpoint_as_cache_control(message) + fields: Final = message_fields(renamed) + content: Final = _as_list(fields.get("content")) + tool_calls: Final = _as_list(fields.get("tool_calls")) + if content is None and tool_calls is None: + return renamed + return cast( # cast-ok: same message shape as the input with the marker key renamed on its blocks + AllMessageValues, + { + **fields, + **( + {"content": [_block_with_prompt_cache_breakpoint_as_cache_control(block) for block in content]} + if content is not None + else {} + ), + **( + {"tool_calls": [_block_with_prompt_cache_breakpoint_as_cache_control(call) for call in tool_calls]} + if tool_calls is not None + else {} + ), + }, + ) + + +def messages_with_prompt_cache_breakpoints_as_cache_control( + messages: Sequence[AllMessageValues], +) -> tuple[AllMessageValues, ...]: + return tuple(message_with_prompt_cache_breakpoints_as_cache_control(message) for message in messages) + + +def tool_with_prompt_cache_breakpoint_as_cache_control(tool: Mapping[str, object]) -> Mapping[str, object]: + """A tool definition's mark, written at the top level (Anthropic shape) or under ``function`` (OpenAI shape), + renamed to a top-level ``cache_control``, which every Claude transform reads.""" + function: Final = _as_mapping(tool.get("function")) + function_marked: Final = function is not None and function.get("prompt_cache_breakpoint") is not None + if tool.get("prompt_cache_breakpoint") is None and not function_marked: + return tool + unmarked_function: Final = _without_key(function, "prompt_cache_breakpoint") if function is not None else None + unmarked: Final = { + **_without_key(tool, "prompt_cache_breakpoint"), + **({"function": unmarked_function} if unmarked_function is not None else {}), + } + already_marked: Final = unmarked.get("cache_control") is not None or ( + unmarked_function is not None and unmarked_function.get("cache_control") is not None + ) + return unmarked if already_marked else {**unmarked, "cache_control": ChatCompletionCachedContent(type="ephemeral")} + + +def _block_without_cache_control(block: object) -> object: + fields: Final = _as_mapping(block) + return _without_key(fields, "cache_control") if fields is not None else block + + +def _blocks_without_cache_control(blocks: Sequence[object], dropped: AbstractSet[int]) -> Sequence[object]: + return [ + _block_without_cache_control(block) if position in dropped else block for position, block in enumerate(blocks) + ] + + +def message_without_cache_control_marks( + message: AllMessageValues, *, message_mark: bool, parts: AbstractSet[int], tool_calls: AbstractSet[int] +) -> AllMessageValues: + """``message`` minus its message-level ``cache_control`` (when ``message_mark``) and the marks on the content + parts and tool calls at the given positions.""" + fields: Final = message_fields(message) + kept: Final = _without_key(fields, "cache_control") if message_mark else fields + content: Final = _as_list(kept.get("content")) + calls: Final = _as_list(kept.get("tool_calls")) + return cast( # cast-ok: same message shape as the input minus the dropped marks + AllMessageValues, + { + **kept, + **({"content": _blocks_without_cache_control(content, parts)} if content is not None else {}), + **({"tool_calls": _blocks_without_cache_control(calls, tool_calls)} if calls is not None else {}), + }, + ) + + +def _tool_block_with_prompt_cache_breakpoint_as_cache_control(tool: object) -> object: + fields: Final = _as_mapping(tool) + return tool_with_prompt_cache_breakpoint_as_cache_control(fields) if fields is not None else tool + + +def tools_with_prompt_cache_breakpoints_as_cache_control(tools: object) -> tuple[object, ...]: + """Every tool definition in ``tools`` (anything but a list is treated as none) with its mark renamed.""" + return tuple(_tool_block_with_prompt_cache_breakpoint_as_cache_control(tool) for tool in _as_list(tools) or ()) + + LITELLM_INTERNAL_MESSAGE_FIELDS: Final = frozenset({"thinking_blocks", "reasoning_content", "provider_specific_fields"}) diff --git a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py new file mode 100644 index 00000000000..2bad4d8920f --- /dev/null +++ b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py @@ -0,0 +1,189 @@ +"""OpenAI's ``prompt_cache_breakpoint`` on a request bound for Claude. + +The marks become Anthropic's ``cache_control`` on the same blocks, and a request that carried at least one is held +to Claude's cap of four breakpoints, which a client writing the OpenAI dialect cannot see. ``prompt_cache_options`` +has no Claude equivalent and is rejected or dropped like any other unsupported param.""" + +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from itertools import chain +from typing import Final, Literal, TypeAlias, assert_never + +from litellm._logging import verbose_logger +from litellm.constants import MAX_CACHE_CONTROL_BLOCKS +from litellm.exceptions import BadRequestError, UnsupportedParamsError +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + message_without_cache_control_marks, + messages_with_prompt_cache_breakpoints_as_cache_control, + tools_with_prompt_cache_breakpoints_as_cache_control, +) +from litellm.types.llms.openai import AllMessageValues + +PROMPT_CACHE_BREAKPOINT_ON_TOOLS: Final = "_litellm_prompt_cache_breakpoint_on_tools" +_Site: TypeAlias = tuple[Literal["tool", "message", "part", "tool_call"], int, int] + + +@dataclass(frozen=True, slots=True) +class ClaudeCacheMarks: + messages: tuple[AllMessageValues, ...] + tools: tuple[object, ...] + + +@dataclass(frozen=True, slots=True) +class CacheBreakpointCapExceeded: + found: int + + +def _as_mapping(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, Mapping) else None # pyright: ignore[reportUnknownVariableType] # keys are read one by one + + +def _as_blocks(value: object) -> Sequence[object] | None: + return value if isinstance(value, list) else None # pyright: ignore[reportUnknownVariableType] # blocks are parsed one by one + + +def _carries(block: object, key: str) -> bool: + fields: Final = _as_mapping(block) + return fields is not None and fields.get(key) is not None + + +def _tool_carries(tool: object, key: str) -> bool: + fields: Final = _as_mapping(tool) + return fields is not None and (fields.get(key) is not None or _carries(fields.get("function"), key)) + + +def tools_carry_prompt_cache_breakpoint(tools: object) -> bool: + return any(_tool_carries(tool, "prompt_cache_breakpoint") for tool in _as_blocks(tools) or ()) + + +def _message_sites(index: int, message: object, key: str) -> Iterator[_Site]: + fields: Final = _as_mapping(message) + if fields is None: + return + if fields.get(key) is not None: + yield ("message", index, -1) + for position, part in enumerate(_as_blocks(fields.get("content")) or ()): + if _carries(part, key): + yield ("part", index, position) + for position, call in enumerate(_as_blocks(fields.get("tool_calls")) or ()): + if _carries(call, key): + yield ("tool_call", index, position) + + +def _sites(messages: Sequence[object], tools: Sequence[object], key: str) -> tuple[_Site, ...]: + """Every block carrying ``key`` in wire order: tool definitions first, then the messages as listed.""" + return ( + *(("tool", index, -1) for index, tool in enumerate(tools) if _tool_carries(tool, key)), + *chain.from_iterable(_message_sites(index, message, key) for index, message in enumerate(messages)), + ) + + +def _without_cache_control(block: Mapping[str, object]) -> Mapping[str, object]: + return {key: value for key, value in block.items() if key != "cache_control"} + + +def _tool_without_cache_control(tool: object) -> object: + fields: Final = _as_mapping(tool) + if fields is None: + return tool + function: Final = _as_mapping(fields.get("function")) + return { + **_without_cache_control(fields), + **({"function": _without_cache_control(function)} if function is not None else {}), + } + + +def _dropped_positions(dropped: frozenset[_Site], kind: str, index: int) -> frozenset[int]: + return frozenset( + position for site_kind, site_index, position in dropped if (site_kind, site_index) == (kind, index) + ) + + +def _message_without(index: int, message: AllMessageValues, dropped: frozenset[_Site]) -> AllMessageValues: + return message_without_cache_control_marks( + message, + message_mark=("message", index, -1) in dropped, + parts=_dropped_positions(dropped, "part", index), + tool_calls=_dropped_positions(dropped, "tool_call", index), + ) + + +def claude_cache_marks( + messages: Sequence[AllMessageValues], + tools: object, + *, + automatic_breakpoint: bool, + drop_params: bool, + tools_carried_breakpoint: bool = False, +) -> ClaudeCacheMarks | CacheBreakpointCapExceeded: + """The request's ``prompt_cache_breakpoint`` marks renamed to ``cache_control`` and, when it carried any, held to + Claude's cap: ``drop_params`` keeps the latest four in wire order, otherwise the excess is reported. A request + that carried none is forwarded as written so the provider's own answer reaches the client. + + ``automatic_breakpoint`` is the request-level ``cache_control`` (Anthropic's automatic caching), which takes one + slot of its own and is never dropped. ``tools_carried_breakpoint`` covers tools already mapped to Anthropic's + shape, where the mark was renamed before this census ran.""" + listed_tools: Final = _as_blocks(tools) or () + if not tools_carried_breakpoint and not _sites(messages, listed_tools, "prompt_cache_breakpoint"): + return ClaudeCacheMarks(tuple(messages), tuple(listed_tools)) + renamed_messages: Final = messages_with_prompt_cache_breakpoints_as_cache_control(messages) + renamed_tools: Final = tools_with_prompt_cache_breakpoints_as_cache_control(listed_tools) + sites: Final = _sites(renamed_messages, renamed_tools, "cache_control") + found: Final = len(sites) + int(automatic_breakpoint) + if found <= MAX_CACHE_CONTROL_BLOCKS: + return ClaudeCacheMarks(renamed_messages, renamed_tools) + if not drop_params: + return CacheBreakpointCapExceeded(found) + dropped: Final = frozenset(sites[: found - MAX_CACHE_CONTROL_BLOCKS]) + verbose_logger.debug( + "Dropping the earliest %d of %d cache breakpoints: Claude accepts at most %d.", + len(dropped), + found, + MAX_CACHE_CONTROL_BLOCKS, + ) + return ClaudeCacheMarks( + messages=tuple(_message_without(index, message, dropped) for index, message in enumerate(renamed_messages)), + tools=tuple( + _tool_without_cache_control(tool) if ("tool", index, -1) in dropped else tool + for index, tool in enumerate(renamed_tools) + ), + ) + + +def resolved_claude_cache_marks( + outcome: ClaudeCacheMarks | CacheBreakpointCapExceeded, model: str, llm_provider: str +) -> ClaudeCacheMarks: + match outcome: + case ClaudeCacheMarks(): + return outcome + case CacheBreakpointCapExceeded(found=found): + raise BadRequestError( + message=( + f"Claude accepts at most {MAX_CACHE_CONTROL_BLOCKS} cache breakpoints, found {found}. " + "Set drop_params=True (proxy: litellm_settings: drop_params: true) to keep the latest four." + ), + model=model, + llm_provider=llm_provider, + ) + case _: + assert_never(outcome) + + +def reject_prompt_cache_options(model: str, options: object, drop_params: bool) -> None: + """``prompt_cache_options`` has already been popped from the request; this decides whether that was allowed.""" + if options is None: + return + if drop_params: + verbose_logger.debug( + "Dropping prompt_cache_options=%r for %s: Claude caches through cache_control.", options, model + ) + return + raise UnsupportedParamsError( + message=( + f"{model} does not support prompt_cache_options={options!r}. Claude caches through cache_control " + "marks (or prompt_cache_breakpoint on the OpenAI dialect). To drop unsupported params, set " + "`litellm.drop_params = True`." + ), + model=model, + status_code=400, + ) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index c43e5f9b05b..e15714f1c40 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -25,6 +25,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.prompt_templates.common_utils import ( sanitize_input_schema_for_anthropic, + tools_with_prompt_cache_breakpoints_as_cache_control, ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( RemoteMedia, @@ -110,6 +111,13 @@ from ..common_utils import ( requires_native_compaction_beta, strip_advisor_blocks_from_messages, ) +from .prompt_cache_breakpoints import ( + PROMPT_CACHE_BREAKPOINT_ON_TOOLS, + claude_cache_marks, + reject_prompt_cache_options, + resolved_claude_cache_marks, + tools_carry_prompt_cache_breakpoint, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -1568,12 +1576,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if param == "max_tokens" or param == "max_completion_tokens": optional_params["max_tokens"] = value if isinstance(value, int) else max(1, int(round(value))) elif param == "tools": - anthropic_tools, mcp_servers = self._map_tools(value) + requested_tools = cast(object, value) # cast-ok: the loop yields untyped values + anthropic_tools, mcp_servers = self._map_tools( + list(tools_with_prompt_cache_breakpoints_as_cache_control(requested_tools)) + ) optional_params = self._add_tools_to_optional_params( optional_params=optional_params, tools=anthropic_tools ) if mcp_servers: optional_params["mcp_servers"] = mcp_servers + if tools_carry_prompt_cache_breakpoint(requested_tools): + optional_params[PROMPT_CACHE_BREAKPOINT_ON_TOOLS] = True # rebind-ok: out-param store elif param == "tool_choice" or param == "parallel_tool_calls": _tool_choice: AnthropicMessagesToolChoice | None = self._map_tool_choice( tool_choice=non_default_params.get("tool_choice"), @@ -2010,10 +2023,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "has no thinking_blocks. The model won't use extended thinking for this turn." ) + drop_params: Final = litellm.drop_params or litellm_params.get("drop_params") is True AnthropicConfig._maybe_drop_speed_param( model=model, optional_params=optional_params, - drop_params=litellm.drop_params or litellm_params.get("drop_params") is True, + drop_params=drop_params, custom_llm_provider=self.custom_llm_provider, ) @@ -2023,6 +2037,24 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): custom_llm_provider=self._resolved_provider, ) + body: Final = cast(dict[str, object], optional_params) # cast-ok: BaseConfig passes an untyped dict + reject_prompt_cache_options( + model=model, options=body.pop("prompt_cache_options", None), drop_params=drop_params + ) + cache_marks: Final = resolved_claude_cache_marks( + claude_cache_marks( + messages=messages, + tools=body.get("tools"), + automatic_breakpoint=body.get("cache_control") is not None, + drop_params=drop_params, + tools_carried_breakpoint=bool(body.pop(PROMPT_CACHE_BREAKPOINT_ON_TOOLS, False)), + ), + model=model, + llm_provider=self._resolved_provider, + ) + if "tools" in body: + optional_params["tools"] = list(cache_marks.tools) # rebind-ok: out-param store + # === Tool-name sanitization (single chokepoint) === # Anthropic enforces ^[a-zA-Z0-9_-]{1,128}$ on every tool name. We # sanitize *here* -- not in map_openai_params -- because: @@ -2046,15 +2078,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): _name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request( optional_params=optional_params, ) - if _name_forward_map: - messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map) + marked_messages: Final = ( + self._rewrite_tool_names_in_messages(list(cache_marks.messages), _name_forward_map) + if _name_forward_map + else list(cache_marks.messages) + ) if _name_reverse_map and isinstance(litellm_params, dict): litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map # Only the leading system run becomes the top-level system prompt. A later # system message stays in the conversation: hoisting it rewrites the cached # prefix and re-bills the whole history at cache-write pricing (#36559). - leading_system_run, later_messages = split_leading_system_run(messages) + leading_system_run, later_messages = split_leading_system_run(marked_messages) anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run)) # Handling anthropic API Prompt Caching if len(anthropic_system_message_list) > 0: @@ -2075,7 +2110,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): except Exception as e: raise AnthropicError( status_code=400, - message=f"{e}\nReceived Messages={messages}", + message=f"{e}\nReceived Messages={marked_messages}", ) # don't use verbose_logger.exception, if exception is raised self.update_headers_with_optional_anthropic_beta( diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 47a82aa097c..e54164c74c5 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -106,6 +106,7 @@ _INFERENCE_PROFILE_MINOR_RE: Final = re.compile(r":\d+$") _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$") _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)") _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" +CACHE_MARK_KEYS: Final = ("cache_control", "prompt_cache_breakpoint") _CLAUDE_CODE_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) @@ -461,18 +462,15 @@ class AnthropicModelInfo(BaseLLMModelInfo): _workload_identity_eligible: ClassVar[bool] = True def is_cache_control_set(self, messages: list[AllMessageValues]) -> bool: - """ - Return if {"cache_control": ..} in message content block - - Used to check if anthropic prompt caching headers need to be set. - """ + """Whether a message or one of its content blocks carries a cache mark, Anthropic's ``cache_control`` or + OpenAI's ``prompt_cache_breakpoint`` (which the chat transform renames on the way to Claude).""" for message in messages: - if message.get("cache_control", None) is not None: + if any(message.get(key) is not None for key in CACHE_MARK_KEYS): return True _message_content = message.get("content") if _message_content is not None and isinstance(_message_content, list): for content in _message_content: - if isinstance(content, dict) and "cache_control" in content: + if isinstance(content, dict) and any(key in content for key in CACHE_MARK_KEYS): return True return False diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 1cdcf0d6e4c..3221b902373 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -44,6 +44,11 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of, ) +from litellm.llms.anthropic.chat.prompt_cache_breakpoints import ( + claude_cache_marks, + reject_prompt_cache_options, + resolved_claude_cache_marks, +) from litellm.llms.anthropic.chat.transformation import ( DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING, DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, @@ -2012,6 +2017,36 @@ class AmazonConverseConfig(BaseConfig): return data + def _messages_with_claude_cache_marks( + self, + model: str, + messages: Sequence[AllMessageValues], + optional_params: dict[str, object], # mutable-ok: the request body is built from this out-param + litellm_params: Mapping[str, object], + ) -> tuple[AllMessageValues, ...]: + """OpenAI ``prompt_cache_breakpoint`` marks as ``cache_control`` so the factory emits ``cachePoint`` blocks, + held to the provider's cap, with ``prompt_cache_options`` rejected or dropped. Models that never emit + ``cachePoint`` blocks are left as written.""" + if not bedrock_model_accepts_cache_points(model): + return tuple(messages) + drop_params: Final = litellm.drop_params or litellm_params.get("drop_params") is True + reject_prompt_cache_options( + model=model, options=optional_params.pop("prompt_cache_options", None), drop_params=drop_params + ) + cache_marks: Final = resolved_claude_cache_marks( + claude_cache_marks( + messages=messages, + tools=optional_params.get("tools"), + automatic_breakpoint=optional_params.get("cache_control") is not None, + drop_params=drop_params, + ), + model=model, + llm_provider=self.custom_llm_provider or "bedrock", + ) + if "tools" in optional_params: + optional_params["tools"] = list(cache_marks.tools) # rebind-ok: out-param store + return cache_marks.messages + async def _async_transform_request( self, model: str, @@ -2020,7 +2055,11 @@ class AmazonConverseConfig(BaseConfig): litellm_params: dict, headers: dict | None = None, ) -> RequestObject: - messages, system_content_blocks = self._transform_system_message(messages, model=model) + body: Final = cast(dict[str, object], optional_params) # cast-ok: BaseConfig passes an untyped dict + params: Final = cast(Mapping[str, object], litellm_params) # cast-ok: BaseConfig passes an untyped dict + messages, system_content_blocks = self._transform_system_message( + list(self._messages_with_claude_cache_marks(model, messages, body, params)), model=model + ) # Convert last user message to guarded_text if guardrailConfig is present messages = self._convert_consecutive_user_messages_to_guarded_text( @@ -2088,7 +2127,11 @@ class AmazonConverseConfig(BaseConfig): litellm_params: dict, headers: dict | None = None, ) -> RequestObject: - messages, system_content_blocks = self._transform_system_message(messages, model=model) + body: Final = cast(dict[str, object], optional_params) # cast-ok: BaseConfig passes an untyped dict + params: Final = cast(Mapping[str, object], litellm_params) # cast-ok: BaseConfig passes an untyped dict + messages, system_content_blocks = self._transform_system_message( + list(self._messages_with_claude_cache_marks(model, messages, body, params)), model=model + ) # Convert last user message to guarded_text if guardrailConfig is present messages = self._convert_consecutive_user_messages_to_guarded_text(