From fedb3f43cf982fefa048129e6723591c30cae80b Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 14 Oct 2026 07:27:57 -0700 Subject: [PATCH] fix(anthropic): hold every prompt_cache_options source and Converse cache point model to the Claude rules prompt_cache_options sent through extra_body skipped the drop_params rule on the routes that merge extra_body into the Claude body (Bedrock Invoke, Bedrock Claude Platform, Azure AI, Databricks Claude), so Bedrock rejected it with its own 400. AnthropicConfig.transform_extra_body now applies the same rule, and Databricks only applies it to Claude models. Converse now gates the rename, the cap and the prompt_cache_options rule on whether the model takes cachePoint blocks, so an application inference profile ARN gets cache points, and a model without cachePoint support gets the request as written. prompt_cache_options is now a refusal value from claude_cache_marks next to the cap refusal, mapped to the public exception in one place. assert_never comes from typing_extensions so the module imports on Python 3.10. is_cache_control_set is back to main since no header depends on it, and unmarked tools reach _map_tools as on main. --- .../chat/prompt_cache_breakpoints.py | 107 +++++++++++++----- litellm/llms/anthropic/chat/transformation.py | 28 +++-- litellm/llms/anthropic/common_utils.py | 12 +- .../bedrock/chat/converse_transformation.py | 27 ++--- .../llms/databricks/chat/transformation.py | 27 +++-- ...laude_chat_prompt_cache_breakpoint_wire.py | 52 ++++++++- .../test_anthropic_chat_transformation.py | 12 +- .../chat/test_prompt_cache_breakpoints.py | 66 ++++++++++- .../anthropic/test_anthropic_common_utils.py | 15 --- ...ations_anthropic_claude3_transformation.py | 43 +++++++ .../chat/test_converse_transformation.py | 76 ++++++++----- .../bedrock/test_claude_platform_provider.py | 98 ++++++++++++++++ .../test_databricks_chat_transformation.py | 15 ++- 13 files changed, 457 insertions(+), 121 deletions(-) diff --git a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py index b3bdd96f249..0a6f30de610 100644 --- a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py +++ b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py @@ -7,11 +7,15 @@ has no Claude equivalent and is rejected or dropped like any other unsupported p from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from itertools import chain -from typing import Final, Literal, TypeAlias, assert_never +from typing import Final, Literal, NoReturn, TypeAlias +from typing_extensions import assert_never + +import litellm from litellm._logging import verbose_logger from litellm.constants import MAX_CACHE_CONTROL_BLOCKS from litellm.exceptions import BadRequestError, UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( message_fields, message_without_cache_control_marks, @@ -37,6 +41,19 @@ class CacheBreakpointCapExceeded: found: int +@dataclass(frozen=True, slots=True) +class PromptCacheOptionsUnsupported: + options: object + + +ClaudeCacheRefusal: TypeAlias = CacheBreakpointCapExceeded | PromptCacheOptionsUnsupported + + +def drop_params_enabled(litellm_params: Mapping[str, object]) -> bool: + """``drop_params`` from the global setting, or from the request or deployment (both arrive in ``litellm_params``).""" + return bool(litellm.drop_params) or normalize_drop_params(litellm_params.get("drop_params")) is True + + def _as_mapping(value: object) -> Mapping[str, object] | None: return value if isinstance(value, Mapping) else None # pyright: ignore[reportUnknownVariableType] # keys are read one by one @@ -150,23 +167,37 @@ def _message_without(index: int, message: AllMessageValues, dropped: frozenset[_ ) +def _prompt_cache_options_refusal(options: object, drop_params: bool) -> PromptCacheOptionsUnsupported | None: + if options is None: + return None + if not drop_params: + return PromptCacheOptionsUnsupported(options) + verbose_logger.debug("Dropping prompt_cache_options=%r: Claude caches through cache_control.", options) + return None + + def claude_cache_marks( messages: Sequence[AllMessageValues], tools: object, *, automatic_breakpoint: bool, drop_params: bool, + prompt_cache_options: object = None, tools_carried_breakpoint: bool = False, wire: ClaudeWire = "anthropic", -) -> ClaudeCacheMarks | CacheBreakpointCapExceeded: +) -> ClaudeCacheMarks | ClaudeCacheRefusal: """The request's ``prompt_cache_breakpoint`` marks renamed to ``cache_control`` and, when it carried any, held to Claude's cap: ``drop_params`` keeps the latest four in wire order, otherwise the excess is reported. A request that carried none is forwarded as written so the provider's own answer reaches the client. - ``automatic_breakpoint`` is the request-level ``cache_control`` (Anthropic's automatic caching), which takes one - slot of its own and is never dropped. ``tools_carried_breakpoint`` covers tools already mapped to Anthropic's - shape, where the mark was renamed before this census ran. ``wire`` is the body the marks are rendered into, so - the count matches the breakpoints the provider receives.""" + ``prompt_cache_options`` has no Claude equivalent: the caller has already taken it out of the request, and it is + reported unless ``drop_params``. ``automatic_breakpoint`` is the request-level ``cache_control`` (Anthropic's + automatic caching), which takes one slot of its own and is never dropped. ``tools_carried_breakpoint`` covers + tools already mapped to Anthropic's shape, where the mark was renamed before this census ran. ``wire`` is the + body the marks are rendered into, so the count matches the breakpoints the provider receives.""" + options_refusal: Final = _prompt_cache_options_refusal(prompt_cache_options, drop_params) + if options_refusal is not None: + return options_refusal listed_tools: Final = _as_blocks(tools) or () if not tools_carried_breakpoint and not _sites(messages, listed_tools, "prompt_cache_breakpoint", wire): return ClaudeCacheMarks(tuple(messages), tuple(listed_tools)) @@ -194,12 +225,20 @@ def claude_cache_marks( ) -def resolved_claude_cache_marks( - outcome: ClaudeCacheMarks | CacheBreakpointCapExceeded, model: str, llm_provider: str -) -> ClaudeCacheMarks: - match outcome: - case ClaudeCacheMarks(): - return outcome +def claude_extra_body( + extra_body: Mapping[str, object], drop_params: bool +) -> Mapping[str, object] | PromptCacheOptionsUnsupported: + """``extra_body`` merged into a Claude body, held to the same ``prompt_cache_options`` rule as the request.""" + refusal: Final = _prompt_cache_options_refusal(extra_body.get("prompt_cache_options"), drop_params) + if refusal is not None: + return refusal + if "prompt_cache_options" not in extra_body: + return extra_body + return {key: value for key, value in extra_body.items() if key != "prompt_cache_options"} + + +def raise_claude_cache_refusal(refusal: ClaudeCacheRefusal, model: str, llm_provider: str) -> NoReturn: + match refusal: case CacheBreakpointCapExceeded(found=found): raise BadRequestError( message=( @@ -209,25 +248,31 @@ def resolved_claude_cache_marks( model=model, llm_provider=llm_provider, ) + case PromptCacheOptionsUnsupported(options=options): + raise UnsupportedParamsError( + message=( + f"{model} does not support prompt_cache_options={options!r}. It caches through cache_control " + "marks (or prompt_cache_breakpoint on the OpenAI dialect). To drop unsupported params, set " + "`litellm.drop_params = True`." + ), + model=model, + status_code=400, + ) case _: - assert_never(outcome) + assert_never(refusal) -def reject_prompt_cache_options(model: str, options: object, drop_params: bool) -> None: - """``prompt_cache_options`` has already been popped from the request; this decides whether that was allowed.""" - if options is None: - return - if drop_params: - verbose_logger.debug( - "Dropping prompt_cache_options=%r for %s: Claude caches through cache_control.", options, model - ) - return - raise UnsupportedParamsError( - message=( - f"{model} does not support prompt_cache_options={options!r}. Claude caches through cache_control " - "marks (or prompt_cache_breakpoint on the OpenAI dialect). To drop unsupported params, set " - "`litellm.drop_params = True`." - ), - model=model, - status_code=400, - ) +def resolved_claude_cache_marks( + outcome: ClaudeCacheMarks | ClaudeCacheRefusal, model: str, llm_provider: str +) -> ClaudeCacheMarks: + if isinstance(outcome, ClaudeCacheMarks): + return outcome + raise_claude_cache_refusal(outcome, model=model, llm_provider=llm_provider) + + +def resolved_claude_extra_body( + outcome: Mapping[str, object] | PromptCacheOptionsUnsupported, model: str, llm_provider: str +) -> Mapping[str, object]: + if isinstance(outcome, PromptCacheOptionsUnsupported): + raise_claude_cache_refusal(outcome, model=model, llm_provider=llm_provider) + return outcome diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index e15714f1c40..db9c388f0ad 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -114,8 +114,10 @@ from ..common_utils import ( from .prompt_cache_breakpoints import ( PROMPT_CACHE_BREAKPOINT_ON_TOOLS, claude_cache_marks, - reject_prompt_cache_options, + claude_extra_body, + drop_params_enabled, resolved_claude_cache_marks, + resolved_claude_extra_body, tools_carry_prompt_cache_breakpoint, ) @@ -1579,6 +1581,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): requested_tools = cast(object, value) # cast-ok: the loop yields untyped values anthropic_tools, mcp_servers = self._map_tools( list(tools_with_prompt_cache_breakpoints_as_cache_control(requested_tools)) + if tools_carry_prompt_cache_breakpoint(requested_tools) + else value ) optional_params = self._add_tools_to_optional_params( optional_params=optional_params, tools=anthropic_tools @@ -1983,6 +1987,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): headers=headers, ) + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: Mapping[str, object], + ) -> Mapping[str, object]: + return resolved_claude_extra_body( + claude_extra_body(extra_body, drop_params=drop_params_enabled(litellm_params)), + model=model, + llm_provider=self._resolved_provider, + ) + def transform_request( self, model: str, @@ -2023,11 +2040,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "has no thinking_blocks. The model won't use extended thinking for this turn." ) - drop_params: Final = litellm.drop_params or litellm_params.get("drop_params") is True AnthropicConfig._maybe_drop_speed_param( model=model, optional_params=optional_params, - drop_params=drop_params, + drop_params=litellm.drop_params or litellm_params.get("drop_params") is True, custom_llm_provider=self.custom_llm_provider, ) @@ -2038,15 +2054,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) body: Final = cast(dict[str, object], optional_params) # cast-ok: BaseConfig passes an untyped dict - reject_prompt_cache_options( - model=model, options=body.pop("prompt_cache_options", None), drop_params=drop_params - ) cache_marks: Final = resolved_claude_cache_marks( claude_cache_marks( messages=messages, tools=body.get("tools"), automatic_breakpoint=body.get("cache_control") is not None, - drop_params=drop_params, + drop_params=drop_params_enabled(litellm_params), + prompt_cache_options=body.pop("prompt_cache_options", None), tools_carried_breakpoint=bool(body.pop(PROMPT_CACHE_BREAKPOINT_ON_TOOLS, False)), ), model=model, diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index e54164c74c5..47a82aa097c 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -106,7 +106,6 @@ _INFERENCE_PROFILE_MINOR_RE: Final = re.compile(r":\d+$") _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$") _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)") _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" -CACHE_MARK_KEYS: Final = ("cache_control", "prompt_cache_breakpoint") _CLAUDE_CODE_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) @@ -462,15 +461,18 @@ class AnthropicModelInfo(BaseLLMModelInfo): _workload_identity_eligible: ClassVar[bool] = True def is_cache_control_set(self, messages: list[AllMessageValues]) -> bool: - """Whether a message or one of its content blocks carries a cache mark, Anthropic's ``cache_control`` or - OpenAI's ``prompt_cache_breakpoint`` (which the chat transform renames on the way to Claude).""" + """ + Return if {"cache_control": ..} in message content block + + Used to check if anthropic prompt caching headers need to be set. + """ for message in messages: - if any(message.get(key) is not None for key in CACHE_MARK_KEYS): + if message.get("cache_control", None) is not None: return True _message_content = message.get("content") if _message_content is not None and isinstance(_message_content, list): for content in _message_content: - if isinstance(content, dict) and any(key in content for key in CACHE_MARK_KEYS): + if isinstance(content, dict) and "cache_control" in content: return True return False diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index e1a38d0ebe9..005ca5e71cc 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -46,7 +46,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ) from litellm.llms.anthropic.chat.prompt_cache_breakpoints import ( claude_cache_marks, - reject_prompt_cache_options, + drop_params_enabled, resolved_claude_cache_marks, ) from litellm.llms.anthropic.chat.transformation import ( @@ -57,7 +57,6 @@ from litellm.llms.anthropic.chat.transformation import ( ) from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException -from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_images from litellm.llms.bedrock.common_utils import bedrock_model_supports_regex_lookaround from litellm.llms.bedrock.request_metadata import ( @@ -2018,31 +2017,25 @@ class AmazonConverseConfig(BaseConfig): return data - def _messages_with_claude_cache_marks( + def _messages_with_prompt_cache_breakpoints( self, model: str, messages: Sequence[AllMessageValues], optional_params: dict[str, object], # mutable-ok: the request body is built from this out-param litellm_params: Mapping[str, object], ) -> tuple[AllMessageValues, ...]: - """On Claude, OpenAI ``prompt_cache_breakpoint`` marks as ``cache_control`` so the factory emits - ``cachePoint`` blocks, held to Claude's cap, with ``prompt_cache_options`` rejected or dropped. A Claude model - that never emits ``cachePoint`` blocks keeps its marks as written. Other models are left as they are.""" - if BaseAWSLLM.get_bedrock_invoke_provider(model) != "anthropic": - return tuple(messages) - emits_cache_points: Final = bedrock_model_accepts_cache_points(model) - drop_params: Final = litellm.drop_params or litellm_params.get("drop_params") is True - reject_prompt_cache_options( - model=model, options=optional_params.pop("prompt_cache_options", None), drop_params=drop_params - ) - if not emits_cache_points: + """On a model that takes ``cachePoint`` blocks, OpenAI ``prompt_cache_breakpoint`` marks as ``cache_control`` + so the factory emits them, held to the cap of four, with ``prompt_cache_options`` rejected or dropped. Any + other model gets the request as written.""" + if not bedrock_model_accepts_cache_points(model): return tuple(messages) cache_marks: Final = resolved_claude_cache_marks( claude_cache_marks( messages=messages, tools=optional_params.get("tools"), automatic_breakpoint=optional_params.get("cache_control") is not None, - drop_params=drop_params, + drop_params=drop_params_enabled(litellm_params), + prompt_cache_options=optional_params.pop("prompt_cache_options", None), wire="bedrock_converse", ), model=model, @@ -2063,7 +2056,7 @@ class AmazonConverseConfig(BaseConfig): body: Final = cast(dict[str, object], optional_params) # cast-ok: BaseConfig passes an untyped dict params: Final = cast(Mapping[str, object], litellm_params) # cast-ok: BaseConfig passes an untyped dict messages, system_content_blocks = self._transform_system_message( - list(self._messages_with_claude_cache_marks(model, messages, body, params)), model=model + list(self._messages_with_prompt_cache_breakpoints(model, messages, body, params)), model=model ) # Convert last user message to guarded_text if guardrailConfig is present @@ -2135,7 +2128,7 @@ class AmazonConverseConfig(BaseConfig): body: Final = cast(dict[str, object], optional_params) # cast-ok: BaseConfig passes an untyped dict params: Final = cast(Mapping[str, object], litellm_params) # cast-ok: BaseConfig passes an untyped dict messages, system_content_blocks = self._transform_system_message( - list(self._messages_with_claude_cache_marks(model, messages, body, params)), model=model + list(self._messages_with_prompt_cache_breakpoints(model, messages, body, params)), model=model ) # Convert last user message to guarded_text if guardrailConfig is present diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 9e64826d928..b59ebb160fc 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -9,7 +9,6 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload import httpx from pydantic import BaseModel, TypeAdapter -import litellm from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( handle_invalid_parallel_tool_calls, @@ -59,8 +58,10 @@ from litellm.types.utils import ( from ...anthropic.chat.prompt_cache_breakpoints import ( PROMPT_CACHE_BREAKPOINT_ON_TOOLS, claude_cache_marks, - reject_prompt_cache_options, + claude_extra_body, + drop_params_enabled, resolved_claude_cache_marks, + resolved_claude_extra_body, tools_carry_prompt_cache_breakpoint, ) from ...anthropic.chat.transformation import ( @@ -468,6 +469,21 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): ), ) + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: Mapping[str, object], + ) -> Mapping[str, object]: + if "claude" not in model: + return extra_body + return resolved_claude_extra_body( + claude_extra_body(extra_body, drop_params=drop_params_enabled(litellm_params)), + model=model, + llm_provider="databricks", + ) + def _messages_with_claude_cache_marks( self, model: str, @@ -479,16 +495,13 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): """OpenAI ``prompt_cache_breakpoint`` marks as the ``cache_control`` Databricks forwards to Claude, held to Claude's cap, with ``prompt_cache_options`` rejected or dropped. Tool definitions reach Databricks without any mark, so only the message marks count toward the cap.""" - drop_params: Final = litellm.drop_params or litellm_params.get("drop_params") is True - reject_prompt_cache_options( - model=model, options=optional_params.pop("prompt_cache_options", None), drop_params=drop_params - ) cache_marks: Final = resolved_claude_cache_marks( claude_cache_marks( messages=messages, tools=(), automatic_breakpoint=optional_params.get("cache_control") is not None, - drop_params=drop_params, + drop_params=drop_params_enabled(litellm_params), + prompt_cache_options=optional_params.pop("prompt_cache_options", None), tools_carried_breakpoint=tools_carried_breakpoint, ), model=model, diff --git a/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_wire.py b/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_wire.py index 0b5ee90083f..09be8e8caa0 100644 --- a/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_wire.py +++ b/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_wire.py @@ -39,10 +39,20 @@ from integration.providers._cache_control_marks_support import ( ) from pydantic import BaseModel, ConfigDict, JsonValue -Provider = Literal["anthropic", "bedrock_converse", "bedrock_invoke", "vertex_ai", "azure_ai"] +Provider = Literal[ + "anthropic", "bedrock_converse", "bedrock_invoke", "bedrock_claude_platform", "vertex_ai", "azure_ai" +] Site = Literal["tool", "system", "user_part", "tool_call", "tool_result_part", "string_message"] -_PROVIDERS: Final[tuple[Provider, ...]] = ("anthropic", "bedrock_converse", "bedrock_invoke", "vertex_ai", "azure_ai") +_PROVIDERS: Final[tuple[Provider, ...]] = ( + "anthropic", + "bedrock_converse", + "bedrock_invoke", + "bedrock_claude_platform", + "vertex_ai", + "azure_ai", +) +_EXTRA_BODY_PROVIDERS: Final[tuple[Provider, ...]] = ("bedrock_invoke", "bedrock_claude_platform", "azure_ai") _SITES: Final[tuple[Site, ...]] = ("tool", "system", "user_part", "tool_call", "tool_result_part", "string_message") _FIVE_SITES: Final[tuple[Site, ...]] = ("tool", "system", "user_part", "tool_call", "string_message") _EXPLICIT: Final[dict[str, JsonValue]] = {"mode": "explicit"} @@ -59,6 +69,7 @@ _TARGETS: Final[Mapping[Provider, str]] = MappingProxyType( "anthropic": "/v1/messages", "bedrock_converse": f"/model/{BEDROCK_MODEL}/converse", "bedrock_invoke": f"/model/{BEDROCK_MODEL}/invoke", + "bedrock_claude_platform": "/v1/messages", "vertex_ai": ( f"/v1/projects/{_VERTEX_PROJECT}/locations/{_VERTEX_LOCATION}" f"/publishers/anthropic/models/{ANTHROPIC_MODEL}:rawPredict" @@ -138,6 +149,14 @@ def _litellm_params(gateway: Gateway, wire: Wire, provider: Provider) -> dict[st "api_base": wire.url, "aws_bedrock_runtime_endpoint": wire.url, } + case "bedrock_claude_platform": + return { + "model": f"bedrock/claude_platform/{ANTHROPIC_MODEL}", + "api_base": wire.url, + "api_key": PROVIDER_KEY, + "workspace_id": "wrkspc_scripted", + "aws_region_name": "us-east-1", + } case "vertex_ai": return { "model": f"vertex_ai/{ANTHROPIC_MODEL}", @@ -351,6 +370,35 @@ def test_prompt_cache_options_are_a_400_before_any_provider_call_without_drop_pa assert received == (), [request.target for request in received] +@pytest.mark.parametrize("provider", _EXTRA_BODY_PROVIDERS) +def test_prompt_cache_options_in_extra_body_are_a_400_before_any_provider_call_without_drop_params( + gateway: Gateway, provider: Provider +) -> None: + marker: Final = new_marker() + extra_body: Final[dict[str, JsonValue]] = {"prompt_cache_options": _EXPLICIT} + with wire_server(lambda request: _respond(provider, request)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(gateway, scenario, wire, provider) + status, _, text = post_chat(gateway, _body(model, marker, _marked(("user_part",)), extra_body=extra_body)) + received: Final = wire.drain() + assert status == 400, text + assert "prompt_cache_options" in text, text + assert "drop_params" in text, text + assert received == (), [request.target for request in received] + + +@pytest.mark.parametrize("provider", _EXTRA_BODY_PROVIDERS) +def test_prompt_cache_options_in_extra_body_are_dropped_under_drop_params(gateway: Gateway, provider: Provider) -> None: + marker: Final = new_marker() + extra_body: Final[dict[str, JsonValue]] = {"prompt_cache_options": _EXPLICIT} + with wire_server(lambda request: _respond(provider, request)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(gateway, scenario, wire, provider, drop_params=True) + status, _, text = post_chat(gateway, _body(model, marker, _marked(("user_part",)), extra_body=extra_body)) + assert status == 200, text + request: Final = _only_request(wire) + assert b"prompt_cache_options" not in request.body, request.body + assert _labels(provider, request) == [ASK_LABEL] + + @pytest.mark.parametrize("provider", _PROVIDERS) def test_prompt_cache_options_are_dropped_under_drop_params(gateway: Gateway, provider: Provider) -> None: marker: Final = new_marker() diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 9bbb4628f6a..19efaa9bf4d 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,5 +1,6 @@ import copy import json +from collections.abc import Callable from typing import Final from unittest.mock import MagicMock, patch @@ -9085,9 +9086,16 @@ async def test_a_message_level_breakpoint_on_list_content_does_not_count_toward_ ] -def test_map_openai_params_maps_a_tuple_of_tools() -> None: +@pytest.mark.parametrize( + "tools", + [ + pytest.param(lambda tool: (tool,), id="tuple"), + pytest.param(lambda tool: iter([tool]), id="iterator"), + ], +) +def test_map_openai_params_maps_tools_given_as_any_iterable(tools: Callable[[object], object]) -> None: optional_params: Final = AnthropicConfig().map_openai_params( - non_default_params={"tools": ({"type": "function", "function": _WEATHER_FUNCTION},)}, + non_default_params={"tools": tools({"type": "function", "function": _WEATHER_FUNCTION})}, optional_params={}, model=PROMPT_CACHING_MODEL, drop_params=False, diff --git a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py index 3b3b793a521..257fa92743b 100644 --- a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py +++ b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py @@ -6,9 +6,12 @@ import litellm from litellm.llms.anthropic.chat.prompt_cache_breakpoints import ( CacheBreakpointCapExceeded, ClaudeCacheMarks, + PromptCacheOptionsUnsupported, claude_cache_marks, - reject_prompt_cache_options, + claude_extra_body, + drop_params_enabled, resolved_claude_cache_marks, + resolved_claude_extra_body, tools_carry_prompt_cache_breakpoint, ) @@ -327,15 +330,68 @@ def test_marks_within_the_cap_resolve_to_themselves() -> None: assert resolved_claude_cache_marks(marks, model=MODEL, llm_provider="anthropic") is marks -def test_prompt_cache_options_are_rejected_unless_drop_params() -> None: +def test_prompt_cache_options_are_reported_before_the_marks_unless_drop_params() -> None: + messages, tools = _six_breakpoints() + + assert claude_cache_marks( + messages, tools, automatic_breakpoint=False, drop_params=False, prompt_cache_options=EXPLICIT + ) == PromptCacheOptionsUnsupported(EXPLICIT) + kept: Final = claude_cache_marks( + messages, tools, automatic_breakpoint=False, drop_params=True, prompt_cache_options=EXPLICIT + ) + assert isinstance(kept, ClaudeCacheMarks) + assert claude_cache_marks(messages, tools, automatic_breakpoint=False, drop_params=True) == kept + + +def test_the_prompt_cache_options_error_is_an_unsupported_param_naming_drop_params() -> None: with pytest.raises(litellm.UnsupportedParamsError) as caught: - reject_prompt_cache_options(model=MODEL, options={"mode": "explicit"}, drop_params=False) + resolved_claude_cache_marks(PromptCacheOptionsUnsupported(EXPLICIT), model=MODEL, llm_provider="anthropic") assert "prompt_cache_options" in str(caught.value) assert "drop_params" in str(caught.value) assert caught.value.status_code == 400 - assert reject_prompt_cache_options(model=MODEL, options={"mode": "explicit"}, drop_params=True) is None - assert reject_prompt_cache_options(model=MODEL, options=None, drop_params=False) is None + + +@pytest.mark.parametrize( + ("extra_body", "drop_params", "expected"), + [ + ({"metadata": {"a": 1}}, False, {"metadata": {"a": 1}}), + ({"metadata": {"a": 1}, "prompt_cache_options": EXPLICIT}, True, {"metadata": {"a": 1}}), + ({"metadata": {"a": 1}, "prompt_cache_options": EXPLICIT}, False, PromptCacheOptionsUnsupported(EXPLICIT)), + ], + ids=["no-options", "dropped", "rejected"], +) +def test_extra_body_follows_the_prompt_cache_options_rule( + extra_body: dict[str, object], drop_params: bool, expected: object +) -> None: + assert claude_extra_body(extra_body, drop_params=drop_params) == expected + + +def test_a_rejected_extra_body_raises_and_an_accepted_one_resolves_to_itself() -> None: + body: Final = {"metadata": {"a": 1}} + + assert resolved_claude_extra_body(body, model=MODEL, llm_provider="anthropic") is body + with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"): + resolved_claude_extra_body(PromptCacheOptionsUnsupported(EXPLICIT), model=MODEL, llm_provider="anthropic") + + +@pytest.mark.parametrize( + ("global_drop", "litellm_params", "expected"), + [ + (False, {}, False), + (True, {}, True), + (False, {"drop_params": True}, True), + (False, {"drop_params": "true"}, True), + (False, {"drop_params": False}, False), + ], + ids=["off", "global", "request", "request-string", "request-false"], +) +def test_drop_params_enabled_reads_the_global_and_the_request( + global_drop: bool, litellm_params: dict[str, object], expected: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", global_drop) + + assert drop_params_enabled(litellm_params) is expected @pytest.mark.parametrize( diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index f065921f449..40307c9e09e 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -4267,18 +4267,3 @@ def test_anthropic_error_frame_exception_classes_an_overloaded_frame_as_internal error: Final = anthropic_error_frame_exception("overloaded_error", "Overloaded", 503, "claude-sonnet-4-5") assert type(error) is litellm.InternalServerError - - -@pytest.mark.parametrize( - "messages", - [ - [{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": {"mode": "explicit"}}]}], - [{"role": "user", "content": "hi", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}}], - [{"role": "system", "content": [{"type": "text", "text": "rules", "prompt_cache_breakpoint": {"mode": "explicit"}}]}], - ], - ids=["content-part", "message-level-on-string", "system-part"], -) -def test_is_cache_control_set_counts_a_prompt_cache_breakpoint_like_a_cache_control(messages: list) -> None: - from litellm.llms.anthropic.common_utils import AnthropicModelInfo - - assert AnthropicModelInfo().is_cache_control_set(messages) is True diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index b2399b9a52b..ddf811206f9 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -873,6 +873,49 @@ async def test_bedrock_invoke_claude_async_completion_inlines_document_url_sourc } in captured["body"]["messages"][0]["content"] +async def test_bedrock_invoke_claude_holds_extra_body_prompt_cache_options_to_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + bodies: Final[list[dict[str, object]]] = [] # mutable-ok: the transport records each request + + def handle(request: httpx.Request) -> httpx.Response: + bodies.append(json.loads(request.content)) + return httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "us.anthropic.claude-sonnet-5", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + client: Final = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(handle)) + request: Final = { + "model": "bedrock/invoke/us.anthropic.claude-sonnet-5", + "messages": [{"role": "user", "content": "hi"}], + "extra_body": {"prompt_cache_options": {"mode": "explicit"}, "top_k": 3}, + "aws_access_key_id": "AKIAEXAMPLE", + "aws_secret_access_key": "fake-secret", + "aws_region_name": "us-east-1", + "client": client, + } + + with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"): + await litellm.acompletion(**request) + assert bodies == [] + + await litellm.acompletion(**request, drop_params=True) + + assert "prompt_cache_options" not in bodies[-1] + assert bodies[-1]["top_k"] == 3 + + @pytest.mark.parametrize( "model, expected_betas", [ diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 9927130935e..13ed9cd5c1a 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -10293,19 +10293,12 @@ async def test_six_breakpoints_are_a_bad_request_without_drop_params( await _breakpoint_request(sync_mode, {"tools": [{**_LOOKUP_TOOL, "prompt_cache_breakpoint": _EXPLICIT}]}, {}) -@pytest.mark.parametrize("emits_cache_points", [True, False], ids=["cache-points", "no-cache-points"]) @pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"]) @pytest.mark.asyncio async def test_prompt_cache_options_are_rejected_unless_drop_params( - sync_mode: bool, emits_cache_points: bool, monkeypatch: pytest.MonkeyPatch + sync_mode: bool, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(litellm, "drop_params", False) - if not emits_cache_points: - monkeypatch.setitem( - litellm.model_cost, - _BREAKPOINT_MODEL, - {"litellm_provider": "bedrock", "mode": "chat", "supports_prompt_cache_breakpoint": False}, - ) with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"): await _breakpoint_request(sync_mode, {"prompt_cache_options": {"mode": "explicit"}}, {}) @@ -10319,6 +10312,54 @@ async def test_prompt_cache_options_are_rejected_unless_drop_params( assert "prompt_cache_options" not in json.dumps(dropped_globally) +_INFERENCE_PROFILE_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123" + + +@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"]) +@pytest.mark.asyncio +async def test_an_inference_profile_arn_gets_cache_points_and_the_prompt_cache_options_rule( + sync_mode: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = [ + {"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": _EXPLICIT}]} + ] + + result: Final = await _breakpoint_converse_request(sync_mode, _INFERENCE_PROFILE_ARN, messages) + + assert result["messages"] == [{"role": "user", "content": [{"text": "hi"}, _CACHE_POINT]}] + with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"): + await _breakpoint_converse_request( + sync_mode, _INFERENCE_PROFILE_ARN, messages, optional_params={"prompt_cache_options": _EXPLICIT} + ) + + +@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"]) +@pytest.mark.asyncio +async def test_a_model_without_cache_points_gets_the_request_as_written( + sync_mode: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setitem( + litellm.model_cost, + _BREAKPOINT_MODEL, + {"litellm_provider": "bedrock", "mode": "chat", "supports_prompt_cache_breakpoint": False}, + ) + marked: Final = [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": _EXPLICIT} for index in range(5) + ] + + result: Final = await _breakpoint_converse_request( + sync_mode, + _BREAKPOINT_MODEL, + [{"role": "user", "content": marked}], + optional_params={"prompt_cache_options": {"mode": "implicit"}}, + ) + + assert result["messages"] == [{"role": "user", "content": [{"text": f"part {index}"} for index in range(5)]}] + assert result["additionalModelRequestFields"] == {"prompt_cache_options": {"mode": "implicit"}} + + def test_the_automatic_cache_control_takes_one_of_the_four_cache_points() -> None: marked: Final = [ {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": _EXPLICIT} for index in range(4) @@ -10426,21 +10467,4 @@ async def test_a_tool_result_with_two_marked_parts_is_one_cache_point_within_the _CACHE_POINT, ], }, - ] - - -@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"]) -@pytest.mark.asyncio -async def test_prompt_cache_options_on_a_non_claude_converse_model_are_forwarded_as_before( - sync_mode: bool, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(litellm, "drop_params", False) - - result: Final = await _breakpoint_converse_request( - sync_mode, - "us.amazon.nova-pro-v1:0", - [{"role": "user", "content": "hi"}], - optional_params={"prompt_cache_options": {"mode": "implicit"}}, - ) - - assert result["additionalModelRequestFields"] == {"prompt_cache_options": {"mode": "implicit"}} + ] \ No newline at end of file diff --git a/tests/unit/llms/bedrock/test_claude_platform_provider.py b/tests/unit/llms/bedrock/test_claude_platform_provider.py index 7b42d1f5a35..60a30446ae8 100644 --- a/tests/unit/llms/bedrock/test_claude_platform_provider.py +++ b/tests/unit/llms/bedrock/test_claude_platform_provider.py @@ -718,6 +718,104 @@ def test_chat_completion_claude_platform_override_kwarg_is_honoured_and_not_sent assert requests[0]["body"] == {**EXPECTED_CHAT_BODY, "context_management": context_management} +EXPLICIT_BREAKPOINT: Final = {"mode": "explicit"} +EPHEMERAL: Final = {"type": "ephemeral"} + + +def test_chat_completion_claude_platform_sends_prompt_cache_breakpoints_as_cache_control( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + + monkeypatch.setattr(litellm, "drop_params", False) + requests: Final[list[dict]] = [] + + litellm.completion( + model="bedrock/claude_platform/claude-sonnet-4-6", + messages=[ + { + "role": "system", + "content": [{"type": "text", "text": "rules", "prompt_cache_breakpoint": EXPLICIT_BREAKPOINT}], + }, + {"role": "user", "content": "hello", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}}, + ], + max_tokens=10, + workspace_id="wrkspc_test", + client=_sync_gateway_client(requests), + **API_KEY_KWARGS, + ) + + assert requests[0]["body"]["system"] == [{"type": "text", "text": "rules", "cache_control": EPHEMERAL}] + assert requests[0]["body"]["messages"] == [ + {"role": "user", "content": [{"type": "text", "text": "hello", "cache_control": EPHEMERAL}]} + ] + + +def test_chat_completion_claude_platform_holds_breakpoints_to_four_unless_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + + monkeypatch.setattr(litellm, "drop_params", False) + requests: Final[list[dict]] = [] + request: Final = { + "model": "bedrock/claude_platform/claude-sonnet-4-6", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": EXPLICIT_BREAKPOINT} + for index in range(5) + ], + } + ], + "max_tokens": 10, + "workspace_id": "wrkspc_test", + "client": _sync_gateway_client(requests), + **API_KEY_KWARGS, + } + + with pytest.raises(litellm.BadRequestError, match=r"Claude accepts at most 4 cache breakpoints, found 5"): + litellm.completion(**request) + assert requests == [] + + litellm.completion(**request, drop_params=True) + + assert requests[0]["body"]["messages"][0]["content"] == [ + {"type": "text", "text": "part 0"}, + *({"type": "text", "text": f"part {index}", "cache_control": EPHEMERAL} for index in range(1, 5)), + ] + + +@pytest.mark.parametrize("source", ["top-level", "extra_body"]) +def test_chat_completion_claude_platform_rejects_prompt_cache_options_unless_drop_params( + source: str, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm + + monkeypatch.setattr(litellm, "drop_params", False) + requests: Final[list[dict]] = [] + options: Final = {"prompt_cache_options": EXPLICIT_BREAKPOINT} + request: Final = { + "model": "bedrock/claude_platform/claude-sonnet-4-6", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 10, + "workspace_id": "wrkspc_test", + "client": _sync_gateway_client(requests), + **(options if source == "top-level" else {"extra_body": options}), + **API_KEY_KWARGS, + } + + with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"): + litellm.completion(**request) + assert requests == [] + + response: Final = litellm.completion(**request, drop_params=True) + + assert response.choices[0].message.content == "ok" + assert requests[0]["body"] == EXPECTED_CHAT_BODY + + def test_fake_claude_platform_gateway_rejects_leaked_internal_fields(): leaked = json.dumps({**EXPECTED_NATIVE_BODY, "workspace_id": "wrkspc_test", "aws_region_name": "us-west-2"}) response = _fake_claude_platform_gateway( diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index 417f3f00d09..169fce62f32 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -2314,14 +2314,20 @@ def test_a_tool_breakpoint_holds_databricks_claude_to_four_marks_under_drop_para ] +def _prompt_cache_options_param(source: str) -> dict[str, object]: + options: Final = {"prompt_cache_options": {"mode": "explicit"}} + return options if source == "top-level" else {"extra_body": options} + + +@pytest.mark.parametrize("source", ["top-level", "extra_body"]) def test_prompt_cache_options_toward_databricks_claude_are_rejected_unless_drop_params( - monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter + source: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter ) -> None: route: Final = _databricks_claude_route(respx_mock, monkeypatch) request: Final = { "model": "databricks/databricks-claude-haiku-4-5", "messages": [{"role": "user", "content": "hi"}], - "prompt_cache_options": {"mode": "explicit"}, + **_prompt_cache_options_param(source), } with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"): @@ -2333,15 +2339,16 @@ def test_prompt_cache_options_toward_databricks_claude_are_rejected_unless_drop_ assert "prompt_cache_options" not in json.loads(route.calls.last.request.content) +@pytest.mark.parametrize("source", ["top-level", "extra_body"]) def test_prompt_cache_options_toward_a_non_claude_databricks_model_are_forwarded_as_before( - monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter + source: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter ) -> None: route: Final = _databricks_claude_route(respx_mock, monkeypatch) litellm.completion( model="databricks/databricks-meta-llama-3-3-70b-instruct", messages=[{"role": "user", "content": "hi"}], - prompt_cache_options={"mode": "explicit"}, + **_prompt_cache_options_param(source), ) assert json.loads(route.calls.last.request.content)["prompt_cache_options"] == {"mode": "explicit"}