scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 200

fix(anthropic): hold every prompt_cache_options source and Converse cache point model to the Claude rules

Browse source 
prompt_cache_options sent through extra_body skipped the drop_params rule on the
routes that merge extra_body into the Claude body (Bedrock Invoke, Bedrock Claude
Platform, Azure AI, Databricks Claude), so Bedrock rejected it with its own 400.
AnthropicConfig.transform_extra_body now applies the same rule, and Databricks only
applies it to Claude models.

Converse now gates the rename, the cap and the prompt_cache_options rule on whether
the model takes cachePoint blocks, so an application inference profile ARN gets
cache points, and a model without cachePoint support gets the request as written.

prompt_cache_options is now a refusal value from claude_cache_marks next to the cap
refusal, mapped to the public exception in one place. assert_never comes from
typing_extensions so the module imports on Python 3.10. is_cache_control_set is back
to main since no header depends on it, and unmarked tools reach _map_tools as on main.
This commit is contained in:
ryan-crabbe-berri 2026-10-14 07:27:57 -07:00
parent fbd7f59e65
commit fedb3f43cf
13 changed files with 457 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

107
litellm/llms/anthropic/chat/prompt_cache_breakpoints.py
View file

@ -7,11 +7,15 @@ has no Claude equivalent and is rejected or dropped like any other unsupported p
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from itertools import chain
from typing import Final, Literal, TypeAlias, assert_never
from typing import Final, Literal, NoReturn, TypeAlias
from typing_extensions import assert_never
import litellm
from litellm._logging import verbose_logger
from litellm.constants import MAX_CACHE_CONTROL_BLOCKS
from litellm.exceptions import BadRequestError, UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
message_fields,
message_without_cache_control_marks,
@ -37,6 +41,19 @@ class CacheBreakpointCapExceeded:
found: int
@dataclass(frozen=True, slots=True)
class PromptCacheOptionsUnsupported:
options: object
ClaudeCacheRefusal: TypeAlias = CacheBreakpointCapExceeded | PromptCacheOptionsUnsupported
def drop_params_enabled(litellm_params: Mapping[str, object]) -> bool:
"""``drop_params`` from the global setting, or from the request or deployment (both arrive in ``litellm_params``)."""
return bool(litellm.drop_params) or normalize_drop_params(litellm_params.get("drop_params")) is True
def _as_mapping(value: object) -> Mapping[str, object] | None:
return value if isinstance(value, Mapping) else None # pyright: ignore[reportUnknownVariableType] # keys are read one by one
@ -150,23 +167,37 @@ def _message_without(index: int, message: AllMessageValues, dropped: frozenset[_
)
def _prompt_cache_options_refusal(options: object, drop_params: bool) -> PromptCacheOptionsUnsupported | None:
if options is None:
return None
if not drop_params:
return PromptCacheOptionsUnsupported(options)
verbose_logger.debug("Dropping prompt_cache_options=%r: Claude caches through cache_control.", options)
return None
def claude_cache_marks(
messages: Sequence[AllMessageValues],
tools: object,
*,
automatic_breakpoint: bool,
drop_params: bool,
prompt_cache_options: object = None,
tools_carried_breakpoint: bool = False,
wire: ClaudeWire = "anthropic",
) -> ClaudeCacheMarks | CacheBreakpointCapExceeded:
) -> ClaudeCacheMarks | ClaudeCacheRefusal:
"""The request's ``prompt_cache_breakpoint`` marks renamed to ``cache_control`` and, when it carried any, held to
Claude's cap: ``drop_params`` keeps the latest four in wire order, otherwise the excess is reported. A request
that carried none is forwarded as written so the provider's own answer reaches the client.
``automatic_breakpoint`` is the request-level ``cache_control`` (Anthropic's automatic caching), which takes one
slot of its own and is never dropped. ``tools_carried_breakpoint`` covers tools already mapped to Anthropic's
shape, where the mark was renamed before this census ran. ``wire`` is the body the marks are rendered into, so
the count matches the breakpoints the provider receives."""
``prompt_cache_options`` has no Claude equivalent: the caller has already taken it out of the request, and it is
reported unless ``drop_params``. ``automatic_breakpoint`` is the request-level ``cache_control`` (Anthropic's
automatic caching), which takes one slot of its own and is never dropped. ``tools_carried_breakpoint`` covers
tools already mapped to Anthropic's shape, where the mark was renamed before this census ran. ``wire`` is the
body the marks are rendered into, so the count matches the breakpoints the provider receives."""
options_refusal: Final = _prompt_cache_options_refusal(prompt_cache_options, drop_params)
if options_refusal is not None:
return options_refusal
listed_tools: Final = _as_blocks(tools) or ()
if not tools_carried_breakpoint and not _sites(messages, listed_tools, "prompt_cache_breakpoint", wire):
return ClaudeCacheMarks(tuple(messages), tuple(listed_tools))
@ -194,12 +225,20 @@ def claude_cache_marks(
)
def resolved_claude_cache_marks(
outcome: ClaudeCacheMarks | CacheBreakpointCapExceeded, model: str, llm_provider: str
) -> ClaudeCacheMarks:
match outcome:
case ClaudeCacheMarks():
return outcome
def claude_extra_body(
extra_body: Mapping[str, object], drop_params: bool
) -> Mapping[str, object] | PromptCacheOptionsUnsupported:
"""``extra_body`` merged into a Claude body, held to the same ``prompt_cache_options`` rule as the request."""
refusal: Final = _prompt_cache_options_refusal(extra_body.get("prompt_cache_options"), drop_params)
if refusal is not None:
return refusal
if "prompt_cache_options" not in extra_body:
return extra_body
return {key: value for key, value in extra_body.items() if key != "prompt_cache_options"}
def raise_claude_cache_refusal(refusal: ClaudeCacheRefusal, model: str, llm_provider: str) -> NoReturn:
match refusal:
case CacheBreakpointCapExceeded(found=found):
raise BadRequestError(
message=(
@ -209,25 +248,31 @@ def resolved_claude_cache_marks(
model=model,
llm_provider=llm_provider,
)
case PromptCacheOptionsUnsupported(options=options):
raise UnsupportedParamsError(
message=(
f"{model} does not support prompt_cache_options={options!r}. It caches through cache_control "
"marks (or prompt_cache_breakpoint on the OpenAI dialect). To drop unsupported params, set "
"`litellm.drop_params = True`."
),
model=model,
status_code=400,
)
case _:
assert_never(outcome)
assert_never(refusal)
def reject_prompt_cache_options(model: str, options: object, drop_params: bool) -> None:
"""``prompt_cache_options`` has already been popped from the request; this decides whether that was allowed."""
if options is None:
return
if drop_params:
verbose_logger.debug(
"Dropping prompt_cache_options=%r for %s: Claude caches through cache_control.", options, model
)
return
raise UnsupportedParamsError(
message=(
f"{model} does not support prompt_cache_options={options!r}. Claude caches through cache_control "
"marks (or prompt_cache_breakpoint on the OpenAI dialect). To drop unsupported params, set "
"`litellm.drop_params = True`."
),
model=model,
status_code=400,
)
def resolved_claude_cache_marks(
outcome: ClaudeCacheMarks | ClaudeCacheRefusal, model: str, llm_provider: str
) -> ClaudeCacheMarks:
if isinstance(outcome, ClaudeCacheMarks):
return outcome
raise_claude_cache_refusal(outcome, model=model, llm_provider=llm_provider)
def resolved_claude_extra_body(
outcome: Mapping[str, object] | PromptCacheOptionsUnsupported, model: str, llm_provider: str
) -> Mapping[str, object]:
if isinstance(outcome, PromptCacheOptionsUnsupported):
raise_claude_cache_refusal(outcome, model=model, llm_provider=llm_provider)
return outcome

28
litellm/llms/anthropic/chat/transformation.py
View file

@ -114,8 +114,10 @@ from ..common_utils import (
from .prompt_cache_breakpoints import (
PROMPT_CACHE_BREAKPOINT_ON_TOOLS,
claude_cache_marks,
reject_prompt_cache_options,
claude_extra_body,
drop_params_enabled,
resolved_claude_cache_marks,
resolved_claude_extra_body,
tools_carry_prompt_cache_breakpoint,
)
@ -1579,6 +1581,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
requested_tools = cast(object, value) # cast-ok: the loop yields untyped values
anthropic_tools, mcp_servers = self._map_tools(
list(tools_with_prompt_cache_breakpoints_as_cache_control(requested_tools))
if tools_carry_prompt_cache_breakpoint(requested_tools)
else value
)
optional_params = self._add_tools_to_optional_params(
optional_params=optional_params, tools=anthropic_tools
@ -1983,6 +1987,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
headers=headers,
)
def transform_extra_body(
self,
extra_body: Mapping[str, object],
request: Mapping[str, object],
model: str,
litellm_params: Mapping[str, object],
) -> Mapping[str, object]:
return resolved_claude_extra_body(
claude_extra_body(extra_body, drop_params=drop_params_enabled(litellm_params)),
model=model,
llm_provider=self._resolved_provider,
)
def transform_request(
self,
model: str,
@ -2023,11 +2040,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"has no thinking_blocks. The model won't use extended thinking for this turn."
)
drop_params: Final = litellm.drop_params or litellm_params.get("drop_params") is True
AnthropicConfig._maybe_drop_speed_param(
model=model,
optional_params=optional_params,
drop_params=drop_params,
drop_params=litellm.drop_params or litellm_params.get("drop_params") is True,
custom_llm_provider=self.custom_llm_provider,
)
@ -2038,15 +2054,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
body: Final = cast(dict[str, object], optional_params) # cast-ok: BaseConfig passes an untyped dict
reject_prompt_cache_options(
model=model, options=body.pop("prompt_cache_options", None), drop_params=drop_params
)
cache_marks: Final = resolved_claude_cache_marks(
claude_cache_marks(
messages=messages,
tools=body.get("tools"),
automatic_breakpoint=body.get("cache_control") is not None,
drop_params=drop_params,
drop_params=drop_params_enabled(litellm_params),
prompt_cache_options=body.pop("prompt_cache_options", None),
tools_carried_breakpoint=bool(body.pop(PROMPT_CACHE_BREAKPOINT_ON_TOOLS, False)),
),
model=model,

12
litellm/llms/anthropic/common_utils.py
View file

@ -106,7 +106,6 @@ _INFERENCE_PROFILE_MINOR_RE: Final = re.compile(r":\d+$")
_DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$")
_DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)")
_CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
CACHE_MARK_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
_CLAUDE_CODE_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
_CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
@ -462,15 +461,18 @@ class AnthropicModelInfo(BaseLLMModelInfo):
_workload_identity_eligible: ClassVar[bool] = True
def is_cache_control_set(self, messages: list[AllMessageValues]) -> bool:
"""Whether a message or one of its content blocks carries a cache mark, Anthropic's ``cache_control`` or
OpenAI's ``prompt_cache_breakpoint`` (which the chat transform renames on the way to Claude)."""
"""
Return if {"cache_control": ..} in message content block
Used to check if anthropic prompt caching headers need to be set.
"""
for message in messages:
if any(message.get(key) is not None for key in CACHE_MARK_KEYS):
if message.get("cache_control", None) is not None:
return True
_message_content = message.get("content")
if _message_content is not None and isinstance(_message_content, list):
for content in _message_content:
if isinstance(content, dict) and any(key in content for key in CACHE_MARK_KEYS):
if isinstance(content, dict) and "cache_control" in content:
return True
return False

27
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -46,7 +46,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
)
from litellm.llms.anthropic.chat.prompt_cache_breakpoints import (
claude_cache_marks,
reject_prompt_cache_options,
drop_params_enabled,
resolved_claude_cache_marks,
)
from litellm.llms.anthropic.chat.transformation import (
@ -57,7 +57,6 @@ from litellm.llms.anthropic.chat.transformation import (
)
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_images
from litellm.llms.bedrock.common_utils import bedrock_model_supports_regex_lookaround
from litellm.llms.bedrock.request_metadata import (
@ -2018,31 +2017,25 @@ class AmazonConverseConfig(BaseConfig):
return data
def _messages_with_claude_cache_marks(
def _messages_with_prompt_cache_breakpoints(
self,
model: str,
messages: Sequence[AllMessageValues],
optional_params: dict[str, object], # mutable-ok: the request body is built from this out-param
litellm_params: Mapping[str, object],
) -> tuple[AllMessageValues, ...]:
"""On Claude, OpenAI ``prompt_cache_breakpoint`` marks as ``cache_control`` so the factory emits
``cachePoint`` blocks, held to Claude's cap, with ``prompt_cache_options`` rejected or dropped. A Claude model
that never emits ``cachePoint`` blocks keeps its marks as written. Other models are left as they are."""
if BaseAWSLLM.get_bedrock_invoke_provider(model) != "anthropic":
return tuple(messages)
emits_cache_points: Final = bedrock_model_accepts_cache_points(model)
drop_params: Final = litellm.drop_params or litellm_params.get("drop_params") is True
reject_prompt_cache_options(
model=model, options=optional_params.pop("prompt_cache_options", None), drop_params=drop_params
)
if not emits_cache_points:
"""On a model that takes ``cachePoint`` blocks, OpenAI ``prompt_cache_breakpoint`` marks as ``cache_control``
so the factory emits them, held to the cap of four, with ``prompt_cache_options`` rejected or dropped. Any
other model gets the request as written."""
if not bedrock_model_accepts_cache_points(model):
return tuple(messages)
cache_marks: Final = resolved_claude_cache_marks(
claude_cache_marks(
messages=messages,
tools=optional_params.get("tools"),
automatic_breakpoint=optional_params.get("cache_control") is not None,
drop_params=drop_params,
drop_params=drop_params_enabled(litellm_params),
prompt_cache_options=optional_params.pop("prompt_cache_options", None),
wire="bedrock_converse",
),
model=model,
@ -2063,7 +2056,7 @@ class AmazonConverseConfig(BaseConfig):
body: Final = cast(dict[str, object], optional_params) # cast-ok: BaseConfig passes an untyped dict
params: Final = cast(Mapping[str, object], litellm_params) # cast-ok: BaseConfig passes an untyped dict
messages, system_content_blocks = self._transform_system_message(
list(self._messages_with_claude_cache_marks(model, messages, body, params)), model=model
list(self._messages_with_prompt_cache_breakpoints(model, messages, body, params)), model=model
)
# Convert last user message to guarded_text if guardrailConfig is present
@ -2135,7 +2128,7 @@ class AmazonConverseConfig(BaseConfig):
body: Final = cast(dict[str, object], optional_params) # cast-ok: BaseConfig passes an untyped dict
params: Final = cast(Mapping[str, object], litellm_params) # cast-ok: BaseConfig passes an untyped dict
messages, system_content_blocks = self._transform_system_message(
list(self._messages_with_claude_cache_marks(model, messages, body, params)), model=model
list(self._messages_with_prompt_cache_breakpoints(model, messages, body, params)), model=model
)
# Convert last user message to guarded_text if guardrailConfig is present

27
litellm/llms/databricks/chat/transformation.py
View file

@ -9,7 +9,6 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload
import httpx
from pydantic import BaseModel, TypeAdapter
import litellm
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
handle_invalid_parallel_tool_calls,
@ -59,8 +58,10 @@ from litellm.types.utils import (
from ...anthropic.chat.prompt_cache_breakpoints import (
PROMPT_CACHE_BREAKPOINT_ON_TOOLS,
claude_cache_marks,
reject_prompt_cache_options,
claude_extra_body,
drop_params_enabled,
resolved_claude_cache_marks,
resolved_claude_extra_body,
tools_carry_prompt_cache_breakpoint,
)
from ...anthropic.chat.transformation import (
@ -468,6 +469,21 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
),
)
def transform_extra_body(
self,
extra_body: Mapping[str, object],
request: Mapping[str, object],
model: str,
litellm_params: Mapping[str, object],
) -> Mapping[str, object]:
if "claude" not in model:
return extra_body
return resolved_claude_extra_body(
claude_extra_body(extra_body, drop_params=drop_params_enabled(litellm_params)),
model=model,
llm_provider="databricks",
)
def _messages_with_claude_cache_marks(
self,
model: str,
@ -479,16 +495,13 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
"""OpenAI ``prompt_cache_breakpoint`` marks as the ``cache_control`` Databricks forwards to Claude, held to
Claude's cap, with ``prompt_cache_options`` rejected or dropped. Tool definitions reach Databricks without
any mark, so only the message marks count toward the cap."""
drop_params: Final = litellm.drop_params or litellm_params.get("drop_params") is True
reject_prompt_cache_options(
model=model, options=optional_params.pop("prompt_cache_options", None), drop_params=drop_params
)
cache_marks: Final = resolved_claude_cache_marks(
claude_cache_marks(
messages=messages,
tools=(),
automatic_breakpoint=optional_params.get("cache_control") is not None,
drop_params=drop_params,
drop_params=drop_params_enabled(litellm_params),
prompt_cache_options=optional_params.pop("prompt_cache_options", None),
tools_carried_breakpoint=tools_carried_breakpoint,
),
model=model,

52
tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_wire.py
View file

@ -39,10 +39,20 @@ from integration.providers._cache_control_marks_support import (
)
from pydantic import BaseModel, ConfigDict, JsonValue
Provider = Literal["anthropic", "bedrock_converse", "bedrock_invoke", "vertex_ai", "azure_ai"]
Provider = Literal[
"anthropic", "bedrock_converse", "bedrock_invoke", "bedrock_claude_platform", "vertex_ai", "azure_ai"
]
Site = Literal["tool", "system", "user_part", "tool_call", "tool_result_part", "string_message"]
_PROVIDERS: Final[tuple[Provider, ...]] = ("anthropic", "bedrock_converse", "bedrock_invoke", "vertex_ai", "azure_ai")
_PROVIDERS: Final[tuple[Provider, ...]] = (
"anthropic",
"bedrock_converse",
"bedrock_invoke",
"bedrock_claude_platform",
"vertex_ai",
"azure_ai",
)
_EXTRA_BODY_PROVIDERS: Final[tuple[Provider, ...]] = ("bedrock_invoke", "bedrock_claude_platform", "azure_ai")
_SITES: Final[tuple[Site, ...]] = ("tool", "system", "user_part", "tool_call", "tool_result_part", "string_message")
_FIVE_SITES: Final[tuple[Site, ...]] = ("tool", "system", "user_part", "tool_call", "string_message")
_EXPLICIT: Final[dict[str, JsonValue]] = {"mode": "explicit"}
@ -59,6 +69,7 @@ _TARGETS: Final[Mapping[Provider, str]] = MappingProxyType(
"anthropic": "/v1/messages",
"bedrock_converse": f"/model/{BEDROCK_MODEL}/converse",
"bedrock_invoke": f"/model/{BEDROCK_MODEL}/invoke",
"bedrock_claude_platform": "/v1/messages",
"vertex_ai": (
f"/v1/projects/{_VERTEX_PROJECT}/locations/{_VERTEX_LOCATION}"
f"/publishers/anthropic/models/{ANTHROPIC_MODEL}:rawPredict"
@ -138,6 +149,14 @@ def _litellm_params(gateway: Gateway, wire: Wire, provider: Provider) -> dict[st
"api_base": wire.url,
"aws_bedrock_runtime_endpoint": wire.url,
}
case "bedrock_claude_platform":
return {
"model": f"bedrock/claude_platform/{ANTHROPIC_MODEL}",
"api_base": wire.url,
"api_key": PROVIDER_KEY,
"workspace_id": "wrkspc_scripted",
"aws_region_name": "us-east-1",
}
case "vertex_ai":
return {
"model": f"vertex_ai/{ANTHROPIC_MODEL}",
@ -351,6 +370,35 @@ def test_prompt_cache_options_are_a_400_before_any_provider_call_without_drop_pa
assert received == (), [request.target for request in received]
@pytest.mark.parametrize("provider", _EXTRA_BODY_PROVIDERS)
def test_prompt_cache_options_in_extra_body_are_a_400_before_any_provider_call_without_drop_params(
gateway: Gateway, provider: Provider
) -> None:
marker: Final = new_marker()
extra_body: Final[dict[str, JsonValue]] = {"prompt_cache_options": _EXPLICIT}
with wire_server(lambda request: _respond(provider, request)) as wire, gateway.scenario() as scenario:
model: Final = _deployment(gateway, scenario, wire, provider)
status, _, text = post_chat(gateway, _body(model, marker, _marked(("user_part",)), extra_body=extra_body))
received: Final = wire.drain()
assert status == 400, text
assert "prompt_cache_options" in text, text
assert "drop_params" in text, text
assert received == (), [request.target for request in received]
@pytest.mark.parametrize("provider", _EXTRA_BODY_PROVIDERS)
def test_prompt_cache_options_in_extra_body_are_dropped_under_drop_params(gateway: Gateway, provider: Provider) -> None:
marker: Final = new_marker()
extra_body: Final[dict[str, JsonValue]] = {"prompt_cache_options": _EXPLICIT}
with wire_server(lambda request: _respond(provider, request)) as wire, gateway.scenario() as scenario:
model: Final = _deployment(gateway, scenario, wire, provider, drop_params=True)
status, _, text = post_chat(gateway, _body(model, marker, _marked(("user_part",)), extra_body=extra_body))
assert status == 200, text
request: Final = _only_request(wire)
assert b"prompt_cache_options" not in request.body, request.body
assert _labels(provider, request) == [ASK_LABEL]
@pytest.mark.parametrize("provider", _PROVIDERS)
def test_prompt_cache_options_are_dropped_under_drop_params(gateway: Gateway, provider: Provider) -> None:
marker: Final = new_marker()

12
tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -1,5 +1,6 @@
import copy
import json
from collections.abc import Callable
from typing import Final
from unittest.mock import MagicMock, patch
@ -9085,9 +9086,16 @@ async def test_a_message_level_breakpoint_on_list_content_does_not_count_toward_
]
def test_map_openai_params_maps_a_tuple_of_tools() -> None:
@pytest.mark.parametrize(
"tools",
[
pytest.param(lambda tool: (tool,), id="tuple"),
pytest.param(lambda tool: iter([tool]), id="iterator"),
],
)
def test_map_openai_params_maps_tools_given_as_any_iterable(tools: Callable[[object], object]) -> None:
optional_params: Final = AnthropicConfig().map_openai_params(
non_default_params={"tools": ({"type": "function", "function": _WEATHER_FUNCTION},)},
non_default_params={"tools": tools({"type": "function", "function": _WEATHER_FUNCTION})},
optional_params={},
model=PROMPT_CACHING_MODEL,
drop_params=False,

66
tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py
View file

@ -6,9 +6,12 @@ import litellm
from litellm.llms.anthropic.chat.prompt_cache_breakpoints import (
CacheBreakpointCapExceeded,
ClaudeCacheMarks,
PromptCacheOptionsUnsupported,
claude_cache_marks,
reject_prompt_cache_options,
claude_extra_body,
drop_params_enabled,
resolved_claude_cache_marks,
resolved_claude_extra_body,
tools_carry_prompt_cache_breakpoint,
)
@ -327,15 +330,68 @@ def test_marks_within_the_cap_resolve_to_themselves() -> None:
assert resolved_claude_cache_marks(marks, model=MODEL, llm_provider="anthropic") is marks
def test_prompt_cache_options_are_rejected_unless_drop_params() -> None:
def test_prompt_cache_options_are_reported_before_the_marks_unless_drop_params() -> None:
messages, tools = _six_breakpoints()
assert claude_cache_marks(
messages, tools, automatic_breakpoint=False, drop_params=False, prompt_cache_options=EXPLICIT
) == PromptCacheOptionsUnsupported(EXPLICIT)
kept: Final = claude_cache_marks(
messages, tools, automatic_breakpoint=False, drop_params=True, prompt_cache_options=EXPLICIT
)
assert isinstance(kept, ClaudeCacheMarks)
assert claude_cache_marks(messages, tools, automatic_breakpoint=False, drop_params=True) == kept
def test_the_prompt_cache_options_error_is_an_unsupported_param_naming_drop_params() -> None:
with pytest.raises(litellm.UnsupportedParamsError) as caught:
reject_prompt_cache_options(model=MODEL, options={"mode": "explicit"}, drop_params=False)
resolved_claude_cache_marks(PromptCacheOptionsUnsupported(EXPLICIT), model=MODEL, llm_provider="anthropic")
assert "prompt_cache_options" in str(caught.value)
assert "drop_params" in str(caught.value)
assert caught.value.status_code == 400
assert reject_prompt_cache_options(model=MODEL, options={"mode": "explicit"}, drop_params=True) is None
assert reject_prompt_cache_options(model=MODEL, options=None, drop_params=False) is None
@pytest.mark.parametrize(
("extra_body", "drop_params", "expected"),
[
({"metadata": {"a": 1}}, False, {"metadata": {"a": 1}}),
({"metadata": {"a": 1}, "prompt_cache_options": EXPLICIT}, True, {"metadata": {"a": 1}}),
({"metadata": {"a": 1}, "prompt_cache_options": EXPLICIT}, False, PromptCacheOptionsUnsupported(EXPLICIT)),
],
ids=["no-options", "dropped", "rejected"],
)
def test_extra_body_follows_the_prompt_cache_options_rule(
extra_body: dict[str, object], drop_params: bool, expected: object
) -> None:
assert claude_extra_body(extra_body, drop_params=drop_params) == expected
def test_a_rejected_extra_body_raises_and_an_accepted_one_resolves_to_itself() -> None:
body: Final = {"metadata": {"a": 1}}
assert resolved_claude_extra_body(body, model=MODEL, llm_provider="anthropic") is body
with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"):
resolved_claude_extra_body(PromptCacheOptionsUnsupported(EXPLICIT), model=MODEL, llm_provider="anthropic")
@pytest.mark.parametrize(
("global_drop", "litellm_params", "expected"),
[
(False, {}, False),
(True, {}, True),
(False, {"drop_params": True}, True),
(False, {"drop_params": "true"}, True),
(False, {"drop_params": False}, False),
],
ids=["off", "global", "request", "request-string", "request-false"],
)
def test_drop_params_enabled_reads_the_global_and_the_request(
global_drop: bool, litellm_params: dict[str, object], expected: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", global_drop)
assert drop_params_enabled(litellm_params) is expected
@pytest.mark.parametrize(

15
tests/unit/llms/anthropic/test_anthropic_common_utils.py
View file

@ -4267,18 +4267,3 @@ def test_anthropic_error_frame_exception_classes_an_overloaded_frame_as_internal
error: Final = anthropic_error_frame_exception("overloaded_error", "Overloaded", 503, "claude-sonnet-4-5")
assert type(error) is litellm.InternalServerError
@pytest.mark.parametrize(
"messages",
[
[{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": {"mode": "explicit"}}]}],
[{"role": "user", "content": "hi", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}}],
[{"role": "system", "content": [{"type": "text", "text": "rules", "prompt_cache_breakpoint": {"mode": "explicit"}}]}],
],
ids=["content-part", "message-level-on-string", "system-part"],
)
def test_is_cache_control_set_counts_a_prompt_cache_breakpoint_like_a_cache_control(messages: list) -> None:
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
assert AnthropicModelInfo().is_cache_control_set(messages) is True

43
tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py
View file

@ -873,6 +873,49 @@ async def test_bedrock_invoke_claude_async_completion_inlines_document_url_sourc
} in captured["body"]["messages"][0]["content"]
async def test_bedrock_invoke_claude_holds_extra_body_prompt_cache_options_to_drop_params(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
bodies: Final[list[dict[str, object]]] = [] # mutable-ok: the transport records each request
def handle(request: httpx.Request) -> httpx.Response:
bodies.append(json.loads(request.content))
return httpx.Response(
200,
json={
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "us.anthropic.claude-sonnet-5",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1},
},
)
client: Final = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
request: Final = {
"model": "bedrock/invoke/us.anthropic.claude-sonnet-5",
"messages": [{"role": "user", "content": "hi"}],
"extra_body": {"prompt_cache_options": {"mode": "explicit"}, "top_k": 3},
"aws_access_key_id": "AKIAEXAMPLE",
"aws_secret_access_key": "fake-secret",
"aws_region_name": "us-east-1",
"client": client,
}
with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"):
await litellm.acompletion(**request)
assert bodies == []
await litellm.acompletion(**request, drop_params=True)
assert "prompt_cache_options" not in bodies[-1]
assert bodies[-1]["top_k"] == 3
@pytest.mark.parametrize(
"model, expected_betas",
[

76
tests/unit/llms/bedrock/chat/test_converse_transformation.py
View file

@ -10293,19 +10293,12 @@ async def test_six_breakpoints_are_a_bad_request_without_drop_params(
await _breakpoint_request(sync_mode, {"tools": [{**_LOOKUP_TOOL, "prompt_cache_breakpoint": _EXPLICIT}]}, {})
@pytest.mark.parametrize("emits_cache_points", [True, False], ids=["cache-points", "no-cache-points"])
@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"])
@pytest.mark.asyncio
async def test_prompt_cache_options_are_rejected_unless_drop_params(
sync_mode: bool, emits_cache_points: bool, monkeypatch: pytest.MonkeyPatch
sync_mode: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
if not emits_cache_points:
monkeypatch.setitem(
litellm.model_cost,
_BREAKPOINT_MODEL,
{"litellm_provider": "bedrock", "mode": "chat", "supports_prompt_cache_breakpoint": False},
)
with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"):
await _breakpoint_request(sync_mode, {"prompt_cache_options": {"mode": "explicit"}}, {})
@ -10319,6 +10312,54 @@ async def test_prompt_cache_options_are_rejected_unless_drop_params(
assert "prompt_cache_options" not in json.dumps(dropped_globally)
_INFERENCE_PROFILE_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123"
@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"])
@pytest.mark.asyncio
async def test_an_inference_profile_arn_gets_cache_points_and_the_prompt_cache_options_rule(
sync_mode: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = [
{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": _EXPLICIT}]}
]
result: Final = await _breakpoint_converse_request(sync_mode, _INFERENCE_PROFILE_ARN, messages)
assert result["messages"] == [{"role": "user", "content": [{"text": "hi"}, _CACHE_POINT]}]
with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"):
await _breakpoint_converse_request(
sync_mode, _INFERENCE_PROFILE_ARN, messages, optional_params={"prompt_cache_options": _EXPLICIT}
)
@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"])
@pytest.mark.asyncio
async def test_a_model_without_cache_points_gets_the_request_as_written(
sync_mode: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
monkeypatch.setitem(
litellm.model_cost,
_BREAKPOINT_MODEL,
{"litellm_provider": "bedrock", "mode": "chat", "supports_prompt_cache_breakpoint": False},
)
marked: Final = [
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": _EXPLICIT} for index in range(5)
]
result: Final = await _breakpoint_converse_request(
sync_mode,
_BREAKPOINT_MODEL,
[{"role": "user", "content": marked}],
optional_params={"prompt_cache_options": {"mode": "implicit"}},
)
assert result["messages"] == [{"role": "user", "content": [{"text": f"part {index}"} for index in range(5)]}]
assert result["additionalModelRequestFields"] == {"prompt_cache_options": {"mode": "implicit"}}
def test_the_automatic_cache_control_takes_one_of_the_four_cache_points() -> None:
marked: Final = [
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": _EXPLICIT} for index in range(4)
@ -10426,21 +10467,4 @@ async def test_a_tool_result_with_two_marked_parts_is_one_cache_point_within_the
_CACHE_POINT,
],
},
]
@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"])
@pytest.mark.asyncio
async def test_prompt_cache_options_on_a_non_claude_converse_model_are_forwarded_as_before(
sync_mode: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
result: Final = await _breakpoint_converse_request(
sync_mode,
"us.amazon.nova-pro-v1:0",
[{"role": "user", "content": "hi"}],
optional_params={"prompt_cache_options": {"mode": "implicit"}},
)
assert result["additionalModelRequestFields"] == {"prompt_cache_options": {"mode": "implicit"}}
]

98
tests/unit/llms/bedrock/test_claude_platform_provider.py
View file

@ -718,6 +718,104 @@ def test_chat_completion_claude_platform_override_kwarg_is_honoured_and_not_sent
assert requests[0]["body"] == {**EXPECTED_CHAT_BODY, "context_management": context_management}
EXPLICIT_BREAKPOINT: Final = {"mode": "explicit"}
EPHEMERAL: Final = {"type": "ephemeral"}
def test_chat_completion_claude_platform_sends_prompt_cache_breakpoints_as_cache_control(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import litellm
monkeypatch.setattr(litellm, "drop_params", False)
requests: Final[list[dict]] = []
litellm.completion(
model="bedrock/claude_platform/claude-sonnet-4-6",
messages=[
{
"role": "system",
"content": [{"type": "text", "text": "rules", "prompt_cache_breakpoint": EXPLICIT_BREAKPOINT}],
},
{"role": "user", "content": "hello", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}},
],
max_tokens=10,
workspace_id="wrkspc_test",
client=_sync_gateway_client(requests),
**API_KEY_KWARGS,
)
assert requests[0]["body"]["system"] == [{"type": "text", "text": "rules", "cache_control": EPHEMERAL}]
assert requests[0]["body"]["messages"] == [
{"role": "user", "content": [{"type": "text", "text": "hello", "cache_control": EPHEMERAL}]}
]
def test_chat_completion_claude_platform_holds_breakpoints_to_four_unless_drop_params(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import litellm
monkeypatch.setattr(litellm, "drop_params", False)
requests: Final[list[dict]] = []
request: Final = {
"model": "bedrock/claude_platform/claude-sonnet-4-6",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": EXPLICIT_BREAKPOINT}
for index in range(5)
],
}
],
"max_tokens": 10,
"workspace_id": "wrkspc_test",
"client": _sync_gateway_client(requests),
**API_KEY_KWARGS,
}
with pytest.raises(litellm.BadRequestError, match=r"Claude accepts at most 4 cache breakpoints, found 5"):
litellm.completion(**request)
assert requests == []
litellm.completion(**request, drop_params=True)
assert requests[0]["body"]["messages"][0]["content"] == [
{"type": "text", "text": "part 0"},
*({"type": "text", "text": f"part {index}", "cache_control": EPHEMERAL} for index in range(1, 5)),
]
@pytest.mark.parametrize("source", ["top-level", "extra_body"])
def test_chat_completion_claude_platform_rejects_prompt_cache_options_unless_drop_params(
source: str, monkeypatch: pytest.MonkeyPatch
) -> None:
import litellm
monkeypatch.setattr(litellm, "drop_params", False)
requests: Final[list[dict]] = []
options: Final = {"prompt_cache_options": EXPLICIT_BREAKPOINT}
request: Final = {
"model": "bedrock/claude_platform/claude-sonnet-4-6",
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 10,
"workspace_id": "wrkspc_test",
"client": _sync_gateway_client(requests),
**(options if source == "top-level" else {"extra_body": options}),
**API_KEY_KWARGS,
}
with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"):
litellm.completion(**request)
assert requests == []
response: Final = litellm.completion(**request, drop_params=True)
assert response.choices[0].message.content == "ok"
assert requests[0]["body"] == EXPECTED_CHAT_BODY
def test_fake_claude_platform_gateway_rejects_leaked_internal_fields():
leaked = json.dumps({**EXPECTED_NATIVE_BODY, "workspace_id": "wrkspc_test", "aws_region_name": "us-west-2"})
response = _fake_claude_platform_gateway(

15
tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py
View file

@ -2314,14 +2314,20 @@ def test_a_tool_breakpoint_holds_databricks_claude_to_four_marks_under_drop_para
]
def _prompt_cache_options_param(source: str) -> dict[str, object]:
options: Final = {"prompt_cache_options": {"mode": "explicit"}}
return options if source == "top-level" else {"extra_body": options}
@pytest.mark.parametrize("source", ["top-level", "extra_body"])
def test_prompt_cache_options_toward_databricks_claude_are_rejected_unless_drop_params(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
source: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> None:
route: Final = _databricks_claude_route(respx_mock, monkeypatch)
request: Final = {
"model": "databricks/databricks-claude-haiku-4-5",
"messages": [{"role": "user", "content": "hi"}],
"prompt_cache_options": {"mode": "explicit"},
**_prompt_cache_options_param(source),
}
with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"):
@ -2333,15 +2339,16 @@ def test_prompt_cache_options_toward_databricks_claude_are_rejected_unless_drop_
assert "prompt_cache_options" not in json.loads(route.calls.last.request.content)
@pytest.mark.parametrize("source", ["top-level", "extra_body"])
def test_prompt_cache_options_toward_a_non_claude_databricks_model_are_forwarded_as_before(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
source: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> None:
route: Final = _databricks_claude_route(respx_mock, monkeypatch)
litellm.completion(
model="databricks/databricks-meta-llama-3-3-70b-instruct",
messages=[{"role": "user", "content": "hi"}],
prompt_cache_options={"mode": "explicit"},
**_prompt_cache_options_param(source),
)
assert json.loads(route.calls.last.request.content)["prompt_cache_options"] == {"mode": "explicit"}