From 3acf5f514a9cc484d2390072fd944f823b70293d Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 13 Oct 2026 21:42:45 -0700 Subject: [PATCH] fix(anthropic): close review gaps in prompt_cache_breakpoint to cache_control for Claude Converse now rejects or drops prompt_cache_options for every Claude model, not only the ones that emit cache points, and honors the global drop_params flag Databricks Claude deployments translate prompt_cache_breakpoint marks to cache_control, hold them to Claude's four-mark cap, and reject prompt_cache_options unless drop_params is on Tuple tool lists are accepted, and a message-level mark on list content no longer takes one of the four slots because Claude never renders it The e2e suite gains a Claude class that writes then reads the cache across anthropic, Bedrock Converse, Bedrock Invoke and Vertex for every breakpoint site --- .../prompt_templates/common_utils.py | 4 +- .../chat/prompt_cache_breakpoints.py | 9 +- .../bedrock/chat/converse_transformation.py | 9 +- .../llms/databricks/chat/transformation.py | 62 +++++ .../coverage_registry/llm_conversational.yaml | 1 + .../e2e/llm_translation/test_cache_control.py | 239 +++++++++++++++++- tests/e2e/models.py | 12 + .../test_anthropic_chat_transformation.py | 90 ++++++- .../chat/test_prompt_cache_breakpoints.py | 41 ++- ...ations_anthropic_claude3_transformation.py | 13 +- .../chat/test_converse_transformation.py | 76 +++++- .../test_databricks_chat_transformation.py | 76 ++++++ 12 files changed, 607 insertions(+), 25 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 31733cf61f3..a6678d56fd7 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1827,7 +1827,7 @@ def _as_mapping(value: object) -> Mapping[str, object] | None: def _as_list(value: object) -> Sequence[object] | None: - return value if isinstance(value, list) else None # pyright: ignore[reportUnknownVariableType] # items are parsed one by one + return value if isinstance(value, (list, tuple)) else None # pyright: ignore[reportUnknownVariableType] # items are parsed one by one def message_fields(message: AllMessageValues) -> Mapping[str, object]: @@ -1930,7 +1930,7 @@ def _tool_block_with_prompt_cache_breakpoint_as_cache_control(tool: object) -> o def tools_with_prompt_cache_breakpoints_as_cache_control(tools: object) -> tuple[object, ...]: - """Every tool definition in ``tools`` (anything but a list is treated as none) with its mark renamed.""" + """Every tool definition in ``tools`` (anything but a list or tuple is treated as none) with its mark renamed.""" return tuple(_tool_block_with_prompt_cache_breakpoint_as_cache_control(tool) for tool in _as_list(tools) or ()) diff --git a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py index 2bad4d8920f..94ac9e5aaab 100644 --- a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py +++ b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py @@ -39,7 +39,7 @@ def _as_mapping(value: object) -> Mapping[str, object] | None: def _as_blocks(value: object) -> Sequence[object] | None: - return value if isinstance(value, list) else None # pyright: ignore[reportUnknownVariableType] # blocks are parsed one by one + return value if isinstance(value, (list, tuple)) else None # pyright: ignore[reportUnknownVariableType] # blocks are parsed one by one def _carries(block: object, key: str) -> bool: @@ -56,11 +56,16 @@ def tools_carry_prompt_cache_breakpoint(tools: object) -> bool: return any(_tool_carries(tool, "prompt_cache_breakpoint") for tool in _as_blocks(tools) or ()) +def _renders_message_mark(message: Mapping[str, object]) -> bool: + """Claude's message builder keeps a message-level mark on string content and on tool results only.""" + return isinstance(message.get("content"), str) or message.get("role") in ("tool", "function") + + def _message_sites(index: int, message: object, key: str) -> Iterator[_Site]: fields: Final = _as_mapping(message) if fields is None: return - if fields.get(key) is not None: + if fields.get(key) is not None and _renders_message_mark(fields): yield ("message", index, -1) for position, part in enumerate(_as_blocks(fields.get("content")) or ()): if _carries(part, key): diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 3221b902373..b156e6f7556 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -57,6 +57,7 @@ from litellm.llms.anthropic.chat.transformation import ( ) from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_images from litellm.llms.bedrock.common_utils import bedrock_model_supports_regex_lookaround from litellm.llms.bedrock.request_metadata import ( @@ -2026,13 +2027,17 @@ class AmazonConverseConfig(BaseConfig): ) -> tuple[AllMessageValues, ...]: """OpenAI ``prompt_cache_breakpoint`` marks as ``cache_control`` so the factory emits ``cachePoint`` blocks, held to the provider's cap, with ``prompt_cache_options`` rejected or dropped. Models that never emit - ``cachePoint`` blocks are left as written.""" - if not bedrock_model_accepts_cache_points(model): + ``cachePoint`` blocks keep their marks as written, but a Claude model among them still has + ``prompt_cache_options`` rejected or dropped.""" + emits_cache_points: Final = bedrock_model_accepts_cache_points(model) + if not emits_cache_points and BaseAWSLLM.get_bedrock_invoke_provider(model) != "anthropic": return tuple(messages) drop_params: Final = litellm.drop_params or litellm_params.get("drop_params") is True reject_prompt_cache_options( model=model, options=optional_params.pop("prompt_cache_options", None), drop_params=drop_params ) + if not emits_cache_points: + return tuple(messages) cache_marks: Final = resolved_claude_cache_marks( claude_cache_marks( messages=messages, diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 4b6f178fa1a..9e64826d928 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload import httpx from pydantic import BaseModel, TypeAdapter +import litellm from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( handle_invalid_parallel_tool_calls, @@ -55,6 +56,13 @@ from litellm.types.utils import ( Usage, ) +from ...anthropic.chat.prompt_cache_breakpoints import ( + PROMPT_CACHE_BREAKPOINT_ON_TOOLS, + claude_cache_marks, + reject_prompt_cache_options, + resolved_claude_cache_marks, + tools_carry_prompt_cache_breakpoint, +) from ...anthropic.chat.transformation import ( REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT, AnthropicConfig, @@ -375,6 +383,9 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): ) if "tools" in mapped_params: mapped_params["tools"] = self._map_openai_to_dbrx_tool(model=model, tools=mapped_params["tools"]) + requested_params: Final = cast(Mapping[str, object], non_default_params) # cast-ok: base signature is untyped + if "claude" in model and tools_carry_prompt_cache_breakpoint(requested_params.get("tools")): + mapped_params[PROMPT_CACHE_BREAKPOINT_ON_TOOLS] = True if "max_completion_tokens" in non_default_params and replace_max_completion_tokens_with_max_tokens: mapped_params["max_tokens"] = non_default_params[ "max_completion_tokens" @@ -434,6 +445,57 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): return mapped_params + def transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: overrides BaseConfig.transform_request + optional_params: dict[str, object], # mutable-ok: overrides BaseConfig.transform_request + litellm_params: dict[str, object], # mutable-ok: overrides BaseConfig.transform_request + headers: dict[str, object], # mutable-ok: overrides BaseConfig.transform_request + ) -> dict[str, object]: # mutable-ok: overrides BaseConfig.transform_request + tools_carried_breakpoint: Final = bool(optional_params.pop(PROMPT_CACHE_BREAKPOINT_ON_TOOLS, False)) + marked_messages: Final = ( + self._messages_with_claude_cache_marks( + model, messages, optional_params, litellm_params, tools_carried_breakpoint + ) + if "claude" in model + else messages + ) + return cast( # cast-ok: OpenAIGPTConfig.transform_request returns an untyped dict + dict[str, object], + super().transform_request( # pyright: ignore[reportUnknownMemberType] # base signature uses bare dict + model, list(marked_messages), optional_params, litellm_params, headers + ), + ) + + def _messages_with_claude_cache_marks( + self, + model: str, + messages: Sequence[AllMessageValues], + optional_params: dict[str, object], # mutable-ok: the request body is built from this out-param + litellm_params: Mapping[str, object], + tools_carried_breakpoint: bool, + ) -> tuple[AllMessageValues, ...]: + """OpenAI ``prompt_cache_breakpoint`` marks as the ``cache_control`` Databricks forwards to Claude, held to + Claude's cap, with ``prompt_cache_options`` rejected or dropped. Tool definitions reach Databricks without + any mark, so only the message marks count toward the cap.""" + drop_params: Final = litellm.drop_params or litellm_params.get("drop_params") is True + reject_prompt_cache_options( + model=model, options=optional_params.pop("prompt_cache_options", None), drop_params=drop_params + ) + cache_marks: Final = resolved_claude_cache_marks( + claude_cache_marks( + messages=messages, + tools=(), + automatic_breakpoint=optional_params.get("cache_control") is not None, + drop_params=drop_params, + tools_carried_breakpoint=tools_carried_breakpoint, + ), + model=model, + llm_provider="databricks", + ) + return cache_marks.messages + def _should_fake_stream(self, optional_params: dict) -> bool: """ Databricks doesn't support 'response_format' while streaming diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index b11ae907cfb..0ad1277d56c 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -38,6 +38,7 @@ - {id: llm.chat_completions.bedrock_converse.batch_deployment.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: batch_deployment, streaming: nonstream, assertions: [works], source: "types/utils.py bedrock_batch_litellm_params", rationale: "A deployment carrying the documented batch-only S3 keys (s3_access_key_id, s3_secret_access_key, s3_encryption_key_id) must still serve ordinary chat; unregistered keys fall into optional_params and are forwarded as additionalModelRequestFields, which Bedrock 400s and which puts the S3 secret in the request body and debug log (LIT-8290)", fail_before_fix: proven} - {id: llm.chat_completions.bedrock_invoke.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "Regional inference-profile ids (us.anthropic.*) over the invoke route, the deployment shape behind a customer timeout report on v1.90.0"} - {id: llm.chat_completions.bedrock_invoke.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Streaming with regional inference-profile ids over the invoke route"} +- {id: llm.chat_completions.bedrock_invoke.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: bedrock_invoke, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "llms/anthropic/chat/transformation.py", rationale: "OpenAI-format chat to Bedrock Invoke builds the Anthropic request through AnthropicConfig.transform_request, so a prompt_cache_breakpoint must arrive as cache_control and write then read the cache"} - {id: llm.chat_completions.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/anthropic/chat/transformation.py", rationale: "OpenAI-format chat to Bedrock Invoke builds the Anthropic request through AnthropicConfig.transform_request, so flagged Claude 4.8+/5 must keep a mid-conversation role system reminder in messages; hoisting mutates the cached prefix and collapses the prompt cache (#36559)", fail_before_fix: proven} - {id: llm.chat_completions.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/anthropic/chat/transformation.py", rationale: "OpenAI-format chat to Bedrock Invoke: Claude <= 4.7 and Haiku 4.5 reject role system inside messages, so unflagged models must convert a mid-conversation reminder to a user turn in place (hoisting collapses the prompt cache) and still answer (#36559)", fail_before_fix: proven} - {id: llm.chat_completions.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Vertex AI"} diff --git a/tests/e2e/llm_translation/test_cache_control.py b/tests/e2e/llm_translation/test_cache_control.py index 12f707cf6f1..9bc3e7dc5b2 100644 --- a/tests/e2e/llm_translation/test_cache_control.py +++ b/tests/e2e/llm_translation/test_cache_control.py @@ -31,6 +31,13 @@ Each case asserts the feature actually happened, not just a 200. Coverage matrix system message, a user part or a tool result must write the cache on the first call and read it on the second. +- Claude (claude-haiku-5-5 on anthropic, Bedrock Converse and Bedrock Invoke, + claude-sonnet-5-5 on Vertex, which serves no haiku-5-5): OpenAI's ``prompt_cache_breakpoint`` on a system part, a string + system message, a user part, a tool result, a tool call or a tool definition + must write the cache on the first call and read it on the second. On + anthropic and Converse, a 30m breakpoint, five breakpoints and + ``prompt_cache_options`` under drop_params must still cache. + service_tier lives in test_provider_features_e2e.py. The provider-native cache_control request shape is not expressible with the @@ -40,16 +47,16 @@ built from the typed content blocks shared in ``models.py``. from __future__ import annotations +import os import time from collections.abc import Callable -from typing import Final +from typing import Final, Literal, TypeAlias import pytest -from pydantic import BaseModel, ConfigDict - +from _pytest.mark.structures import ParameterSet from e2e_config import unique_marker -from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from e2e_http import Result, UnknownApiError, unwrap +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ( CacheControl, @@ -60,7 +67,10 @@ from models import ( ChatTool, ChatToolFunction, ChatToolResultTurn, + ChatTurn, + ContentPart, LiteLLMParamsBody, + PromptCacheBreakpoint, PromptCacheOptions, RichMessage, TextBlock, @@ -70,7 +80,7 @@ from models import ( Usage, ) from passthrough_client import PassthroughClient -import os +from pydantic import BaseModel, ConfigDict pytestmark = [pytest.mark.e2e, pytest.mark.provider_live] @@ -551,3 +561,222 @@ class TestCacheControlOnOpenAIExplicitCaching: ], ) _assert_openai_writes_then_reads_cache(client, scoped_key, body) + + +CLAUDE_ANTHROPIC_MODEL: Final = "anthropic/claude-haiku-5-5" +CLAUDE_BEDROCK_CONVERSE_MODEL: Final = "bedrock/converse/us.anthropic.claude-haiku-5-5" +CLAUDE_BEDROCK_INVOKE_MODEL: Final = "bedrock/invoke/us.anthropic.claude-haiku-5-5" +CLAUDE_VERTEX_MODEL: Final = "vertex_ai/claude-sonnet-5-5" +ClaudeRoute: TypeAlias = Literal["anthropic", "bedrock_converse", "bedrock_invoke", "vertex"] +BreakpointSite: TypeAlias = Literal[ + "system_part", "user_part", "string_message", "tool_result_part", "tool_call", "tool_definition" +] +_LOOKUP_TOOL: Final = ChatToolFunction( + name="lookup_reference", description="Look up the reference material.", parameters={"type": "object"} +) + + +def _claude_route_param(route: ClaudeRoute) -> ParameterSet: + return pytest.param( + route, id=route, marks=pytest.mark.covers(f"llm.chat_completions.{route}.prompt_cache_5m.nonstream.works") + ) + + +def _claude_deployment(route: ClaudeRoute, *, drop_params: bool | None = None) -> LiteLLMParamsBody: + match route: + case "anthropic": + return LiteLLMParamsBody( + model=CLAUDE_ANTHROPIC_MODEL, api_key="os.environ/ANTHROPIC_API_KEY", drop_params=drop_params + ) + case "bedrock_converse": + return LiteLLMParamsBody( + model=CLAUDE_BEDROCK_CONVERSE_MODEL, aws_region_name="us-east-1", drop_params=drop_params + ) + case "bedrock_invoke": + return LiteLLMParamsBody( + model=CLAUDE_BEDROCK_INVOKE_MODEL, aws_region_name="us-east-1", drop_params=drop_params + ) + case "vertex": + return LiteLLMParamsBody( + model=CLAUDE_VERTEX_MODEL, + vertex_project="os.environ/VERTEXAI_PROJECT", + vertex_location="os.environ/VERTEXAI_LOCATION", + vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", + drop_params=drop_params, + ) + + +def _register_claude( + client: PassthroughClient, resources: ResourceManager, route: ClaudeRoute, *, drop_params: bool | None = None +) -> str: + name: Final = f"e2e-claude-breakpoint-{route}-{unique_marker()}" + model_id: Final = client.proxy.create_model(name, _claude_deployment(route, drop_params=drop_params)) + resources.defer(lambda: client.proxy.delete_model(model_id)) + return name + + +def _marked(text: str, ttl: Literal["30m"] | None = None) -> TextContentPart: + return TextContentPart(text=text, prompt_cache_breakpoint=PromptCacheBreakpoint(ttl=ttl)) + + +def _tool_turns(result: list[ContentPart] | str, *, marked_call: bool = False) -> list[ChatTurn]: + return [ + ChatAssistantTurn( + tool_calls=[ + ToolCall( + id="call_1", + type="function", + function=ToolCallFunction(name="lookup_reference", arguments="{}"), + prompt_cache_breakpoint=PromptCacheBreakpoint() if marked_call else None, + ) + ] + ), + ChatToolResultTurn(tool_call_id="call_1", content=result), + ] + + +def _breakpoint_messages(site: BreakpointSite, prefix: str) -> list[ChatTurn]: + ask: Final = ChatMessage(role="user", content="Reply with one word.") + match site: + case "system_part": + return [ChatMessage(role="system", content=[_marked(prefix)]), ask] + case "user_part": + return [ChatMessage(role="user", content=[_marked(prefix), TextContentPart(text="Reply with one word.")])] + case "string_message": + return [ChatMessage(role="system", content=prefix, prompt_cache_breakpoint=PromptCacheBreakpoint()), ask] + case "tool_result_part": + return [ChatMessage(role="user", content="Look up the reference."), *_tool_turns([_marked(prefix)])] + case "tool_call": + return [ChatMessage(role="user", content=prefix), *_tool_turns("Done.", marked_call=True)] + case "tool_definition": + return [ask] + + +def _breakpoint_tool(site: BreakpointSite, prefix: str) -> ChatTool: + if site != "tool_definition": + return ChatTool(function=_LOOKUP_TOOL) + return ChatTool( + function=ChatToolFunction(name="lookup_reference", description=prefix, parameters={"type": "object"}), + prompt_cache_breakpoint=PromptCacheBreakpoint(), + ) + + +def _breakpoint_body(model: str, site: BreakpointSite) -> ChatBody: + """A never-seen cacheable prefix whose one ``prompt_cache_breakpoint`` sits at ``site``.""" + prefix: Final = _cacheable_prefix() + return ChatBody( + model=model, + max_tokens=64, + tools=[_breakpoint_tool(site, prefix)], + messages=_breakpoint_messages(site, prefix), + ) + + +def _assert_claude_writes_then_reads_cache(client: PassthroughClient, key: str, body: ChatBody) -> None: + first: Final = unwrap(client.proxy.chat(key, body)) + assert first.usage and first.usage.cache_creation_input_tokens, ( + f"{body.model}: the first call on a never-seen prefix wrote no cache ({first.usage}), " + "so the prompt_cache_breakpoint never reached Claude as a cache mark" + ) + + deadline: Final = time.monotonic() + 30.0 + while True: + second = unwrap(client.proxy.chat(key, body)) + read_tokens = _cached_read_tokens(second.usage) + if read_tokens > 0 or time.monotonic() >= deadline: + break + time.sleep(3.0) + + assert read_tokens > 0, f"{body.model}: the identical second call read no cache ({second.usage})" + + +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC, Provider.BEDROCK, Provider.VERTEX_AI), + models=( + CLAUDE_ANTHROPIC_MODEL, + CLAUDE_BEDROCK_CONVERSE_MODEL, + CLAUDE_BEDROCK_INVOKE_MODEL, + CLAUDE_VERTEX_MODEL, + ), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) +class TestPromptCacheBreakpointOnClaude: + """OpenAI's ``prompt_cache_breakpoint`` sent to a Claude deployment caches the prefix it marks. + + The calls that leave drop_params off (more than four breakpoints, or ``prompt_cache_options``) answer 400 before + any provider call, so the wire tests in tests/integration prove them: the shared stack sets + ``litellm_settings.drop_params: true``, which no deployment can switch back off.""" + + @pytest.mark.parametrize( + "route", [_claude_route_param(route) for route in ("anthropic", "bedrock_converse", "bedrock_invoke", "vertex")] + ) + @pytest.mark.parametrize( + "site", ["system_part", "user_part", "string_message", "tool_result_part", "tool_call", "tool_definition"] + ) + def test_a_breakpoint_writes_then_reads_the_cache( + self, + client: PassthroughClient, + resources: ResourceManager, + scoped_key: str, + route: ClaudeRoute, + site: BreakpointSite, + ) -> None: + model: Final = _register_claude(client, resources, route) + _assert_claude_writes_then_reads_cache(client, scoped_key, _breakpoint_body(model, site)) + + @pytest.mark.parametrize("route", [_claude_route_param(route) for route in ("anthropic", "bedrock_converse")]) + def test_a_30m_breakpoint_still_caches( + self, client: PassthroughClient, resources: ResourceManager, scoped_key: str, route: ClaudeRoute + ) -> None: + model: Final = _register_claude(client, resources, route) + body: Final = ChatBody( + model=model, + max_tokens=64, + messages=[ + ChatMessage(role="system", content=[_marked(_cacheable_prefix(), ttl="30m")]), + ChatMessage(role="user", content="Reply with one word."), + ], + ) + _assert_claude_writes_then_reads_cache(client, scoped_key, body) + + @pytest.mark.parametrize("route", [_claude_route_param(route) for route in ("anthropic", "bedrock_converse")]) + def test_five_breakpoints_keep_the_latest_four_under_drop_params( + self, client: PassthroughClient, resources: ResourceManager, scoped_key: str, route: ClaudeRoute + ) -> None: + model: Final = _register_claude(client, resources, route, drop_params=True) + body: Final = ChatBody( + model=model, + max_tokens=64, + messages=[ + ChatMessage( + role="user", + content=[ + _marked(_cacheable_prefix()), + *(_marked(f"Note {index}.") for index in range(4)), + TextContentPart(text="Reply with one word."), + ], + ) + ], + ) + _assert_claude_writes_then_reads_cache(client, scoped_key, body) + + @pytest.mark.parametrize("route", [_claude_route_param(route) for route in ("anthropic", "bedrock_converse")]) + def test_prompt_cache_options_are_dropped_under_drop_params( + self, client: PassthroughClient, resources: ResourceManager, scoped_key: str, route: ClaudeRoute + ) -> None: + model: Final = _register_claude(client, resources, route, drop_params=True) + body: Final = ChatBody( + model=model, + max_tokens=64, + prompt_cache_options=PromptCacheOptions(mode="explicit"), + messages=[ + ChatMessage(role="system", content=[_marked(_cacheable_prefix())]), + ChatMessage(role="user", content="Reply with one word."), + ], + ) + _assert_claude_writes_then_reads_cache(client, scoped_key, body) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 5259166e9a9..1635ab2e890 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -216,10 +216,18 @@ class ImageUrl(BaseModel): url: str +class PromptCacheBreakpoint(BaseModel): + """OpenAI's explicit cache marker on a content part, message, tool definition or tool call.""" + + mode: Literal["explicit"] = "explicit" + ttl: Literal["30m"] | None = None + + class TextContentPart(BaseModel): type: str = "text" text: str cache_control: CacheControl | None = None + prompt_cache_breakpoint: PromptCacheBreakpoint | None = None class ImageContentPart(BaseModel): @@ -234,6 +242,7 @@ class ChatMessage(BaseModel): role: str content: str | list[ContentPart] cache_control: CacheControl | None = None + prompt_cache_breakpoint: PromptCacheBreakpoint | None = None class CacheControl(BaseModel): @@ -271,6 +280,7 @@ class ChatToolFunction(BaseModel): class ChatTool(BaseModel): type: str = "function" function: ChatToolFunction + prompt_cache_breakpoint: PromptCacheBreakpoint | None = None class AnthropicToolBody(BaseModel, frozen=True): @@ -318,6 +328,7 @@ class ToolCall(BaseModel): type: str | None = None function: ToolCallFunction = ToolCallFunction() cache_control: CacheControl | None = None + prompt_cache_breakpoint: PromptCacheBreakpoint | None = None class ThinkingBlock(BaseModel): @@ -363,6 +374,7 @@ class ChatBody(BaseModel): thinking: ThinkingParam | None = None service_tier: str | None = None prompt_cache_key: str | None = None + prompt_cache_options: PromptCacheOptions | None = None tools: Sequence[ChatTool | McpChatTool | GeminiUrlContextTool] | None = None tool_choice: str | None = None guardrails: list[str] | None = None diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index d8ea793f2ca..9bbb4628f6a 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -8879,7 +8879,10 @@ async def test_a_cache_control_the_client_wrote_wins_over_the_breakpoint_on_the_ assert "prompt_cache_breakpoint" not in request.content.decode() assert json.loads(request.content)["messages"] == [ - {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"}}]} + { + "role": "user", + "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"}}], + } ] @@ -9006,3 +9009,88 @@ async def test_five_native_cache_control_marks_are_forwarded_and_the_provider_an assert route.call_count == 1 assert json.loads(route.calls.last.request.content)["messages"] == [{"role": "user", "content": marked}] + + +async def test_a_breakpoint_only_on_a_tool_definition_still_holds_the_request_to_four_marks( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + marked: Final = [{"type": "text", "text": f"part {index}", "cache_control": EPHEMERAL} for index in range(4)] + request: Final = await _send_prompt_caching_request( + respx_mock, + monkeypatch, + messages=[{"role": "user", "content": marked}], + tools=[{"type": "function", "function": _WEATHER_FUNCTION, "prompt_cache_breakpoint": EXPLICIT}], + drop_params=True, + ) + + body: Final = json.loads(request.content) + assert body["tools"] == [_ANTHROPIC_WEATHER_TOOL] + assert body["messages"] == [{"role": "user", "content": marked}] + + +async def test_the_automatic_cache_control_takes_one_of_the_four_slots( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + request: Final = await _send_prompt_caching_request( + respx_mock, + monkeypatch, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": EXPLICIT} for index in range(4) + ], + } + ], + cache_control=EPHEMERAL, + drop_params=True, + ) + + body: Final = json.loads(request.content) + assert body["cache_control"] == EPHEMERAL + assert body["messages"] == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "part 0"}, + *({"type": "text", "text": f"part {index}", "cache_control": EPHEMERAL} for index in range(1, 4)), + ], + } + ] + + +async def test_a_message_level_breakpoint_on_list_content_does_not_count_toward_the_cap( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + request: Final = await _send_prompt_caching_request( + respx_mock, + monkeypatch, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": EXPLICIT} for index in range(4) + ], + "prompt_cache_breakpoint": EXPLICIT, + } + ], + ) + + assert json.loads(request.content)["messages"] == [ + { + "role": "user", + "content": [{"type": "text", "text": f"part {index}", "cache_control": EPHEMERAL} for index in range(4)], + } + ] + + +def test_map_openai_params_maps_a_tuple_of_tools() -> None: + optional_params: Final = AnthropicConfig().map_openai_params( + non_default_params={"tools": ({"type": "function", "function": _WEATHER_FUNCTION},)}, + optional_params={}, + model=PROMPT_CACHING_MODEL, + drop_params=False, + ) + + assert optional_params["tools"] == [_ANTHROPIC_WEATHER_TOOL] diff --git a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py index afb451a4999..b77c25a9e87 100644 --- a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py +++ b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py @@ -116,6 +116,44 @@ def test_a_request_without_breakpoints_is_forwarded_as_written_even_past_the_cap assert marks.messages[0] is messages[0] +def test_a_message_level_mark_on_list_content_takes_no_slot_because_claude_never_renders_it() -> None: + marked_parts: Final = [_text(f"part {index}", prompt_cache_breakpoint=EXPLICIT) for index in range(3)] + messages: Final = [ + {"role": "user", "content": "first", "prompt_cache_breakpoint": EXPLICIT}, + {"role": "user", "content": [*marked_parts, _text("tail")], "prompt_cache_breakpoint": EXPLICIT}, + ] + + marks: Final = claude_cache_marks(messages, None, automatic_breakpoint=False, drop_params=False) + + assert marks == ClaudeCacheMarks( + messages=( + {"role": "user", "content": "first", "cache_control": EPHEMERAL}, + { + "role": "user", + "content": [*(_text(f"part {index}", cache_control=EPHEMERAL) for index in range(3)), _text("tail")], + "cache_control": EPHEMERAL, + }, + ), + tools=(), + ) + + +def test_a_message_level_mark_on_a_tool_result_with_list_content_takes_a_slot() -> None: + messages: Final = [ + {"role": "user", "content": [_text(f"part {index}", prompt_cache_breakpoint=EXPLICIT) for index in range(4)]}, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [_text("sunny")], + "prompt_cache_breakpoint": EXPLICIT, + }, + ] + + assert claude_cache_marks(messages, None, automatic_breakpoint=False, drop_params=False) == ( + CacheBreakpointCapExceeded(found=5) + ) + + def test_tools_already_mapped_to_the_anthropic_shape_still_arm_the_cap() -> None: mapped_tools: Final = [ {"name": f"tool_{index}", "input_schema": {}, "cache_control": EPHEMERAL} for index in range(4) @@ -165,11 +203,12 @@ def test_prompt_cache_options_are_rejected_unless_drop_params() -> None: [ ([_tool(prompt_cache_breakpoint=EXPLICIT)], True), ([{"type": "function", "function": {"name": "lookup", "prompt_cache_breakpoint": EXPLICIT}}], True), + ((_tool(prompt_cache_breakpoint=EXPLICIT),), True), ([_tool(cache_control=EPHEMERAL), _tool()], False), (None, False), ("not a list", False), ], - ids=["top-level", "under-function", "cache-control-only", "none", "opaque"], + ids=["top-level", "under-function", "tuple", "cache-control-only", "none", "opaque"], ) def test_tools_carry_prompt_cache_breakpoint_reads_both_tool_shapes(tools: object, expected: bool) -> None: assert tools_carry_prompt_cache_breakpoint(tools) is expected diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index f062e4fbd1f..b2399b9a52b 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -1393,9 +1393,10 @@ class TestBedrockAnthropicPromptCachingRegression: {"type": "text", "text": "What are the payment terms?"}, ] + @pytest.mark.parametrize("drop_params_source", ["litellm_params", "global"]) @pytest.mark.parametrize("model_prefix", ["bedrock/invoke/", "bedrock/converse/"]) def test_five_breakpoints_are_cut_to_four_only_under_drop_params_for_both_routes( - self, model_prefix: str, monkeypatch: pytest.MonkeyPatch + self, model_prefix: str, drop_params_source: str, monkeypatch: pytest.MonkeyPatch ) -> None: import litellm @@ -1415,11 +1416,12 @@ class TestBedrockAnthropicPromptCachingRegression: headers={}, ) + monkeypatch.setattr(litellm, "drop_params", drop_params_source == "global") result: Final = config.transform_request( model="anthropic.claude-opus-5-5", messages=[{"role": "user", "content": marked}], optional_params={}, - litellm_params={"drop_params": True}, + litellm_params={"drop_params": True} if drop_params_source == "litellm_params" else {}, headers={}, ) if "converse" in model_prefix: @@ -1434,7 +1436,10 @@ class TestBedrockAnthropicPromptCachingRegression: else: assert result["messages"][0]["content"] == [ {"type": "text", "text": "part 0"}, - *({"type": "text", "text": f"part {index}", "cache_control": {"type": "ephemeral"}} for index in range(1, 5)), + *( + {"type": "text", "text": f"part {index}", "cache_control": {"type": "ephemeral"}} + for index in range(1, 5) + ), ] @pytest.mark.parametrize( @@ -1506,6 +1511,7 @@ class TestBedrockAnthropicPromptCachingRegression: if "anthropic_beta" in additional_fields: assert "prompt-caching-2024-07-31" not in additional_fields["anthropic_beta"] + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBedrockAnthropic1MContextRegression: """ @@ -1691,6 +1697,7 @@ class TestBedrockAnthropic1MContextRegression: assert "context-1m-2025-08-07" in beta_headers assert "computer-use-2024-10-22" in beta_headers + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBedrockAnthropicCombinedRegressions: """ diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 111a6d4727d..298d17657b7 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -10184,7 +10184,11 @@ def _breakpoint_conversation() -> list[dict[str, object]]: } ], }, - {"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "sunny", "prompt_cache_breakpoint": _EXPLICIT}]}, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "text", "text": "sunny", "prompt_cache_breakpoint": _EXPLICIT}], + }, {"role": "user", "content": "done", "prompt_cache_breakpoint": _EXPLICIT}, ] @@ -10241,11 +10245,19 @@ async def test_prompt_cache_breakpoints_become_cache_points_after_the_same_block optional_params: Final[dict[str, object]] = {"tools": [{**_LOOKUP_TOOL, "prompt_cache_breakpoint": _EXPLICIT}]} if sync_mode: result = config._transform_request( - model=_BREAKPOINT_MODEL, messages=conversation[2:], optional_params=optional_params, litellm_params={}, headers={} + model=_BREAKPOINT_MODEL, + messages=conversation[2:], + optional_params=optional_params, + litellm_params={}, + headers={}, ) else: result = await config._async_transform_request( - model=_BREAKPOINT_MODEL, messages=conversation[2:], optional_params=optional_params, litellm_params={}, headers={} + model=_BREAKPOINT_MODEL, + messages=conversation[2:], + optional_params=optional_params, + litellm_params={}, + headers={}, ) assert "prompt_cache_breakpoint" not in json.dumps(result) @@ -10253,13 +10265,17 @@ async def test_prompt_cache_breakpoints_become_cache_points_after_the_same_block assert result["messages"] == _expected_latest_four()["messages"][1:] +@pytest.mark.parametrize("drop_params_source", ["litellm_params", "global"]) @pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"]) @pytest.mark.asyncio -async def test_six_breakpoints_keep_the_latest_four_cache_points_under_drop_params(sync_mode: bool) -> None: +async def test_six_breakpoints_keep_the_latest_four_cache_points_under_drop_params( + sync_mode: bool, drop_params_source: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_params_source == "global") result: Final = await _breakpoint_request( sync_mode, {"tools": [{**_LOOKUP_TOOL, "prompt_cache_breakpoint": _EXPLICIT}]}, - {"drop_params": True}, + {"drop_params": True} if drop_params_source == "litellm_params" else {}, ) assert json.dumps(result).count("cachePoint") == 4 @@ -10268,27 +10284,69 @@ async def test_six_breakpoints_keep_the_latest_four_cache_points_under_drop_para @pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"]) @pytest.mark.asyncio -async def test_six_breakpoints_are_a_bad_request_without_drop_params(sync_mode: bool, monkeypatch: pytest.MonkeyPatch) -> None: +async def test_six_breakpoints_are_a_bad_request_without_drop_params( + sync_mode: bool, monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.setattr(litellm, "drop_params", False) with pytest.raises(litellm.BadRequestError, match=r"Claude accepts at most 4 cache breakpoints, found 6"): await _breakpoint_request(sync_mode, {"tools": [{**_LOOKUP_TOOL, "prompt_cache_breakpoint": _EXPLICIT}]}, {}) +@pytest.mark.parametrize("emits_cache_points", [True, False], ids=["cache-points", "no-cache-points"]) @pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"]) @pytest.mark.asyncio -async def test_prompt_cache_options_are_rejected_unless_drop_params(sync_mode: bool, monkeypatch: pytest.MonkeyPatch) -> None: +async def test_prompt_cache_options_are_rejected_unless_drop_params( + sync_mode: bool, emits_cache_points: bool, monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.setattr(litellm, "drop_params", False) + if not emits_cache_points: + monkeypatch.setitem( + litellm.model_cost, + _BREAKPOINT_MODEL, + {"litellm_provider": "bedrock", "mode": "chat", "supports_prompt_cache_breakpoint": False}, + ) with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"): await _breakpoint_request(sync_mode, {"prompt_cache_options": {"mode": "explicit"}}, {}) - dropped: Final = await _breakpoint_request(sync_mode, {"prompt_cache_options": {"mode": "explicit"}}, {"drop_params": True}) + dropped: Final = await _breakpoint_request( + sync_mode, {"prompt_cache_options": {"mode": "explicit"}}, {"drop_params": True} + ) assert "prompt_cache_options" not in json.dumps(dropped) + monkeypatch.setattr(litellm, "drop_params", True) + dropped_globally: Final = await _breakpoint_request(sync_mode, {"prompt_cache_options": {"mode": "explicit"}}, {}) + assert "prompt_cache_options" not in json.dumps(dropped_globally) + + +def test_the_automatic_cache_control_takes_one_of_the_four_cache_points() -> None: + marked: Final = [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": _EXPLICIT} for index in range(4) + ] + + result: Final = AmazonConverseConfig()._transform_request( + model=_BREAKPOINT_MODEL, + messages=[{"role": "user", "content": marked}], + optional_params={"cache_control": {"type": "ephemeral"}}, + litellm_params={"drop_params": True}, + headers={}, + ) + + assert result["messages"] == [ + { + "role": "user", + "content": [ + {"text": "part 0"}, + *(block for index in range(1, 4) for block in ({"text": f"part {index}"}, _CACHE_POINT)), + ], + } + ] def test_five_native_cache_control_marks_are_forwarded_as_written_past_the_cap() -> None: - marked: Final = [{"type": "text", "text": f"part {index}", "cache_control": {"type": "ephemeral"}} for index in range(5)] + marked: Final = [ + {"type": "text", "text": f"part {index}", "cache_control": {"type": "ephemeral"}} for index in range(5) + ] result: Final = AmazonConverseConfig()._transform_request( model=_BREAKPOINT_MODEL, diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index fe76a667f3e..8c80b852cec 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -2255,3 +2255,79 @@ def test_transform_response_carries_served_service_tier(served_tier: str | None) ) assert result.to_dict().get("service_tier") == served_tier + + +_EXPLICIT: Final = {"mode": "explicit"} +_EPHEMERAL: Final = {"type": "ephemeral"} +_LOOKUP_FUNCTION: Final = {"name": "lookup", "parameters": {"type": "object", "properties": {}}} + + +def _databricks_claude_route(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> respx.Route: + monkeypatch.setenv("DATABRICKS_API_BASE", DATABRICKS_API_BASE) + monkeypatch.setenv("DATABRICKS_API_KEY", DATABRICKS_API_KEY) + monkeypatch.setattr(litellm, "drop_params", False) + return respx_mock.post(DATABRICKS_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response( + 200, json=_databricks_anthropic_cache_response(cache_read_input_tokens=0, cache_creation_input_tokens=0) + ) + ) + + +def test_prompt_cache_breakpoints_reach_databricks_claude_as_cache_control( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + route: Final = _databricks_claude_route(respx_mock, monkeypatch) + + litellm.completion( + model="databricks/databricks-claude-haiku-4-5", + messages=[ + {"role": "system", "content": [{"type": "text", "text": "rules", "prompt_cache_breakpoint": _EXPLICIT}]}, + {"role": "user", "content": "hi", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}}, + ], + ) + + assert json.loads(route.calls.last.request.content)["messages"] == [ + {"role": "system", "content": [{"type": "text", "text": "rules", "cache_control": _EPHEMERAL}]}, + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _EPHEMERAL}]}, + ] + + +def test_a_tool_breakpoint_holds_databricks_claude_to_four_marks_under_drop_params( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + route: Final = _databricks_claude_route(respx_mock, monkeypatch) + parts: Final = [{"type": "text", "text": f"part {index}", "cache_control": _EPHEMERAL} for index in range(5)] + request: Final = { + "model": "databricks/databricks-claude-haiku-4-5", + "messages": [{"role": "user", "content": parts}], + "tools": [{"type": "function", "function": _LOOKUP_FUNCTION, "prompt_cache_breakpoint": _EXPLICIT}], + } + + with pytest.raises(litellm.BadRequestError, match=r"Claude accepts at most 4 cache breakpoints, found 5"): + litellm.completion(**request) + assert route.call_count == 0 + + litellm.completion(**request, drop_params=True) + + assert json.loads(route.calls.last.request.content)["messages"] == [ + {"role": "user", "content": [{"type": "text", "text": "part 0"}, *parts[1:]]} + ] + + +def test_prompt_cache_options_toward_databricks_claude_are_rejected_unless_drop_params( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + route: Final = _databricks_claude_route(respx_mock, monkeypatch) + request: Final = { + "model": "databricks/databricks-claude-haiku-4-5", + "messages": [{"role": "user", "content": "hi"}], + "prompt_cache_options": {"mode": "explicit"}, + } + + with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"): + litellm.completion(**request) + assert route.call_count == 0 + + litellm.completion(**request, drop_params=True) + + assert "prompt_cache_options" not in json.loads(route.calls.last.request.content)