mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
fix(anthropic): close review gaps in prompt_cache_breakpoint to cache_control for Claude
Converse now rejects or drops prompt_cache_options for every Claude model, not only the ones that emit cache points, and honors the global drop_params flag Databricks Claude deployments translate prompt_cache_breakpoint marks to cache_control, hold them to Claude's four-mark cap, and reject prompt_cache_options unless drop_params is on Tuple tool lists are accepted, and a message-level mark on list content no longer takes one of the four slots because Claude never renders it The e2e suite gains a Claude class that writes then reads the cache across anthropic, Bedrock Converse, Bedrock Invoke and Vertex for every breakpoint site
This commit is contained in:
ryan-crabbe-berri
parent
e671037110
commit
3acf5f514a
12 changed files with 607 additions and 25 deletions
|
|
@ -1827,7 +1827,7 @@ def _as_mapping(value: object) -> Mapping[str, object] | None:
|
|||
|
||||
|
||||
def _as_list(value: object) -> Sequence[object] | None:
|
||||
return value if isinstance(value, list) else None # pyright: ignore[reportUnknownVariableType] # items are parsed one by one
|
||||
return value if isinstance(value, (list, tuple)) else None # pyright: ignore[reportUnknownVariableType] # items are parsed one by one
|
||||
|
||||
|
||||
def message_fields(message: AllMessageValues) -> Mapping[str, object]:
|
||||
|
|
@ -1930,7 +1930,7 @@ def _tool_block_with_prompt_cache_breakpoint_as_cache_control(tool: object) -> o
|
|||
|
||||
|
||||
def tools_with_prompt_cache_breakpoints_as_cache_control(tools: object) -> tuple[object, ...]:
|
||||
"""Every tool definition in ``tools`` (anything but a list is treated as none) with its mark renamed."""
|
||||
"""Every tool definition in ``tools`` (anything but a list or tuple is treated as none) with its mark renamed."""
|
||||
return tuple(_tool_block_with_prompt_cache_breakpoint_as_cache_control(tool) for tool in _as_list(tools) or ())
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ def _as_mapping(value: object) -> Mapping[str, object] | None:
|
|||
|
||||
|
||||
def _as_blocks(value: object) -> Sequence[object] | None:
|
||||
return value if isinstance(value, list) else None # pyright: ignore[reportUnknownVariableType] # blocks are parsed one by one
|
||||
return value if isinstance(value, (list, tuple)) else None # pyright: ignore[reportUnknownVariableType] # blocks are parsed one by one
|
||||
|
||||
|
||||
def _carries(block: object, key: str) -> bool:
|
||||
|
|
@ -56,11 +56,16 @@ def tools_carry_prompt_cache_breakpoint(tools: object) -> bool:
|
|||
return any(_tool_carries(tool, "prompt_cache_breakpoint") for tool in _as_blocks(tools) or ())
|
||||
|
||||
|
||||
def _renders_message_mark(message: Mapping[str, object]) -> bool:
|
||||
"""Claude's message builder keeps a message-level mark on string content and on tool results only."""
|
||||
return isinstance(message.get("content"), str) or message.get("role") in ("tool", "function")
|
||||
|
||||
|
||||
def _message_sites(index: int, message: object, key: str) -> Iterator[_Site]:
|
||||
fields: Final = _as_mapping(message)
|
||||
if fields is None:
|
||||
return
|
||||
if fields.get(key) is not None:
|
||||
if fields.get(key) is not None and _renders_message_mark(fields):
|
||||
yield ("message", index, -1)
|
||||
for position, part in enumerate(_as_blocks(fields.get("content")) or ()):
|
||||
if _carries(part, key):
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ from litellm.llms.anthropic.chat.transformation import (
|
|||
)
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_images
|
||||
from litellm.llms.bedrock.common_utils import bedrock_model_supports_regex_lookaround
|
||||
from litellm.llms.bedrock.request_metadata import (
|
||||
|
|
@ -2026,13 +2027,17 @@ class AmazonConverseConfig(BaseConfig):
|
|||
) -> tuple[AllMessageValues, ...]:
|
||||
"""OpenAI ``prompt_cache_breakpoint`` marks as ``cache_control`` so the factory emits ``cachePoint`` blocks,
|
||||
held to the provider's cap, with ``prompt_cache_options`` rejected or dropped. Models that never emit
|
||||
``cachePoint`` blocks are left as written."""
|
||||
if not bedrock_model_accepts_cache_points(model):
|
||||
``cachePoint`` blocks keep their marks as written, but a Claude model among them still has
|
||||
``prompt_cache_options`` rejected or dropped."""
|
||||
emits_cache_points: Final = bedrock_model_accepts_cache_points(model)
|
||||
if not emits_cache_points and BaseAWSLLM.get_bedrock_invoke_provider(model) != "anthropic":
|
||||
return tuple(messages)
|
||||
drop_params: Final = litellm.drop_params or litellm_params.get("drop_params") is True
|
||||
reject_prompt_cache_options(
|
||||
model=model, options=optional_params.pop("prompt_cache_options", None), drop_params=drop_params
|
||||
)
|
||||
if not emits_cache_points:
|
||||
return tuple(messages)
|
||||
cache_marks: Final = resolved_claude_cache_marks(
|
||||
claude_cache_marks(
|
||||
messages=messages,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload
|
|||
import httpx
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
handle_invalid_parallel_tool_calls,
|
||||
|
|
@ -55,6 +56,13 @@ from litellm.types.utils import (
|
|||
Usage,
|
||||
)
|
||||
|
||||
from ...anthropic.chat.prompt_cache_breakpoints import (
|
||||
PROMPT_CACHE_BREAKPOINT_ON_TOOLS,
|
||||
claude_cache_marks,
|
||||
reject_prompt_cache_options,
|
||||
resolved_claude_cache_marks,
|
||||
tools_carry_prompt_cache_breakpoint,
|
||||
)
|
||||
from ...anthropic.chat.transformation import (
|
||||
REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT,
|
||||
AnthropicConfig,
|
||||
|
|
@ -375,6 +383,9 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
|
|||
)
|
||||
if "tools" in mapped_params:
|
||||
mapped_params["tools"] = self._map_openai_to_dbrx_tool(model=model, tools=mapped_params["tools"])
|
||||
requested_params: Final = cast(Mapping[str, object], non_default_params) # cast-ok: base signature is untyped
|
||||
if "claude" in model and tools_carry_prompt_cache_breakpoint(requested_params.get("tools")):
|
||||
mapped_params[PROMPT_CACHE_BREAKPOINT_ON_TOOLS] = True
|
||||
if "max_completion_tokens" in non_default_params and replace_max_completion_tokens_with_max_tokens:
|
||||
mapped_params["max_tokens"] = non_default_params[
|
||||
"max_completion_tokens"
|
||||
|
|
@ -434,6 +445,57 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
|
|||
|
||||
return mapped_params
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: list[AllMessageValues], # mutable-ok: overrides BaseConfig.transform_request
|
||||
optional_params: dict[str, object], # mutable-ok: overrides BaseConfig.transform_request
|
||||
litellm_params: dict[str, object], # mutable-ok: overrides BaseConfig.transform_request
|
||||
headers: dict[str, object], # mutable-ok: overrides BaseConfig.transform_request
|
||||
) -> dict[str, object]: # mutable-ok: overrides BaseConfig.transform_request
|
||||
tools_carried_breakpoint: Final = bool(optional_params.pop(PROMPT_CACHE_BREAKPOINT_ON_TOOLS, False))
|
||||
marked_messages: Final = (
|
||||
self._messages_with_claude_cache_marks(
|
||||
model, messages, optional_params, litellm_params, tools_carried_breakpoint
|
||||
)
|
||||
if "claude" in model
|
||||
else messages
|
||||
)
|
||||
return cast( # cast-ok: OpenAIGPTConfig.transform_request returns an untyped dict
|
||||
dict[str, object],
|
||||
super().transform_request( # pyright: ignore[reportUnknownMemberType] # base signature uses bare dict
|
||||
model, list(marked_messages), optional_params, litellm_params, headers
|
||||
),
|
||||
)
|
||||
|
||||
def _messages_with_claude_cache_marks(
|
||||
self,
|
||||
model: str,
|
||||
messages: Sequence[AllMessageValues],
|
||||
optional_params: dict[str, object], # mutable-ok: the request body is built from this out-param
|
||||
litellm_params: Mapping[str, object],
|
||||
tools_carried_breakpoint: bool,
|
||||
) -> tuple[AllMessageValues, ...]:
|
||||
"""OpenAI ``prompt_cache_breakpoint`` marks as the ``cache_control`` Databricks forwards to Claude, held to
|
||||
Claude's cap, with ``prompt_cache_options`` rejected or dropped. Tool definitions reach Databricks without
|
||||
any mark, so only the message marks count toward the cap."""
|
||||
drop_params: Final = litellm.drop_params or litellm_params.get("drop_params") is True
|
||||
reject_prompt_cache_options(
|
||||
model=model, options=optional_params.pop("prompt_cache_options", None), drop_params=drop_params
|
||||
)
|
||||
cache_marks: Final = resolved_claude_cache_marks(
|
||||
claude_cache_marks(
|
||||
messages=messages,
|
||||
tools=(),
|
||||
automatic_breakpoint=optional_params.get("cache_control") is not None,
|
||||
drop_params=drop_params,
|
||||
tools_carried_breakpoint=tools_carried_breakpoint,
|
||||
),
|
||||
model=model,
|
||||
llm_provider="databricks",
|
||||
)
|
||||
return cache_marks.messages
|
||||
|
||||
def _should_fake_stream(self, optional_params: dict) -> bool:
|
||||
"""
|
||||
Databricks doesn't support 'response_format' while streaming
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@
|
|||
- {id: llm.chat_completions.bedrock_converse.batch_deployment.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: batch_deployment, streaming: nonstream, assertions: [works], source: "types/utils.py bedrock_batch_litellm_params", rationale: "A deployment carrying the documented batch-only S3 keys (s3_access_key_id, s3_secret_access_key, s3_encryption_key_id) must still serve ordinary chat; unregistered keys fall into optional_params and are forwarded as additionalModelRequestFields, which Bedrock 400s and which puts the S3 secret in the request body and debug log (LIT-8290)", fail_before_fix: proven}
|
||||
- {id: llm.chat_completions.bedrock_invoke.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "Regional inference-profile ids (us.anthropic.*) over the invoke route, the deployment shape behind a customer timeout report on v1.90.0"}
|
||||
- {id: llm.chat_completions.bedrock_invoke.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Streaming with regional inference-profile ids over the invoke route"}
|
||||
- {id: llm.chat_completions.bedrock_invoke.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: bedrock_invoke, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "llms/anthropic/chat/transformation.py", rationale: "OpenAI-format chat to Bedrock Invoke builds the Anthropic request through AnthropicConfig.transform_request, so a prompt_cache_breakpoint must arrive as cache_control and write then read the cache"}
|
||||
- {id: llm.chat_completions.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/anthropic/chat/transformation.py", rationale: "OpenAI-format chat to Bedrock Invoke builds the Anthropic request through AnthropicConfig.transform_request, so flagged Claude 4.8+/5 must keep a mid-conversation role system reminder in messages; hoisting mutates the cached prefix and collapses the prompt cache (#36559)", fail_before_fix: proven}
|
||||
- {id: llm.chat_completions.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/anthropic/chat/transformation.py", rationale: "OpenAI-format chat to Bedrock Invoke: Claude <= 4.7 and Haiku 4.5 reject role system inside messages, so unflagged models must convert a mid-conversation reminder to a user turn in place (hoisting collapses the prompt cache) and still answer (#36559)", fail_before_fix: proven}
|
||||
- {id: llm.chat_completions.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Vertex AI"}
|
||||
|
|
|
|||
|
|
@ -31,6 +31,13 @@ Each case asserts the feature actually happened, not just a 200. Coverage matrix
|
|||
system message, a user part or a tool result must write the cache on the
|
||||
first call and read it on the second.
|
||||
|
||||
- Claude (claude-haiku-5-5 on anthropic, Bedrock Converse and Bedrock Invoke,
|
||||
claude-sonnet-5-5 on Vertex, which serves no haiku-5-5): OpenAI's ``prompt_cache_breakpoint`` on a system part, a string
|
||||
system message, a user part, a tool result, a tool call or a tool definition
|
||||
must write the cache on the first call and read it on the second. On
|
||||
anthropic and Converse, a 30m breakpoint, five breakpoints and
|
||||
``prompt_cache_options`` under drop_params must still cache.
|
||||
|
||||
service_tier lives in test_provider_features_e2e.py.
|
||||
|
||||
The provider-native cache_control request shape is not expressible with the
|
||||
|
|
@ -40,16 +47,16 @@ built from the typed content blocks shared in ``models.py``.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from _pytest.mark.structures import ParameterSet
|
||||
from e2e_config import unique_marker
|
||||
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
|
||||
from e2e_http import Result, UnknownApiError, unwrap
|
||||
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import (
|
||||
CacheControl,
|
||||
|
|
@ -60,7 +67,10 @@ from models import (
|
|||
ChatTool,
|
||||
ChatToolFunction,
|
||||
ChatToolResultTurn,
|
||||
ChatTurn,
|
||||
ContentPart,
|
||||
LiteLLMParamsBody,
|
||||
PromptCacheBreakpoint,
|
||||
PromptCacheOptions,
|
||||
RichMessage,
|
||||
TextBlock,
|
||||
|
|
@ -70,7 +80,7 @@ from models import (
|
|||
Usage,
|
||||
)
|
||||
from passthrough_client import PassthroughClient
|
||||
import os
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
pytestmark = [pytest.mark.e2e, pytest.mark.provider_live]
|
||||
|
||||
|
|
@ -551,3 +561,222 @@ class TestCacheControlOnOpenAIExplicitCaching:
|
|||
],
|
||||
)
|
||||
_assert_openai_writes_then_reads_cache(client, scoped_key, body)
|
||||
|
||||
|
||||
CLAUDE_ANTHROPIC_MODEL: Final = "anthropic/claude-haiku-5-5"
|
||||
CLAUDE_BEDROCK_CONVERSE_MODEL: Final = "bedrock/converse/us.anthropic.claude-haiku-5-5"
|
||||
CLAUDE_BEDROCK_INVOKE_MODEL: Final = "bedrock/invoke/us.anthropic.claude-haiku-5-5"
|
||||
CLAUDE_VERTEX_MODEL: Final = "vertex_ai/claude-sonnet-5-5"
|
||||
ClaudeRoute: TypeAlias = Literal["anthropic", "bedrock_converse", "bedrock_invoke", "vertex"]
|
||||
BreakpointSite: TypeAlias = Literal[
|
||||
"system_part", "user_part", "string_message", "tool_result_part", "tool_call", "tool_definition"
|
||||
]
|
||||
_LOOKUP_TOOL: Final = ChatToolFunction(
|
||||
name="lookup_reference", description="Look up the reference material.", parameters={"type": "object"}
|
||||
)
|
||||
|
||||
|
||||
def _claude_route_param(route: ClaudeRoute) -> ParameterSet:
|
||||
return pytest.param(
|
||||
route, id=route, marks=pytest.mark.covers(f"llm.chat_completions.{route}.prompt_cache_5m.nonstream.works")
|
||||
)
|
||||
|
||||
|
||||
def _claude_deployment(route: ClaudeRoute, *, drop_params: bool | None = None) -> LiteLLMParamsBody:
|
||||
match route:
|
||||
case "anthropic":
|
||||
return LiteLLMParamsBody(
|
||||
model=CLAUDE_ANTHROPIC_MODEL, api_key="os.environ/ANTHROPIC_API_KEY", drop_params=drop_params
|
||||
)
|
||||
case "bedrock_converse":
|
||||
return LiteLLMParamsBody(
|
||||
model=CLAUDE_BEDROCK_CONVERSE_MODEL, aws_region_name="us-east-1", drop_params=drop_params
|
||||
)
|
||||
case "bedrock_invoke":
|
||||
return LiteLLMParamsBody(
|
||||
model=CLAUDE_BEDROCK_INVOKE_MODEL, aws_region_name="us-east-1", drop_params=drop_params
|
||||
)
|
||||
case "vertex":
|
||||
return LiteLLMParamsBody(
|
||||
model=CLAUDE_VERTEX_MODEL,
|
||||
vertex_project="os.environ/VERTEXAI_PROJECT",
|
||||
vertex_location="os.environ/VERTEXAI_LOCATION",
|
||||
vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
||||
|
||||
def _register_claude(
|
||||
client: PassthroughClient, resources: ResourceManager, route: ClaudeRoute, *, drop_params: bool | None = None
|
||||
) -> str:
|
||||
name: Final = f"e2e-claude-breakpoint-{route}-{unique_marker()}"
|
||||
model_id: Final = client.proxy.create_model(name, _claude_deployment(route, drop_params=drop_params))
|
||||
resources.defer(lambda: client.proxy.delete_model(model_id))
|
||||
return name
|
||||
|
||||
|
||||
def _marked(text: str, ttl: Literal["30m"] | None = None) -> TextContentPart:
|
||||
return TextContentPart(text=text, prompt_cache_breakpoint=PromptCacheBreakpoint(ttl=ttl))
|
||||
|
||||
|
||||
def _tool_turns(result: list[ContentPart] | str, *, marked_call: bool = False) -> list[ChatTurn]:
|
||||
return [
|
||||
ChatAssistantTurn(
|
||||
tool_calls=[
|
||||
ToolCall(
|
||||
id="call_1",
|
||||
type="function",
|
||||
function=ToolCallFunction(name="lookup_reference", arguments="{}"),
|
||||
prompt_cache_breakpoint=PromptCacheBreakpoint() if marked_call else None,
|
||||
)
|
||||
]
|
||||
),
|
||||
ChatToolResultTurn(tool_call_id="call_1", content=result),
|
||||
]
|
||||
|
||||
|
||||
def _breakpoint_messages(site: BreakpointSite, prefix: str) -> list[ChatTurn]:
|
||||
ask: Final = ChatMessage(role="user", content="Reply with one word.")
|
||||
match site:
|
||||
case "system_part":
|
||||
return [ChatMessage(role="system", content=[_marked(prefix)]), ask]
|
||||
case "user_part":
|
||||
return [ChatMessage(role="user", content=[_marked(prefix), TextContentPart(text="Reply with one word.")])]
|
||||
case "string_message":
|
||||
return [ChatMessage(role="system", content=prefix, prompt_cache_breakpoint=PromptCacheBreakpoint()), ask]
|
||||
case "tool_result_part":
|
||||
return [ChatMessage(role="user", content="Look up the reference."), *_tool_turns([_marked(prefix)])]
|
||||
case "tool_call":
|
||||
return [ChatMessage(role="user", content=prefix), *_tool_turns("Done.", marked_call=True)]
|
||||
case "tool_definition":
|
||||
return [ask]
|
||||
|
||||
|
||||
def _breakpoint_tool(site: BreakpointSite, prefix: str) -> ChatTool:
|
||||
if site != "tool_definition":
|
||||
return ChatTool(function=_LOOKUP_TOOL)
|
||||
return ChatTool(
|
||||
function=ChatToolFunction(name="lookup_reference", description=prefix, parameters={"type": "object"}),
|
||||
prompt_cache_breakpoint=PromptCacheBreakpoint(),
|
||||
)
|
||||
|
||||
|
||||
def _breakpoint_body(model: str, site: BreakpointSite) -> ChatBody:
|
||||
"""A never-seen cacheable prefix whose one ``prompt_cache_breakpoint`` sits at ``site``."""
|
||||
prefix: Final = _cacheable_prefix()
|
||||
return ChatBody(
|
||||
model=model,
|
||||
max_tokens=64,
|
||||
tools=[_breakpoint_tool(site, prefix)],
|
||||
messages=_breakpoint_messages(site, prefix),
|
||||
)
|
||||
|
||||
|
||||
def _assert_claude_writes_then_reads_cache(client: PassthroughClient, key: str, body: ChatBody) -> None:
|
||||
first: Final = unwrap(client.proxy.chat(key, body))
|
||||
assert first.usage and first.usage.cache_creation_input_tokens, (
|
||||
f"{body.model}: the first call on a never-seen prefix wrote no cache ({first.usage}), "
|
||||
"so the prompt_cache_breakpoint never reached Claude as a cache mark"
|
||||
)
|
||||
|
||||
deadline: Final = time.monotonic() + 30.0
|
||||
while True:
|
||||
second = unwrap(client.proxy.chat(key, body))
|
||||
read_tokens = _cached_read_tokens(second.usage)
|
||||
if read_tokens > 0 or time.monotonic() >= deadline:
|
||||
break
|
||||
time.sleep(3.0)
|
||||
|
||||
assert read_tokens > 0, f"{body.model}: the identical second call read no cache ({second.usage})"
|
||||
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC, Provider.BEDROCK, Provider.VERTEX_AI),
|
||||
models=(
|
||||
CLAUDE_ANTHROPIC_MODEL,
|
||||
CLAUDE_BEDROCK_CONVERSE_MODEL,
|
||||
CLAUDE_BEDROCK_INVOKE_MODEL,
|
||||
CLAUDE_VERTEX_MODEL,
|
||||
),
|
||||
capabilities=(Capability.PROMPT_CACHING,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
class TestPromptCacheBreakpointOnClaude:
|
||||
"""OpenAI's ``prompt_cache_breakpoint`` sent to a Claude deployment caches the prefix it marks.
|
||||
|
||||
The calls that leave drop_params off (more than four breakpoints, or ``prompt_cache_options``) answer 400 before
|
||||
any provider call, so the wire tests in tests/integration prove them: the shared stack sets
|
||||
``litellm_settings.drop_params: true``, which no deployment can switch back off."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"route", [_claude_route_param(route) for route in ("anthropic", "bedrock_converse", "bedrock_invoke", "vertex")]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"site", ["system_part", "user_part", "string_message", "tool_result_part", "tool_call", "tool_definition"]
|
||||
)
|
||||
def test_a_breakpoint_writes_then_reads_the_cache(
|
||||
self,
|
||||
client: PassthroughClient,
|
||||
resources: ResourceManager,
|
||||
scoped_key: str,
|
||||
route: ClaudeRoute,
|
||||
site: BreakpointSite,
|
||||
) -> None:
|
||||
model: Final = _register_claude(client, resources, route)
|
||||
_assert_claude_writes_then_reads_cache(client, scoped_key, _breakpoint_body(model, site))
|
||||
|
||||
@pytest.mark.parametrize("route", [_claude_route_param(route) for route in ("anthropic", "bedrock_converse")])
|
||||
def test_a_30m_breakpoint_still_caches(
|
||||
self, client: PassthroughClient, resources: ResourceManager, scoped_key: str, route: ClaudeRoute
|
||||
) -> None:
|
||||
model: Final = _register_claude(client, resources, route)
|
||||
body: Final = ChatBody(
|
||||
model=model,
|
||||
max_tokens=64,
|
||||
messages=[
|
||||
ChatMessage(role="system", content=[_marked(_cacheable_prefix(), ttl="30m")]),
|
||||
ChatMessage(role="user", content="Reply with one word."),
|
||||
],
|
||||
)
|
||||
_assert_claude_writes_then_reads_cache(client, scoped_key, body)
|
||||
|
||||
@pytest.mark.parametrize("route", [_claude_route_param(route) for route in ("anthropic", "bedrock_converse")])
|
||||
def test_five_breakpoints_keep_the_latest_four_under_drop_params(
|
||||
self, client: PassthroughClient, resources: ResourceManager, scoped_key: str, route: ClaudeRoute
|
||||
) -> None:
|
||||
model: Final = _register_claude(client, resources, route, drop_params=True)
|
||||
body: Final = ChatBody(
|
||||
model=model,
|
||||
max_tokens=64,
|
||||
messages=[
|
||||
ChatMessage(
|
||||
role="user",
|
||||
content=[
|
||||
_marked(_cacheable_prefix()),
|
||||
*(_marked(f"Note {index}.") for index in range(4)),
|
||||
TextContentPart(text="Reply with one word."),
|
||||
],
|
||||
)
|
||||
],
|
||||
)
|
||||
_assert_claude_writes_then_reads_cache(client, scoped_key, body)
|
||||
|
||||
@pytest.mark.parametrize("route", [_claude_route_param(route) for route in ("anthropic", "bedrock_converse")])
|
||||
def test_prompt_cache_options_are_dropped_under_drop_params(
|
||||
self, client: PassthroughClient, resources: ResourceManager, scoped_key: str, route: ClaudeRoute
|
||||
) -> None:
|
||||
model: Final = _register_claude(client, resources, route, drop_params=True)
|
||||
body: Final = ChatBody(
|
||||
model=model,
|
||||
max_tokens=64,
|
||||
prompt_cache_options=PromptCacheOptions(mode="explicit"),
|
||||
messages=[
|
||||
ChatMessage(role="system", content=[_marked(_cacheable_prefix())]),
|
||||
ChatMessage(role="user", content="Reply with one word."),
|
||||
],
|
||||
)
|
||||
_assert_claude_writes_then_reads_cache(client, scoped_key, body)
|
||||
|
|
|
|||
|
|
@ -216,10 +216,18 @@ class ImageUrl(BaseModel):
|
|||
url: str
|
||||
|
||||
|
||||
class PromptCacheBreakpoint(BaseModel):
|
||||
"""OpenAI's explicit cache marker on a content part, message, tool definition or tool call."""
|
||||
|
||||
mode: Literal["explicit"] = "explicit"
|
||||
ttl: Literal["30m"] | None = None
|
||||
|
||||
|
||||
class TextContentPart(BaseModel):
|
||||
type: str = "text"
|
||||
text: str
|
||||
cache_control: CacheControl | None = None
|
||||
prompt_cache_breakpoint: PromptCacheBreakpoint | None = None
|
||||
|
||||
|
||||
class ImageContentPart(BaseModel):
|
||||
|
|
@ -234,6 +242,7 @@ class ChatMessage(BaseModel):
|
|||
role: str
|
||||
content: str | list[ContentPart]
|
||||
cache_control: CacheControl | None = None
|
||||
prompt_cache_breakpoint: PromptCacheBreakpoint | None = None
|
||||
|
||||
|
||||
class CacheControl(BaseModel):
|
||||
|
|
@ -271,6 +280,7 @@ class ChatToolFunction(BaseModel):
|
|||
class ChatTool(BaseModel):
|
||||
type: str = "function"
|
||||
function: ChatToolFunction
|
||||
prompt_cache_breakpoint: PromptCacheBreakpoint | None = None
|
||||
|
||||
|
||||
class AnthropicToolBody(BaseModel, frozen=True):
|
||||
|
|
@ -318,6 +328,7 @@ class ToolCall(BaseModel):
|
|||
type: str | None = None
|
||||
function: ToolCallFunction = ToolCallFunction()
|
||||
cache_control: CacheControl | None = None
|
||||
prompt_cache_breakpoint: PromptCacheBreakpoint | None = None
|
||||
|
||||
|
||||
class ThinkingBlock(BaseModel):
|
||||
|
|
@ -363,6 +374,7 @@ class ChatBody(BaseModel):
|
|||
thinking: ThinkingParam | None = None
|
||||
service_tier: str | None = None
|
||||
prompt_cache_key: str | None = None
|
||||
prompt_cache_options: PromptCacheOptions | None = None
|
||||
tools: Sequence[ChatTool | McpChatTool | GeminiUrlContextTool] | None = None
|
||||
tool_choice: str | None = None
|
||||
guardrails: list[str] | None = None
|
||||
|
|
|
|||
|
|
@ -8879,7 +8879,10 @@ async def test_a_cache_control_the_client_wrote_wins_over_the_breakpoint_on_the_
|
|||
|
||||
assert "prompt_cache_breakpoint" not in request.content.decode()
|
||||
assert json.loads(request.content)["messages"] == [
|
||||
{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"}}]}
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"}}],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -9006,3 +9009,88 @@ async def test_five_native_cache_control_marks_are_forwarded_and_the_provider_an
|
|||
|
||||
assert route.call_count == 1
|
||||
assert json.loads(route.calls.last.request.content)["messages"] == [{"role": "user", "content": marked}]
|
||||
|
||||
|
||||
async def test_a_breakpoint_only_on_a_tool_definition_still_holds_the_request_to_four_marks(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
marked: Final = [{"type": "text", "text": f"part {index}", "cache_control": EPHEMERAL} for index in range(4)]
|
||||
request: Final = await _send_prompt_caching_request(
|
||||
respx_mock,
|
||||
monkeypatch,
|
||||
messages=[{"role": "user", "content": marked}],
|
||||
tools=[{"type": "function", "function": _WEATHER_FUNCTION, "prompt_cache_breakpoint": EXPLICIT}],
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
body: Final = json.loads(request.content)
|
||||
assert body["tools"] == [_ANTHROPIC_WEATHER_TOOL]
|
||||
assert body["messages"] == [{"role": "user", "content": marked}]
|
||||
|
||||
|
||||
async def test_the_automatic_cache_control_takes_one_of_the_four_slots(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
request: Final = await _send_prompt_caching_request(
|
||||
respx_mock,
|
||||
monkeypatch,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": EXPLICIT} for index in range(4)
|
||||
],
|
||||
}
|
||||
],
|
||||
cache_control=EPHEMERAL,
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
body: Final = json.loads(request.content)
|
||||
assert body["cache_control"] == EPHEMERAL
|
||||
assert body["messages"] == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "part 0"},
|
||||
*({"type": "text", "text": f"part {index}", "cache_control": EPHEMERAL} for index in range(1, 4)),
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
async def test_a_message_level_breakpoint_on_list_content_does_not_count_toward_the_cap(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
request: Final = await _send_prompt_caching_request(
|
||||
respx_mock,
|
||||
monkeypatch,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": EXPLICIT} for index in range(4)
|
||||
],
|
||||
"prompt_cache_breakpoint": EXPLICIT,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert json.loads(request.content)["messages"] == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": f"part {index}", "cache_control": EPHEMERAL} for index in range(4)],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_map_openai_params_maps_a_tuple_of_tools() -> None:
|
||||
optional_params: Final = AnthropicConfig().map_openai_params(
|
||||
non_default_params={"tools": ({"type": "function", "function": _WEATHER_FUNCTION},)},
|
||||
optional_params={},
|
||||
model=PROMPT_CACHING_MODEL,
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert optional_params["tools"] == [_ANTHROPIC_WEATHER_TOOL]
|
||||
|
|
|
|||
|
|
@ -116,6 +116,44 @@ def test_a_request_without_breakpoints_is_forwarded_as_written_even_past_the_cap
|
|||
assert marks.messages[0] is messages[0]
|
||||
|
||||
|
||||
def test_a_message_level_mark_on_list_content_takes_no_slot_because_claude_never_renders_it() -> None:
|
||||
marked_parts: Final = [_text(f"part {index}", prompt_cache_breakpoint=EXPLICIT) for index in range(3)]
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "first", "prompt_cache_breakpoint": EXPLICIT},
|
||||
{"role": "user", "content": [*marked_parts, _text("tail")], "prompt_cache_breakpoint": EXPLICIT},
|
||||
]
|
||||
|
||||
marks: Final = claude_cache_marks(messages, None, automatic_breakpoint=False, drop_params=False)
|
||||
|
||||
assert marks == ClaudeCacheMarks(
|
||||
messages=(
|
||||
{"role": "user", "content": "first", "cache_control": EPHEMERAL},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [*(_text(f"part {index}", cache_control=EPHEMERAL) for index in range(3)), _text("tail")],
|
||||
"cache_control": EPHEMERAL,
|
||||
},
|
||||
),
|
||||
tools=(),
|
||||
)
|
||||
|
||||
|
||||
def test_a_message_level_mark_on_a_tool_result_with_list_content_takes_a_slot() -> None:
|
||||
messages: Final = [
|
||||
{"role": "user", "content": [_text(f"part {index}", prompt_cache_breakpoint=EXPLICIT) for index in range(4)]},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": [_text("sunny")],
|
||||
"prompt_cache_breakpoint": EXPLICIT,
|
||||
},
|
||||
]
|
||||
|
||||
assert claude_cache_marks(messages, None, automatic_breakpoint=False, drop_params=False) == (
|
||||
CacheBreakpointCapExceeded(found=5)
|
||||
)
|
||||
|
||||
|
||||
def test_tools_already_mapped_to_the_anthropic_shape_still_arm_the_cap() -> None:
|
||||
mapped_tools: Final = [
|
||||
{"name": f"tool_{index}", "input_schema": {}, "cache_control": EPHEMERAL} for index in range(4)
|
||||
|
|
@ -165,11 +203,12 @@ def test_prompt_cache_options_are_rejected_unless_drop_params() -> None:
|
|||
[
|
||||
([_tool(prompt_cache_breakpoint=EXPLICIT)], True),
|
||||
([{"type": "function", "function": {"name": "lookup", "prompt_cache_breakpoint": EXPLICIT}}], True),
|
||||
((_tool(prompt_cache_breakpoint=EXPLICIT),), True),
|
||||
([_tool(cache_control=EPHEMERAL), _tool()], False),
|
||||
(None, False),
|
||||
("not a list", False),
|
||||
],
|
||||
ids=["top-level", "under-function", "cache-control-only", "none", "opaque"],
|
||||
ids=["top-level", "under-function", "tuple", "cache-control-only", "none", "opaque"],
|
||||
)
|
||||
def test_tools_carry_prompt_cache_breakpoint_reads_both_tool_shapes(tools: object, expected: bool) -> None:
|
||||
assert tools_carry_prompt_cache_breakpoint(tools) is expected
|
||||
|
|
|
|||
|
|
@ -1393,9 +1393,10 @@ class TestBedrockAnthropicPromptCachingRegression:
|
|||
{"type": "text", "text": "What are the payment terms?"},
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize("drop_params_source", ["litellm_params", "global"])
|
||||
@pytest.mark.parametrize("model_prefix", ["bedrock/invoke/", "bedrock/converse/"])
|
||||
def test_five_breakpoints_are_cut_to_four_only_under_drop_params_for_both_routes(
|
||||
self, model_prefix: str, monkeypatch: pytest.MonkeyPatch
|
||||
self, model_prefix: str, drop_params_source: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
import litellm
|
||||
|
||||
|
|
@ -1415,11 +1416,12 @@ class TestBedrockAnthropicPromptCachingRegression:
|
|||
headers={},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(litellm, "drop_params", drop_params_source == "global")
|
||||
result: Final = config.transform_request(
|
||||
model="anthropic.claude-opus-5-5",
|
||||
messages=[{"role": "user", "content": marked}],
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": True},
|
||||
litellm_params={"drop_params": True} if drop_params_source == "litellm_params" else {},
|
||||
headers={},
|
||||
)
|
||||
if "converse" in model_prefix:
|
||||
|
|
@ -1434,7 +1436,10 @@ class TestBedrockAnthropicPromptCachingRegression:
|
|||
else:
|
||||
assert result["messages"][0]["content"] == [
|
||||
{"type": "text", "text": "part 0"},
|
||||
*({"type": "text", "text": f"part {index}", "cache_control": {"type": "ephemeral"}} for index in range(1, 5)),
|
||||
*(
|
||||
{"type": "text", "text": f"part {index}", "cache_control": {"type": "ephemeral"}}
|
||||
for index in range(1, 5)
|
||||
),
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -1506,6 +1511,7 @@ class TestBedrockAnthropicPromptCachingRegression:
|
|||
if "anthropic_beta" in additional_fields:
|
||||
assert "prompt-caching-2024-07-31" not in additional_fields["anthropic_beta"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestBedrockAnthropic1MContextRegression:
|
||||
"""
|
||||
|
|
@ -1691,6 +1697,7 @@ class TestBedrockAnthropic1MContextRegression:
|
|||
assert "context-1m-2025-08-07" in beta_headers
|
||||
assert "computer-use-2024-10-22" in beta_headers
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestBedrockAnthropicCombinedRegressions:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -10184,7 +10184,11 @@ def _breakpoint_conversation() -> list[dict[str, object]]:
|
|||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "sunny", "prompt_cache_breakpoint": _EXPLICIT}]},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": [{"type": "text", "text": "sunny", "prompt_cache_breakpoint": _EXPLICIT}],
|
||||
},
|
||||
{"role": "user", "content": "done", "prompt_cache_breakpoint": _EXPLICIT},
|
||||
]
|
||||
|
||||
|
|
@ -10241,11 +10245,19 @@ async def test_prompt_cache_breakpoints_become_cache_points_after_the_same_block
|
|||
optional_params: Final[dict[str, object]] = {"tools": [{**_LOOKUP_TOOL, "prompt_cache_breakpoint": _EXPLICIT}]}
|
||||
if sync_mode:
|
||||
result = config._transform_request(
|
||||
model=_BREAKPOINT_MODEL, messages=conversation[2:], optional_params=optional_params, litellm_params={}, headers={}
|
||||
model=_BREAKPOINT_MODEL,
|
||||
messages=conversation[2:],
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
else:
|
||||
result = await config._async_transform_request(
|
||||
model=_BREAKPOINT_MODEL, messages=conversation[2:], optional_params=optional_params, litellm_params={}, headers={}
|
||||
model=_BREAKPOINT_MODEL,
|
||||
messages=conversation[2:],
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "prompt_cache_breakpoint" not in json.dumps(result)
|
||||
|
|
@ -10253,13 +10265,17 @@ async def test_prompt_cache_breakpoints_become_cache_points_after_the_same_block
|
|||
assert result["messages"] == _expected_latest_four()["messages"][1:]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop_params_source", ["litellm_params", "global"])
|
||||
@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_six_breakpoints_keep_the_latest_four_cache_points_under_drop_params(sync_mode: bool) -> None:
|
||||
async def test_six_breakpoints_keep_the_latest_four_cache_points_under_drop_params(
|
||||
sync_mode: bool, drop_params_source: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", drop_params_source == "global")
|
||||
result: Final = await _breakpoint_request(
|
||||
sync_mode,
|
||||
{"tools": [{**_LOOKUP_TOOL, "prompt_cache_breakpoint": _EXPLICIT}]},
|
||||
{"drop_params": True},
|
||||
{"drop_params": True} if drop_params_source == "litellm_params" else {},
|
||||
)
|
||||
|
||||
assert json.dumps(result).count("cachePoint") == 4
|
||||
|
|
@ -10268,27 +10284,69 @@ async def test_six_breakpoints_keep_the_latest_four_cache_points_under_drop_para
|
|||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_six_breakpoints_are_a_bad_request_without_drop_params(sync_mode: bool, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
async def test_six_breakpoints_are_a_bad_request_without_drop_params(
|
||||
sync_mode: bool, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=r"Claude accepts at most 4 cache breakpoints, found 6"):
|
||||
await _breakpoint_request(sync_mode, {"tools": [{**_LOOKUP_TOOL, "prompt_cache_breakpoint": _EXPLICIT}]}, {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("emits_cache_points", [True, False], ids=["cache-points", "no-cache-points"])
|
||||
@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_cache_options_are_rejected_unless_drop_params(sync_mode: bool, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
async def test_prompt_cache_options_are_rejected_unless_drop_params(
|
||||
sync_mode: bool, emits_cache_points: bool, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
if not emits_cache_points:
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
_BREAKPOINT_MODEL,
|
||||
{"litellm_provider": "bedrock", "mode": "chat", "supports_prompt_cache_breakpoint": False},
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"):
|
||||
await _breakpoint_request(sync_mode, {"prompt_cache_options": {"mode": "explicit"}}, {})
|
||||
|
||||
dropped: Final = await _breakpoint_request(sync_mode, {"prompt_cache_options": {"mode": "explicit"}}, {"drop_params": True})
|
||||
dropped: Final = await _breakpoint_request(
|
||||
sync_mode, {"prompt_cache_options": {"mode": "explicit"}}, {"drop_params": True}
|
||||
)
|
||||
assert "prompt_cache_options" not in json.dumps(dropped)
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
dropped_globally: Final = await _breakpoint_request(sync_mode, {"prompt_cache_options": {"mode": "explicit"}}, {})
|
||||
assert "prompt_cache_options" not in json.dumps(dropped_globally)
|
||||
|
||||
|
||||
def test_the_automatic_cache_control_takes_one_of_the_four_cache_points() -> None:
|
||||
marked: Final = [
|
||||
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": _EXPLICIT} for index in range(4)
|
||||
]
|
||||
|
||||
result: Final = AmazonConverseConfig()._transform_request(
|
||||
model=_BREAKPOINT_MODEL,
|
||||
messages=[{"role": "user", "content": marked}],
|
||||
optional_params={"cache_control": {"type": "ephemeral"}},
|
||||
litellm_params={"drop_params": True},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert result["messages"] == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"text": "part 0"},
|
||||
*(block for index in range(1, 4) for block in ({"text": f"part {index}"}, _CACHE_POINT)),
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_five_native_cache_control_marks_are_forwarded_as_written_past_the_cap() -> None:
|
||||
marked: Final = [{"type": "text", "text": f"part {index}", "cache_control": {"type": "ephemeral"}} for index in range(5)]
|
||||
marked: Final = [
|
||||
{"type": "text", "text": f"part {index}", "cache_control": {"type": "ephemeral"}} for index in range(5)
|
||||
]
|
||||
|
||||
result: Final = AmazonConverseConfig()._transform_request(
|
||||
model=_BREAKPOINT_MODEL,
|
||||
|
|
|
|||
|
|
@ -2255,3 +2255,79 @@ def test_transform_response_carries_served_service_tier(served_tier: str | None)
|
|||
)
|
||||
|
||||
assert result.to_dict().get("service_tier") == served_tier
|
||||
|
||||
|
||||
_EXPLICIT: Final = {"mode": "explicit"}
|
||||
_EPHEMERAL: Final = {"type": "ephemeral"}
|
||||
_LOOKUP_FUNCTION: Final = {"name": "lookup", "parameters": {"type": "object", "properties": {}}}
|
||||
|
||||
|
||||
def _databricks_claude_route(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> respx.Route:
|
||||
monkeypatch.setenv("DATABRICKS_API_BASE", DATABRICKS_API_BASE)
|
||||
monkeypatch.setenv("DATABRICKS_API_KEY", DATABRICKS_API_KEY)
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
return respx_mock.post(DATABRICKS_CHAT_COMPLETIONS_URL).mock(
|
||||
return_value=httpx.Response(
|
||||
200, json=_databricks_anthropic_cache_response(cache_read_input_tokens=0, cache_creation_input_tokens=0)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_prompt_cache_breakpoints_reach_databricks_claude_as_cache_control(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
route: Final = _databricks_claude_route(respx_mock, monkeypatch)
|
||||
|
||||
litellm.completion(
|
||||
model="databricks/databricks-claude-haiku-4-5",
|
||||
messages=[
|
||||
{"role": "system", "content": [{"type": "text", "text": "rules", "prompt_cache_breakpoint": _EXPLICIT}]},
|
||||
{"role": "user", "content": "hi", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}},
|
||||
],
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["messages"] == [
|
||||
{"role": "system", "content": [{"type": "text", "text": "rules", "cache_control": _EPHEMERAL}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _EPHEMERAL}]},
|
||||
]
|
||||
|
||||
|
||||
def test_a_tool_breakpoint_holds_databricks_claude_to_four_marks_under_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
route: Final = _databricks_claude_route(respx_mock, monkeypatch)
|
||||
parts: Final = [{"type": "text", "text": f"part {index}", "cache_control": _EPHEMERAL} for index in range(5)]
|
||||
request: Final = {
|
||||
"model": "databricks/databricks-claude-haiku-4-5",
|
||||
"messages": [{"role": "user", "content": parts}],
|
||||
"tools": [{"type": "function", "function": _LOOKUP_FUNCTION, "prompt_cache_breakpoint": _EXPLICIT}],
|
||||
}
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=r"Claude accepts at most 4 cache breakpoints, found 5"):
|
||||
litellm.completion(**request)
|
||||
assert route.call_count == 0
|
||||
|
||||
litellm.completion(**request, drop_params=True)
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["messages"] == [
|
||||
{"role": "user", "content": [{"type": "text", "text": "part 0"}, *parts[1:]]}
|
||||
]
|
||||
|
||||
|
||||
def test_prompt_cache_options_toward_databricks_claude_are_rejected_unless_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
route: Final = _databricks_claude_route(respx_mock, monkeypatch)
|
||||
request: Final = {
|
||||
"model": "databricks/databricks-claude-haiku-4-5",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"prompt_cache_options": {"mode": "explicit"},
|
||||
}
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="drop_params"):
|
||||
litellm.completion(**request)
|
||||
assert route.call_count == 0
|
||||
|
||||
litellm.completion(**request, drop_params=True)
|
||||
|
||||
assert "prompt_cache_options" not in json.loads(route.calls.last.request.content)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue