mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
fix(anthropic): stand down when the client caches its tool definitions
_request_has_cache_control only looked at messages and system, so a client that marks cache_control on tools alone did not suppress auto-injection. Tool breakpoints count toward the provider's four-block limit, so three of them plus the two injected here is five, which Anthropic rejects. Thread tools through both entry points and treat a client-marked tool as the stand-down signal it already is for messages and system.
This commit is contained in:
parent
f7a3e22b22
commit
53c285a94a
4 changed files with 76 additions and 6 deletions
|
|
@ -311,16 +311,26 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
return ChatCompletionCachedContent(type="ephemeral")
|
||||
|
||||
@staticmethod
|
||||
def _request_has_cache_control(messages: list[AllMessageValues], system: Optional[Union[str, list]]) -> bool:
|
||||
def _request_has_cache_control(
|
||||
messages: list[AllMessageValues],
|
||||
system: Optional[Union[str, list]],
|
||||
tools: Optional[list] = None,
|
||||
) -> bool:
|
||||
"""Return True if the request already carries any client-supplied cache_control.
|
||||
|
||||
When the client (e.g. Claude Code) already marks its own breakpoints we
|
||||
stand down entirely rather than add more, per the auto-caching contract.
|
||||
Tools count: they are a breakpoint the client can mark, they count toward
|
||||
the provider's four-block limit, and caching only the tool definitions is
|
||||
a common pattern, so injecting alongside them can exceed the cap.
|
||||
"""
|
||||
if any(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages):
|
||||
return True
|
||||
if isinstance(system, list):
|
||||
return any(isinstance(block, dict) and block.get("cache_control") is not None for block in system)
|
||||
if any(isinstance(block, dict) and block.get("cache_control") is not None for block in system):
|
||||
return True
|
||||
if tools is not None:
|
||||
return any(isinstance(tool, dict) and tool.get("cache_control") is not None for tool in tools)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -329,6 +339,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
system: Optional[Union[str, list]],
|
||||
model: str,
|
||||
custom_llm_provider: Optional[str],
|
||||
tools: Optional[list] = None,
|
||||
) -> list[CacheControlInjectionPoint]:
|
||||
"""Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on.
|
||||
|
||||
|
|
@ -363,7 +374,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
if not supports_prompt_caching(model=model, custom_llm_provider=provider):
|
||||
return []
|
||||
|
||||
if AnthropicCacheControlHook._request_has_cache_control(messages, system):
|
||||
if AnthropicCacheControlHook._request_has_cache_control(messages, system, tools):
|
||||
return []
|
||||
|
||||
control = AnthropicCacheControlHook._default_control()
|
||||
|
|
@ -379,6 +390,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
messages: list[AllMessageValues],
|
||||
model: str,
|
||||
custom_llm_provider: Optional[str],
|
||||
tools: Optional[list] = None,
|
||||
) -> None:
|
||||
"""For /chat/completions: add default injection points to the request params.
|
||||
|
||||
|
|
@ -393,6 +405,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
system=None,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
tools=tools,
|
||||
)
|
||||
if points:
|
||||
non_default_params["cache_control_injection_points"] = points
|
||||
|
|
@ -404,6 +417,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
kwargs: Dict[str, Any],
|
||||
model: Optional[str] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
tools: Optional[list[dict]] = None,
|
||||
) -> Tuple[List[Dict], str | list | None]:
|
||||
"""Extract cache_control_injection_points from kwargs and apply if present.
|
||||
|
||||
|
|
@ -420,6 +434,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
injection_points = AnthropicCacheControlHook.get_default_injection_points(
|
||||
messages=cast(list[AllMessageValues], messages), # cast-ok: Anthropic-shaped dicts from v1/messages
|
||||
system=system,
|
||||
tools=tools,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -237,7 +237,7 @@ async def anthropic_messages(
|
|||
)
|
||||
|
||||
messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
|
||||
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider
|
||||
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools
|
||||
)
|
||||
|
||||
original_stream = stream or kwargs.get("_websearch_interception_converted_stream", False)
|
||||
|
|
@ -428,7 +428,7 @@ def anthropic_messages_handler(
|
|||
)
|
||||
|
||||
messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
|
||||
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider
|
||||
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools
|
||||
)
|
||||
|
||||
metadata = validate_anthropic_api_metadata(metadata)
|
||||
|
|
|
|||
|
|
@ -521,6 +521,7 @@ async def acompletion(
|
|||
messages=cast(list[AllMessageValues], messages), # cast-ok: acompletion types messages as a bare List
|
||||
model=model,
|
||||
custom_llm_provider=cast(Optional[str], custom_llm_provider), # cast-ok: read from untyped kwargs
|
||||
tools=tools,
|
||||
)
|
||||
|
||||
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
|
||||
|
|
@ -5078,6 +5079,7 @@ def completion( # type: ignore
|
|||
messages=cast(list[AllMessageValues], messages), # cast-ok: completion types messages as a bare List
|
||||
model=model,
|
||||
custom_llm_provider=cast(Optional[str], kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs
|
||||
tools=tools,
|
||||
)
|
||||
|
||||
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
|
||||
|
|
|
|||
|
|
@ -1547,12 +1547,13 @@ class TestEnableAnthropicPromptCaching:
|
|||
{"role": "user", "content": "latest turn"},
|
||||
]
|
||||
|
||||
def _points(self, model="claude-sonnet-4-5", provider="anthropic", messages=None, system=None):
|
||||
def _points(self, model="claude-sonnet-4-5", provider="anthropic", messages=None, system=None, tools=None):
|
||||
return AnthropicCacheControlHook.get_default_injection_points(
|
||||
messages=copy.deepcopy(self.MESSAGES) if messages is None else messages,
|
||||
system=system,
|
||||
model=model,
|
||||
custom_llm_provider=provider,
|
||||
tools=tools,
|
||||
)
|
||||
|
||||
def test_disabled_by_default(self):
|
||||
|
|
@ -1597,6 +1598,58 @@ class TestEnableAnthropicPromptCaching:
|
|||
system = [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}]
|
||||
assert self._points(messages=[{"role": "user", "content": "hi"}], system=system) == []
|
||||
|
||||
@staticmethod
|
||||
def _tools(count: int, cached: bool) -> List[dict]:
|
||||
tool: dict = {"type": "function", "function": {"name": "t", "description": "d", "parameters": {}}}
|
||||
if cached:
|
||||
tool["cache_control"] = {"type": "ephemeral"}
|
||||
return [{**tool, "function": {**tool["function"], "name": f"t{i}"}} for i in range(count)]
|
||||
|
||||
def test_stands_down_when_only_tools_carry_cache_control(self, monkeypatch):
|
||||
"""Caching just the tool definitions is a normal client pattern, and those
|
||||
breakpoints count toward the provider's four-block limit. Three of them plus
|
||||
our two would be five, which Anthropic rejects outright."""
|
||||
monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
|
||||
assert self._points(tools=self._tools(3, cached=True)) == []
|
||||
|
||||
def test_injects_when_tools_carry_no_cache_control(self, monkeypatch):
|
||||
"""Tools alone must not suppress injection; only client-marked ones do."""
|
||||
monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
|
||||
assert [p["index"] for p in self._points(tools=self._tools(3, cached=False))] == [None, -1]
|
||||
|
||||
@pytest.mark.parametrize("tools", [None, []])
|
||||
def test_absent_tools_do_not_suppress_injection(self, monkeypatch, tools):
|
||||
monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
|
||||
assert [p["index"] for p in self._points(tools=tools)] == [None, -1]
|
||||
|
||||
def test_seed_stands_down_when_only_tools_carry_cache_control(self, monkeypatch):
|
||||
"""Same guard on the /chat/completions seeding path."""
|
||||
monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
|
||||
params: dict = {}
|
||||
AnthropicCacheControlHook.maybe_seed_default_injection_points(
|
||||
non_default_params=params,
|
||||
messages=copy.deepcopy(self.MESSAGES),
|
||||
model="claude-sonnet-4-5",
|
||||
custom_llm_provider="anthropic",
|
||||
tools=self._tools(3, cached=True),
|
||||
)
|
||||
assert "cache_control_injection_points" not in params
|
||||
|
||||
def test_v1_messages_stands_down_when_only_tools_carry_cache_control(self, monkeypatch):
|
||||
"""Same guard on the /v1/messages path, where tools reach the hook directly."""
|
||||
monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
|
||||
messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]
|
||||
result_msgs, result_sys = AnthropicCacheControlHook.maybe_inject_cache_control(
|
||||
copy.deepcopy(messages),
|
||||
"sys",
|
||||
{},
|
||||
model="claude-sonnet-4-5",
|
||||
custom_llm_provider="anthropic",
|
||||
tools=self._tools(3, cached=True),
|
||||
)
|
||||
assert result_sys == "sys"
|
||||
assert result_msgs == messages
|
||||
|
||||
def test_default_ttl_is_anthropics_five_minute_cache(self, monkeypatch):
|
||||
monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
|
||||
assert all(p["control"] == {"type": "ephemeral"} for p in self._points())
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue