mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-15 23:31:29 +00:00
fix(cache_control): stop leaking spent tool_config points to the provider
This commit is contained in:
parent
4b191f68fe
commit
cee59a7feb
3 changed files with 80 additions and 7 deletions
|
|
@ -77,10 +77,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
else:
|
||||
remaining_points.append(point)
|
||||
|
||||
# Non-message points (currently Bedrock tool_config) are handled in the
|
||||
# provider transform, where each tool_config point appends at most one
|
||||
# cachePoint to the tools. That block also counts toward Anthropic's
|
||||
# limit, so reserve a slot for it here to leave room.
|
||||
# A tool_config point ends up as one more cache block on the tools, so
|
||||
# reserve a slot for it here to stay under Anthropic's limit.
|
||||
reserved_blocks = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0
|
||||
|
||||
processed_messages = self._apply_message_injections(
|
||||
|
|
@ -89,10 +87,12 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
max_blocks=MAX_CACHE_CONTROL_BLOCKS - reserved_blocks,
|
||||
)
|
||||
|
||||
# Pass through non-message injection points for provider-specific handling
|
||||
if remaining_points:
|
||||
# Points already written onto the tools are done; the rest are passed
|
||||
# through for provider-specific handling
|
||||
pending_points = [p for p in remaining_points if not p.get("_litellm_applied")]
|
||||
if pending_points:
|
||||
non_default_params["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(
|
||||
remaining_points
|
||||
pending_points
|
||||
)
|
||||
|
||||
return model, processed_messages, non_default_params
|
||||
|
|
@ -428,6 +428,9 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
return tools
|
||||
|
||||
control = point.get("control") or ChatCompletionCachedContent(type="ephemeral")
|
||||
non_default_params["cache_control_injection_points"] = [
|
||||
{**p, "_litellm_applied": True} if p is point else p for p in points
|
||||
]
|
||||
return [{**tool, "cache_control": control} if idx == target_index else tool for idx, tool in enumerate(tools)]
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ class CacheControlToolConfigInjectionPoint(TypedDict):
|
|||
location: Literal["tool_config"]
|
||||
control: Optional[ChatCompletionCachedContent]
|
||||
_litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran
|
||||
_litellm_applied: NotRequired[bool] # Internal: the breakpoint already landed on a tool definition
|
||||
|
||||
|
||||
CacheControlInjectionPoint = Union[
|
||||
|
|
|
|||
|
|
@ -2036,3 +2036,72 @@ class TestToolConfigCacheControlVisibility:
|
|||
logged_tools = capture.optional_params["tools"]
|
||||
assert logged_tools[-1]["cache_control"] == {"type": "ephemeral"}
|
||||
assert client_tools == [self.FUNCTION_TOOL], "client tool list must not be mutated"
|
||||
|
||||
def test_applied_point_is_not_passed_through_to_the_provider(self):
|
||||
"""Once the breakpoint sits on the tools, the point itself is spent.
|
||||
|
||||
Passing it on made the Anthropic provider reject the request with
|
||||
"cache_control_injection_points: Extra inputs are not permitted"."""
|
||||
params = {"cache_control_injection_points": [{"location": "tool_config"}]}
|
||||
AnthropicCacheControlHook.with_tool_config_cache_control(
|
||||
non_default_params=params, tools=[copy.deepcopy(self.FUNCTION_TOOL)]
|
||||
)
|
||||
|
||||
_, _, non_default_params = AnthropicCacheControlHook().get_chat_completion_prompt(
|
||||
model="anthropic/claude-sonnet-4-5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
non_default_params=params,
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
dynamic_callback_params={},
|
||||
)
|
||||
assert "cache_control_injection_points" not in non_default_params
|
||||
|
||||
def test_unapplied_point_still_reaches_the_provider(self):
|
||||
"""Without tools to mark, the Bedrock transform is still the one that
|
||||
applies the breakpoint, so the point must survive the hook."""
|
||||
_, _, non_default_params = AnthropicCacheControlHook().get_chat_completion_prompt(
|
||||
model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
non_default_params={"cache_control_injection_points": [{"location": "tool_config"}]},
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
dynamic_callback_params={},
|
||||
)
|
||||
assert non_default_params["cache_control_injection_points"] == [
|
||||
{"location": "tool_config", "_litellm_judged": True}
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_request_carries_the_tool_breakpoint(self):
|
||||
"""The Anthropic payload gets the breakpoint on the tool and none of
|
||||
litellm's internal injection-point bookkeeping."""
|
||||
litellm.callbacks = []
|
||||
client = AsyncHTTPHandler()
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.headers = {}
|
||||
mock_response.json.return_value = {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-5",
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 10, "output_tokens": 2},
|
||||
}
|
||||
|
||||
with patch.object(client, "post", return_value=mock_response) as mock_post:
|
||||
await litellm.acompletion(
|
||||
model="anthropic/claude-sonnet-4-5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[copy.deepcopy(self.FUNCTION_TOOL)],
|
||||
cache_control_injection_points=[{"location": "tool_config"}],
|
||||
api_key="fake-anthropic-key",
|
||||
client=client,
|
||||
)
|
||||
|
||||
call_kwargs = mock_post.call_args.kwargs
|
||||
request_body = call_kwargs.get("json") or json.loads(call_kwargs["data"])
|
||||
assert "cache_control_injection_points" not in request_body
|
||||
assert request_body["tools"][-1]["cache_control"] == {"type": "ephemeral"}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue