fix(cache_control): stop leaking spent tool_config points to the provider

This commit is contained in:
Devin AI 2026-07-27 09:36:21 +00:00
parent 4b191f68fe
commit cee59a7feb
3 changed files with 80 additions and 7 deletions

View file

@ -77,10 +77,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
else:
remaining_points.append(point)
# Non-message points (currently Bedrock tool_config) are handled in the
# provider transform, where each tool_config point appends at most one
# cachePoint to the tools. That block also counts toward Anthropic's
# limit, so reserve a slot for it here to leave room.
# A tool_config point ends up as one more cache block on the tools, so
# reserve a slot for it here to stay under Anthropic's limit.
reserved_blocks = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0
processed_messages = self._apply_message_injections(
@ -89,10 +87,12 @@ class AnthropicCacheControlHook(CustomPromptManagement):
max_blocks=MAX_CACHE_CONTROL_BLOCKS - reserved_blocks,
)
# Pass through non-message injection points for provider-specific handling
if remaining_points:
# Points already written onto the tools are done; the rest are passed
# through for provider-specific handling
pending_points = [p for p in remaining_points if not p.get("_litellm_applied")]
if pending_points:
non_default_params["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(
remaining_points
pending_points
)
return model, processed_messages, non_default_params
@ -428,6 +428,9 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return tools
control = point.get("control") or ChatCompletionCachedContent(type="ephemeral")
non_default_params["cache_control_injection_points"] = [
{**p, "_litellm_applied": True} if p is point else p for p in points
]
return [{**tool, "cache_control": control} if idx == target_index else tool for idx, tool in enumerate(tools)]
@staticmethod

View file

@ -21,6 +21,7 @@ class CacheControlToolConfigInjectionPoint(TypedDict):
location: Literal["tool_config"]
control: Optional[ChatCompletionCachedContent]
_litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran
_litellm_applied: NotRequired[bool] # Internal: the breakpoint already landed on a tool definition
CacheControlInjectionPoint = Union[

View file

@ -2036,3 +2036,72 @@ class TestToolConfigCacheControlVisibility:
logged_tools = capture.optional_params["tools"]
assert logged_tools[-1]["cache_control"] == {"type": "ephemeral"}
assert client_tools == [self.FUNCTION_TOOL], "client tool list must not be mutated"
def test_applied_point_is_not_passed_through_to_the_provider(self):
"""Once the breakpoint sits on the tools, the point itself is spent.
Passing it on made the Anthropic provider reject the request with
"cache_control_injection_points: Extra inputs are not permitted"."""
params = {"cache_control_injection_points": [{"location": "tool_config"}]}
AnthropicCacheControlHook.with_tool_config_cache_control(
non_default_params=params, tools=[copy.deepcopy(self.FUNCTION_TOOL)]
)
_, _, non_default_params = AnthropicCacheControlHook().get_chat_completion_prompt(
model="anthropic/claude-sonnet-4-5",
messages=[{"role": "user", "content": "hi"}],
non_default_params=params,
prompt_id=None,
prompt_variables=None,
dynamic_callback_params={},
)
assert "cache_control_injection_points" not in non_default_params
def test_unapplied_point_still_reaches_the_provider(self):
"""Without tools to mark, the Bedrock transform is still the one that
applies the breakpoint, so the point must survive the hook."""
_, _, non_default_params = AnthropicCacheControlHook().get_chat_completion_prompt(
model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
messages=[{"role": "user", "content": "hi"}],
non_default_params={"cache_control_injection_points": [{"location": "tool_config"}]},
prompt_id=None,
prompt_variables=None,
dynamic_callback_params={},
)
assert non_default_params["cache_control_injection_points"] == [
{"location": "tool_config", "_litellm_judged": True}
]
@pytest.mark.asyncio
async def test_anthropic_request_carries_the_tool_breakpoint(self):
"""The Anthropic payload gets the breakpoint on the tool and none of
litellm's internal injection-point bookkeeping."""
litellm.callbacks = []
client = AsyncHTTPHandler()
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.headers = {}
mock_response.json.return_value = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 10, "output_tokens": 2},
}
with patch.object(client, "post", return_value=mock_response) as mock_post:
await litellm.acompletion(
model="anthropic/claude-sonnet-4-5",
messages=[{"role": "user", "content": "hi"}],
tools=[copy.deepcopy(self.FUNCTION_TOOL)],
cache_control_injection_points=[{"location": "tool_config"}],
api_key="fake-anthropic-key",
client=client,
)
call_kwargs = mock_post.call_args.kwargs
request_body = call_kwargs.get("json") or json.loads(call_kwargs["data"])
assert "cache_control_injection_points" not in request_body
assert request_body["tools"][-1]["cache_control"] == {"type": "ephemeral"}