From 4b191f68fe3b40d02fe2d54176d45ea3083c3d4b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:14:40 +0000 Subject: [PATCH] fix(cache_control): surface tool_config breakpoint on the request tools --- .../anthropic_cache_control_hook.py | 73 ++++++++-- .../bedrock/chat/converse_transformation.py | 3 +- litellm/main.py | 2 + .../test_anthropic_cache_control_hook.py | 127 ++++++++++++++++++ 4 files changed, 196 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index faedf8ae1a3..eaca538f61a 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -363,16 +363,73 @@ class AnthropicCacheControlHook(CustomPromptManagement): if any(isinstance(block, dict) and block.get("cache_control") is not None for block in system): return True if tools is not None: - return any( - isinstance(tool, dict) - and ( - tool.get("cache_control") is not None - or (isinstance(tool.get("function"), dict) and tool["function"].get("cache_control") is not None) - ) - for tool in tools - ) + return any(AnthropicCacheControlHook._tool_has_cache_control(tool) for tool in tools) return False + @staticmethod + def _tool_has_cache_control(tool: object) -> bool: + """Whether a tool definition carries a cache_control breakpoint. + + Tools carry the mark either at the top level (Anthropic shape) or + nested under ``function`` (OpenAI shape); the Anthropic chat transform + accepts both. + """ + if not isinstance(tool, dict): + return False + if tool.get("cache_control") is not None: + return True + function = tool.get("function") + return isinstance(function, dict) and function.get("cache_control") is not None + + @staticmethod + def _is_cacheable_tool(tool: object) -> bool: + """Whether a tool ends up as a real tool definition in the provider request. + + Built-in / server-side tools (web search, tool search, MCP servers) carry + neither an OpenAI ``function`` nor an Anthropic ``input_schema`` and are + dropped or rewritten by the provider transforms, so a breakpoint placed + on them would be lost. + """ + return isinstance(tool, dict) and ("function" in tool or "input_schema" in tool) + + @staticmethod + def with_tool_config_cache_control( + non_default_params: dict[str, Any], + tools: list[dict] | None, + ) -> list[dict] | None: + """Write a ``tool_config`` injection point onto the last tool definition. + + The breakpoint used to be applied only inside the Bedrock request + transform, so logging callbacks (Langfuse and friends) logged the tools + without any cache marker even though the request carried one. Marking + the OpenAI-shaped tool instead keeps the request and what callbacks log + in sync, and it is the same shape a client sets by hand. Tools already + carrying a client breakpoint are left alone. + """ + points = cast( # cast-ok: untyped params dict; this key only holds the documented injection-point list + list[CacheControlInjectionPoint] | None, + non_default_params.get("cache_control_injection_points"), + ) + if not points or not tools: + return tools + + point = next((p for p in points if p.get("location") == "tool_config"), None) + if point is None: + return tools + + if any(AnthropicCacheControlHook._tool_has_cache_control(tool) for tool in tools): + return tools + + target_index = next( + (idx for idx in reversed(range(len(tools))) if AnthropicCacheControlHook._is_cacheable_tool(tools[idx])), + None, + ) + if target_index is None: + return tools + + control = point.get("control") or ChatCompletionCachedContent(type="ephemeral") + return [{**tool, "cache_control": control} if idx == target_index else tool for idx, tool in enumerate(tools)] + @staticmethod def get_default_injection_points( messages: list[AllMessageValues], diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 8ce2b982955..14f526db70b 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1566,7 +1566,8 @@ class AmazonConverseConfig(BaseConfig): # Append cachePoint to tools if cache_control_injection_points has tool_config cache_injection_points = additional_request_params.pop("cache_control_injection_points", None) - if cache_injection_points and len(bedrock_tools) > 0: + tools_already_cached = any("cachePoint" in tool for tool in bedrock_tools) + if cache_injection_points and len(bedrock_tools) > 0 and not tools_already_cached: for point in cache_injection_points: if point.get("location") == "tool_config": cache_point = self._build_cache_point_block(point.get("control"), model) diff --git a/litellm/main.py b/litellm/main.py index dc3ec469a1b..d7ce87933f4 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -522,6 +522,7 @@ async def acompletion( custom_llm_provider=cast(Optional[str], custom_llm_provider), # cast-ok: read from untyped kwargs tools=tools, ) + tools = AnthropicCacheControlHook.with_tool_config_cache_control(non_default_params=kwargs, tools=tools) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( litellm_logging_obj.should_run_prompt_management_hooks( @@ -5080,6 +5081,7 @@ def completion( # type: ignore custom_llm_provider=cast(Optional[str], kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs tools=tools, ) + tools = AnthropicCacheControlHook.with_tool_config_cache_control(non_default_params=non_default_params, tools=tools) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( litellm_logging_obj.should_run_prompt_management_hooks( diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index d94f0d5f47e..9d92e36236d 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1909,3 +1909,130 @@ class TestAnthropicPromptCachingEnvVars: """An unparseable TTL must fall back to Anthropic's 5m default, never reach the provider verbatim.""" _, ttl = self._import_litellm_with_env({"LITELLM_ANTHROPIC_PROMPT_CACHING_TTL": value}) assert ttl is None + + +class TestToolConfigCacheControlVisibility: + """A tool_config injection point must reach the OpenAI-shaped tools. + + Applying it only inside the Bedrock transform made the breakpoint invisible + to logging callbacks (issue #34758): Langfuse and friends log + ``optional_params["tools"]``, not the provider payload. + """ + + FUNCTION_TOOL = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": {"type": "object", "properties": {"location": {"type": "string"}}}, + }, + } + + def test_marks_last_tool(self): + tools = [copy.deepcopy(self.FUNCTION_TOOL), copy.deepcopy(self.FUNCTION_TOOL)] + result = AnthropicCacheControlHook.with_tool_config_cache_control( + non_default_params={"cache_control_injection_points": [{"location": "tool_config"}]}, + tools=tools, + ) + assert result is not None + assert "cache_control" not in result[0] + assert result[1]["cache_control"] == {"type": "ephemeral"} + assert tools == [self.FUNCTION_TOOL, self.FUNCTION_TOOL], "client tool list must not be mutated" + + def test_honors_configured_control(self): + result = AnthropicCacheControlHook.with_tool_config_cache_control( + non_default_params={ + "cache_control_injection_points": [ + {"location": "tool_config", "control": {"type": "ephemeral", "ttl": "1h"}} + ] + }, + tools=[copy.deepcopy(self.FUNCTION_TOOL)], + ) + assert result is not None + assert result[0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + + def test_skips_builtin_tools_that_carry_no_definition(self): + """Built-in tools are dropped by the provider transforms, so a breakpoint + placed on them would silently disappear from the request.""" + tools = [copy.deepcopy(self.FUNCTION_TOOL), {"type": "web_search_20250305", "name": "web_search"}] + result = AnthropicCacheControlHook.with_tool_config_cache_control( + non_default_params={"cache_control_injection_points": [{"location": "tool_config"}]}, + tools=tools, + ) + assert result is not None + assert result[0]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in result[1] + + def test_leaves_client_marked_tools_alone(self): + tools = [{**copy.deepcopy(self.FUNCTION_TOOL), "cache_control": {"type": "ephemeral", "ttl": "1h"}}] + result = AnthropicCacheControlHook.with_tool_config_cache_control( + non_default_params={"cache_control_injection_points": [{"location": "tool_config"}]}, + tools=tools, + ) + assert result == tools + + @pytest.mark.parametrize( + "points", + [[], [{"location": "message", "role": "system"}]], + ids=["no_points", "message_point_only"], + ) + def test_noop_without_tool_config_point(self, points): + tools = [copy.deepcopy(self.FUNCTION_TOOL)] + result = AnthropicCacheControlHook.with_tool_config_cache_control( + non_default_params={"cache_control_injection_points": points}, + tools=tools, + ) + assert result == tools + + @pytest.mark.asyncio + async def test_bedrock_request_and_logged_tools_agree(self): + """The tools the callback sees carry the breakpoint, and the Bedrock + payload still carries exactly one tool cachePoint (no double injection).""" + import asyncio + + class CaptureLogger(litellm.integrations.custom_logger.CustomLogger): + def __init__(self): + self.optional_params = None + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.optional_params = kwargs.get("optional_params") + + with patch.dict( + os.environ, + { + "AWS_ACCESS_KEY_ID": "fake_access_key_id", + "AWS_SECRET_ACCESS_KEY": "fake_secret_access_key", + "AWS_REGION_NAME": "us-east-1", + }, + ): + capture = CaptureLogger() + litellm.callbacks = [capture] + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 10, "outputTokens": 2, "totalTokens": 12}, + } + + client_tools = [copy.deepcopy(self.FUNCTION_TOOL)] + client = AsyncHTTPHandler() + with patch.object(client, "post", return_value=mock_response) as mock_post: + await litellm.acompletion( + model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=[{"role": "user", "content": "What is the weather?"}], + tools=client_tools, + cache_control_injection_points=[{"location": "tool_config"}], + client=client, + ) + + request_body = json.loads(mock_post.call_args.kwargs["data"]) + bedrock_tools = request_body["toolConfig"]["tools"] + assert sum(1 for tool in bedrock_tools if "cachePoint" in tool) == 1 + assert "cachePoint" in bedrock_tools[-1] + + await asyncio.sleep(1) + logged_tools = capture.optional_params["tools"] + assert logged_tools[-1]["cache_control"] == {"type": "ephemeral"} + assert client_tools == [self.FUNCTION_TOOL], "client tool list must not be mutated"