From 8147fae4d6d21a0ff1acde7f0664b844eee378e4 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:59:48 +0000 Subject: [PATCH] fix(anthropic): apply tool_config cache-control points on chat and /v1/messages (#47394) Backport of #47394 to stable/1.103.x Cherry-picked from 93b79d2587d186fb7266556a34a22d03f187a3da (main) --- litellm/llms/anthropic/chat/transformation.py | 15 +++ litellm/llms/anthropic/common_utils.py | 49 ++++++++-- .../messages/handler.py | 10 +- .../anthropic_cache_control_hook.py | 2 +- litellm/utils.py | 5 +- .../test_anthropic_chat_transformation.py | 91 ++++++++++++++++++- ...erimental_pass_through_messages_handler.py | 48 +++++++++- tests/test_litellm/test_utils.py | 15 +++ 8 files changed, 222 insertions(+), 13 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 1f90d375bc2..a74b8b12652 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -97,6 +97,7 @@ from ..common_utils import ( eager_input_streaming_flag, process_anthropic_headers, strip_advisor_blocks_from_messages, + tools_marked_for_a_tool_config_point, ) if TYPE_CHECKING: @@ -2026,6 +2027,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Remove internal LiteLLM parameters that should not be sent to Anthropic API optional_params.pop("is_vertex_request", None) optional_params.pop("client_metadata", None) + request_params: Final = cast( # cast-ok: optional_params is the untyped dict of the shared transform signature + dict[str, object], optional_params + ) + marked_tools: Final = tools_marked_for_a_tool_config_point( + request_params.get("tools"), request_params.pop("cache_control_injection_points", None) + ) + if marked_tools is not None: + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook + + request_litellm_params: Final = cast( # cast-ok: litellm_params is the untyped dict of the shared signature + Mapping[str, object], litellm_params + ) + AnthropicCacheControlHook.record_gateway_injection(request_litellm_params, 1) # ``top_k`` is a provider-specific kwarg that bypasses # ``map_openai_params``; gate it here, the single boundary shared by @@ -2045,6 +2059,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "model": model, "messages": anthropic_messages, **optional_params, + **({} if marked_tools is None else {"tools": list(marked_tools)}), } self._apply_output_config(data=data, model=model, optional_params=optional_params) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 98e2f6d5bde..f5dfdc1bc24 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -33,6 +33,7 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_HOSTED_TOOLS, ANTHROPIC_OAUTH_BETA_HEADER, ANTHROPIC_OAUTH_TOKEN_PREFIX, + ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AllAnthropicToolsValues, AnthropicMcpServerTool, AnthropicMessagesToolChoice, @@ -71,7 +72,7 @@ _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$") _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)") _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" _CLAUDE_CODE_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) -_CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) +OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") @@ -92,22 +93,56 @@ def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None ) +def _tool_config_point_control(points: object) -> Mapping[object, object] | None: + point_fields: Final = tuple(validated_object_mapping(point) for point in validated_object_list(points) or ()) + tool_config_point: Final = next( + (fields for fields in point_fields if fields is not None and fields.get("location") == "tool_config"), None + ) + if tool_config_point is None: + return None + control: Final = tool_config_point.get("control") + return {"type": "ephemeral"} if control is None else validated_object_mapping(control) + + +def tools_marked_for_a_tool_config_point(tools: object, points: object) -> tuple[object, ...] | None: + """Put the ``tool_config`` point's breakpoint on the last tool Anthropic caches. + + Anthropic caches tools as one prefix ending at the marked tool, so the last one carries + the mark, skipping tool-search tools whose transform drops cache markers. A client that + already marked a tool keeps its own breakpoint. Returns ``None`` when nothing is marked. + """ + control: Final = _tool_config_point_control(points) + tool_list: Final = validated_object_list(tools) or () + tool_fields: Final = tuple(validated_object_mapping(tool) for tool in tool_list) + if control is None or any(fields is not None and "cache_control" in fields for fields in tool_fields): + return None + cacheable: Final = tuple( + (position, fields) + for position, fields in enumerate(tool_fields) + if fields is not None and fields.get("type") not in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES + ) + if not cacheable: + return None + position, fields = cacheable[-1] + return (*tool_list[:position], {**fields, "cache_control": control}, *tool_list[position + 1 :]) + + def is_claude_code_user_agent(user_agent: str) -> bool: """Claude Code sends its API calls through the Anthropic SDK as `claude-cli/` and its own fetches, such as gateway model discovery, as `claude-code/`""" return user_agent.startswith(_CLAUDE_CODE_USER_AGENT_PREFIXES) -def _validated_claude_code_mapping(value: object) -> dict[object, object] | None: +def validated_object_mapping(value: object) -> dict[object, object] | None: try: return _CLAUDE_CODE_OBJECT_MAPPING_ADAPTER.validate_python(value) except ValidationError: return None -def _validated_claude_code_list(value: object) -> list[object] | None: +def validated_object_list(value: object) -> Sequence[object] | None: try: - return _CLAUDE_CODE_OBJECT_LIST_ADAPTER.validate_python(value) + return OBJECT_LIST_ADAPTER.validate_python(value) except ValidationError: return None @@ -134,10 +169,10 @@ def _claude_code_billing_fields(text: str) -> tuple[tuple[str, str], ...] | None def _claude_code_billing_texts(system: object) -> tuple[str, ...] | None: if isinstance(system, str): return (system,) - blocks: Final = _validated_claude_code_list(system) + blocks: Final = validated_object_list(system) if blocks is None: return None - block_mappings: Final = tuple(_validated_claude_code_mapping(block) for block in blocks) + block_mappings: Final = tuple(validated_object_mapping(block) for block in blocks) if any(block is None for block in block_mappings): return None text_values: Final = tuple( @@ -170,7 +205,7 @@ def is_claude_code_one_shot_subagent_request( tools: object, user_agent: str | None, ) -> bool: - only_message: Final = _validated_claude_code_mapping(messages[0]) if len(messages) == 1 else None + only_message: Final = validated_object_mapping(messages[0]) if len(messages) == 1 else None return ( user_agent is not None and is_claude_code_user_agent(user_agent) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 87a4801f987..04ab425f907 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -12,6 +12,7 @@ from functools import partial from typing import Any, Final, cast import litellm +from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.common_utils import ( @@ -19,6 +20,7 @@ from litellm.llms.anthropic.common_utils import ( sanitize_tool_use_ids_in_anthropic_messages, strip_empty_content_blocks_from_anthropic_messages, strip_provider_specific_fields_from_anthropic_messages, + tools_marked_for_a_tool_config_point, ) from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, @@ -634,7 +636,13 @@ def anthropic_messages_handler( f"custom_llm_provider is required for Anthropic messages, passed in model={model}, custom_llm_provider={custom_llm_provider}" ) - local_vars.update(kwargs) + request_kwargs: Final = cast(dict[str, object], kwargs) # cast-ok: **kwargs of the handler is untyped + marked_tools: Final = tools_marked_for_a_tool_config_point( + local_vars.get("tools"), request_kwargs.pop("cache_control_injection_points", None) + ) + if marked_tools is not None: + AnthropicCacheControlHook.record_gateway_injection(request_kwargs, 1) + local_vars.update({**request_kwargs, **({"tools": list(marked_tools)} if marked_tools is not None else {})}) anthropic_messages_optional_request_params: Final = ( AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( params=local_vars, diff --git a/litellm/types/integrations/anthropic_cache_control_hook.py b/litellm/types/integrations/anthropic_cache_control_hook.py index 20e7885a2bf..7f55612f2c7 100644 --- a/litellm/types/integrations/anthropic_cache_control_hook.py +++ b/litellm/types/integrations/anthropic_cache_control_hook.py @@ -22,7 +22,7 @@ class CacheControlMessageInjectionPoint(TypedDict): class CacheControlToolConfigInjectionPoint(TypedDict): - """Type for tool_config-level injection points (Bedrock).""" + """Type for tool_config-level injection points (Anthropic and Bedrock).""" location: Literal["tool_config"] control: ChatCompletionCachedContent | None diff --git a/litellm/utils.py b/litellm/utils.py index 9eed0e8345f..0dae090a26a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4898,6 +4898,9 @@ def get_optional_params( return optional_params +LITELLM_ONLY_PARAMS: Final = frozenset(("cache_control_injection_points",)) + + def add_provider_specific_params_to_optional_params( optional_params: dict, passed_params: dict, @@ -4914,7 +4917,7 @@ def add_provider_specific_params_to_optional_params( if _should_drop_param(k="extra_body", additional_drop_params=additional_drop_params) is False: extra_body: Final = dict(passed_params.pop("extra_body", None) or {}) for k in passed_params: - if k not in openai_params and passed_params[k] is not None: + if k not in openai_params and k not in LITELLM_ONLY_PARAMS and passed_params[k] is not None: extra_body[k] = passed_params[k] if not isinstance(optional_params.get("extra_body"), dict): optional_params["extra_body"] = {} diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 633dd1d9460..dfe4e199bfc 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,3 +1,4 @@ +import copy import pytest @@ -17,7 +18,7 @@ from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, ) -from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES from litellm.types.utils import ServerToolUse, Usage @@ -6451,3 +6452,91 @@ def test_eager_input_streaming_reaches_anthropic_request_tools(): assert result["tools"][0]["eager_input_streaming"] is True assert result["tools"][0]["name"] == "write_file" + + +def _make_transform_request(optional_params: dict, litellm_params: dict) -> dict: + return AnthropicConfig().transform_request( + model="claude-3-5-sonnet-20241022", + messages=[{"role": "user", "content": "hi"}], + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + ) + + +def _two_function_tools() -> list[dict]: + return [ + { + "type": "function", + "function": {"name": f"tool_{i}", "description": f"Tool {i}", "parameters": {"type": "object"}}, + } + for i in range(2) + ] + + +@pytest.mark.parametrize( + "control, expected_control", + [ + (None, {"type": "ephemeral"}), + ({"type": "ephemeral", "ttl": "1h"}, {"type": "ephemeral", "ttl": "1h"}), + ], +) +def test_transform_request_tool_config_point_marks_last_tool_and_drops_the_key(control, expected_control): + """LIT-9880: a tool_config injection point used to be spread into the Anthropic body, which + Anthropic rejected with 400 Extra inputs are not permitted.""" + tools = _two_function_tools() + tools_before = copy.deepcopy(tools) + point = {"location": "tool_config", **({"control": control} if control is not None else {})} + litellm_params = {"metadata": {}, "model_info": {"id": "deployment-1"}} + + data = _make_transform_request( + optional_params={"tools": tools, "cache_control_injection_points": [point]}, + litellm_params=litellm_params, + ) + + assert "cache_control_injection_points" not in data + assert data["tools"][-1]["cache_control"] == expected_control + assert "cache_control" not in data["tools"][0] + assert tools == tools_before + assert litellm_params["metadata"]["litellm_gateway_injected_cache"] == "deployment-1" + + +def test_transform_request_tool_config_point_keeps_the_clients_tool_marker(): + tools = _two_function_tools() + tools[0]["cache_control"] = {"type": "ephemeral"} + + data = _make_transform_request( + optional_params={"tools": tools, "cache_control_injection_points": [{"location": "tool_config"}]}, + litellm_params={}, + ) + + assert "cache_control_injection_points" not in data + assert data["tools"][0]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in data["tools"][1] + + +def test_transform_request_tool_config_point_skips_tool_search_tools(): + tool_search_type = next(iter(ANTHROPIC_TOOL_SEARCH_TOOL_TYPES)) + tools = [*_two_function_tools(), {"type": tool_search_type, "name": "tool_search"}] + + data = _make_transform_request( + optional_params={"tools": tools, "cache_control_injection_points": [{"location": "tool_config"}]}, + litellm_params={}, + ) + + assert data["tools"][1]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in data["tools"][2] + assert "cache_control" not in data["tools"][0] + + +def test_transform_request_tool_config_point_without_tools_drops_the_key(): + litellm_params = {"metadata": {}} + + data = _make_transform_request( + optional_params={"cache_control_injection_points": [{"location": "tool_config"}]}, + litellm_params=litellm_params, + ) + + assert "cache_control_injection_points" not in data + assert "tools" not in data + assert litellm_params["metadata"] == {} diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 4c4bc48ef88..2795a377f9a 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -1,3 +1,4 @@ +import copy import asyncio import json import os @@ -1554,7 +1555,9 @@ def _claude_code_auto_mode_request() -> tuple[list[dict[str, object]], list[dict return safeguards, safeguard_results -def _upstream_answering_with(safeguard_results: list[dict[str, object]], captured: dict[str, object]) -> AsyncHTTPHandler: +def _upstream_answering_with( + safeguard_results: list[dict[str, object]], captured: dict[str, object] +) -> AsyncHTTPHandler: def upstream_records_the_request(request: httpx.Request) -> httpx.Response: captured["body"] = json.loads(request.content) captured["anthropic-beta"] = request.headers.get("anthropic-beta") @@ -1580,7 +1583,9 @@ def _upstream_answering_with(safeguard_results: list[dict[str, object]], capture _CLIENT_BETA_HEADERS: Final = ( - pytest.param({"anthropic-beta": "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"}, id="client_sends_beta"), + pytest.param( + {"anthropic-beta": "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"}, id="client_sends_beta" + ), pytest.param({"anthropic-beta": "interleaved-thinking-2025-05-14"}, id="client_omits_beta"), pytest.param({}, id="client_sends_no_beta_header"), ) @@ -1698,3 +1703,42 @@ async def test_anthropic_messages_adds_no_dangerous_tool_use_beta_to_azure_ai_fo assert "safeguards" not in captured["body"] assert "dangerous-tool-use-2026-09-03" not in (captured["anthropic-beta"] or "").split(",") + + +def test_native_messages_tool_config_point_marks_last_tool_and_drops_the_key(monkeypatch): + """LIT-9880 on /v1/messages: the hook wrote the tool_config point back into kwargs and the + native path dropped it silently, so the tools went out without any cache breakpoint.""" + from litellm.llms.anthropic.experimental_pass_through.messages import handler + + captured = {} + + def fake_native(**kwargs): + captured["params"] = kwargs["anthropic_messages_optional_request_params"] + captured["kwargs"] = kwargs["kwargs"] + captured["messages"] = kwargs["messages"] + return "native-passthrough" + + monkeypatch.setattr(handler.base_llm_http_handler, "anthropic_messages_handler", fake_native) + tools = [{"name": f"tool_{i}", "description": f"Tool {i}", "input_schema": {"type": "object"}} for i in range(2)] + tools_before = copy.deepcopy(tools) + + result = handler.anthropic_messages_handler( + max_tokens=100, + messages=[{"role": "user", "content": "Hello"}], + model="anthropic/claude-opus-4-6", + api_key="sk-test", + tools=tools, + litellm_metadata={}, + cache_control_injection_points=[ + {"location": "message", "role": "user"}, + {"location": "tool_config", "control": {"type": "ephemeral", "ttl": "1h"}}, + ], + ) + + assert result == "native-passthrough" + assert "cache_control_injection_points" not in captured["params"] + assert "cache_control_injection_points" not in captured["kwargs"] + assert captured["params"]["tools"][-1]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + assert "cache_control" not in captured["params"]["tools"][0] + assert captured["messages"][0]["content"][-1]["cache_control"] == {"type": "ephemeral"} + assert tools == tools_before diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index da9f9dbeda6..245ba809879 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -5856,3 +5856,18 @@ async def test_nested_wrapper_exits_schedule_one_async_success_log(monkeypatch: assert len(counting_logger.logged_results) == 1, counting_logger.logged_results assert counting_logger.logged_results[0] is inner_result + + +def test_get_optional_params_keeps_a_tool_config_cache_point_out_of_the_openai_extra_body(): + """LIT-9880: the hook hands a tool_config point on for the provider step, and OpenAI has no + step that consumes it, so it must not ride along in extra_body and 400 at OpenAI.""" + optional_params = litellm.utils.get_optional_params( + model="gpt-4.1", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + cache_control_injection_points=[{"location": "tool_config", "control": {"type": "ephemeral"}}], + extra_body={"kept": True}, + ) + + assert optional_params["extra_body"] == {"kept": True} + assert "cache_control_injection_points" not in optional_params