Merge 694881b777 into dd31692282

litellm/llms/anthropic/chat/transformation.py

@@ -37,10 +37,12 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
 )
 from litellm.llms.base_llm.base_utils import type_to_response_format_param
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
+from litellm.types.integrations.anthropic_cache_control_hook import CacheControlInjectionPoint
 from litellm.types.llms.anthropic import (
     ANTHROPIC_ADVISOR_TOOL_TYPE,
     ANTHROPIC_BETA_HEADER_VALUES,
     ANTHROPIC_HOSTED_TOOLS,
+    ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
     AllAnthropicPassThroughMessageValues,
     AllAnthropicToolsValues,
     AnthropicCodeExecutionTool,
@@ -1893,6 +1895,45 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             headers=headers,
         )
 
+    @staticmethod
+    def _mark_the_last_tool_for_a_tool_config_point(
+        optional_params: dict, points: Sequence[CacheControlInjectionPoint], litellm_params: dict
+    ) -> None:
+        """Write a ``tool_config`` point's marker on the last tool that keeps one.
+
+        Anthropic caches the tools block up to and including the marked tool, and this
+        transform drops a marker written on a tool-search tool. A caller that marked a
+        tool itself already spent the breakpoint, so the point stands down.
+        """
+        from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
+
+        control: Final = next(
+            (
+                point.get("control") or ChatCompletionCachedContent(type="ephemeral")
+                for point in points
+                if point.get("location") == "tool_config"
+                and (point.get("control") is None or isinstance(point.get("control"), dict))
+            ),
+            None,
+        )
+        tools: Final[Sequence[object]] = cast(  # cast-ok: optional_params is untyped; this key holds the tools list
+            Sequence[object], optional_params.get("tools") or ()
+        )
+        if control is None or any(isinstance(tool, dict) and "cache_control" in tool for tool in tools):
+            return
+        for position in range(len(tools) - 1, -1, -1):
+            tool = tools[position]
+            if not isinstance(tool, dict) or tool.get("type") in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES:
+                continue
+            marked: list[object] = list(tools)  # mutable-ok: a copy, so the caller's own tools list is left alone
+            marked[position] = {**tool, "cache_control": control}  # mutable-ok: a copy of the one tool being marked
+            optional_params["tools"] = marked
+            AnthropicCacheControlHook.record_gateway_injection(
+                cast(Mapping[str, object], litellm_params),  # cast-ok: litellm_params is an untyped request dict
+                1,
+            )
+            return
+
     def transform_request(
         self,
         model: str,
@@ -2024,6 +2065,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         if len(tools) > 1:
             optional_params["tools"] = tools
 
+        cache_control_points: Final = cast(  # cast-ok: this key only holds the documented injection-point list
+            list[CacheControlInjectionPoint] | None, optional_params.pop("cache_control_injection_points", None)
+        )
+        if cache_control_points:
+            AnthropicConfig._mark_the_last_tool_for_a_tool_config_point(
+                optional_params=optional_params, points=cache_control_points, litellm_params=litellm_params
+            )
+
         ## Load Config
         config: Final = litellm.AnthropicConfig.get_config(model=model)
         for k, v in config.items():

litellm/types/integrations/anthropic_cache_control_hook.py

@@ -22,7 +22,7 @@ class CacheControlMessageInjectionPoint(TypedDict):
 
 
 class CacheControlToolConfigInjectionPoint(TypedDict):
-    """Type for tool_config-level injection points (Bedrock)."""
+    """Type for tool_config-level injection points (Anthropic and Bedrock)."""
 
     location: Literal["tool_config"]
     control: ChatCompletionCachedContent | None

tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py

@@ -17,6 +17,7 @@ from litellm.constants import (
     DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
     RESPONSE_FORMAT_TOOL_NAME,
 )
+from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
 from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
 from litellm.llms.anthropic.chat.transformation import AnthropicConfig
 from litellm.llms.anthropic.pass_through.messages.transformation import (
@@ -29,6 +30,11 @@ from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transfor
 from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import (
     VertexAIAnthropicConfig,
 )
+from litellm.types.integrations.anthropic_cache_control_hook import (
+    GATEWAY_INJECTED_CACHE_METADATA_KEY,
+    GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT,
+    CacheControlInjectionPoint,
+)
 from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
 from litellm.types.utils import ServerToolUse, Usage
 
@@ -6798,3 +6804,260 @@ def test_chat_dummy_tool_result_for_an_orphaned_tool_call_replays_a_byte_identic
     _assert_prefix_stable(requests)
     assert [m["role"] for m in requests[0]["messages"]] == ["user", "assistant", "user"]
     assert requests[0]["messages"][2]["content"][0]["type"] == "tool_result"
+
+
+_WEATHER_TOOL: Final = {
+    "type": "function",
+    "function": {
+        "name": "get_weather",
+        "description": "Get the weather in a city",
+        "parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
+    },
+}
+_TIME_TOOL: Final = {
+    "type": "function",
+    "function": {
+        "name": "get_time",
+        "description": "Get the time in a city",
+        "parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
+    },
+}
+
+
+async def _anthropic_body_for(
+    tools: list[dict],
+    points: list[CacheControlInjectionPoint],
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+    metadata: dict | None = None,
+) -> dict:
+    monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
+    monkeypatch.setattr(litellm, "callbacks", [AnthropicCacheControlHook()])
+    route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
+        return_value=httpx.Response(
+            200,
+            json={
+                "id": "msg_1",
+                "type": "message",
+                "role": "assistant",
+                "model": "claude-sonnet-4-5",
+                "content": [{"type": "text", "text": "It is sunny."}],
+                "stop_reason": "end_turn",
+                "usage": {"input_tokens": 10, "output_tokens": 3},
+            },
+        )
+    )
+    await litellm.acompletion(
+        model="anthropic/claude-sonnet-4-5",
+        api_key="test",
+        messages=[{"role": "user", "content": "What is the weather in Hanoi?"}],
+        tools=tools,
+        cache_control_injection_points=points,
+        metadata=metadata,
+    )
+    return json.loads(route.calls[0].request.content)
+
+
+async def test_tool_config_point_marks_the_last_tool(
+    respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Anthropic caches the tools block up to and including the marked tool."""
+    body: Final = await _anthropic_body_for(
+        [_WEATHER_TOOL, _TIME_TOOL], [{"location": "tool_config"}], respx_mock, monkeypatch
+    )
+
+    assert body["tools"] == [
+        {
+            "name": "get_weather",
+            "description": "Get the weather in a city",
+            "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
+            "type": "custom",
+        },
+        {
+            "name": "get_time",
+            "description": "Get the time in a city",
+            "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
+            "type": "custom",
+            "cache_control": {"type": "ephemeral"},
+        },
+    ]
+    assert "cache_control_injection_points" not in body
+
+
+async def test_tool_config_point_stands_down_when_the_caller_marked_a_tool(
+    respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """The caller's own marker already spent the breakpoint the point asked for."""
+    body: Final = await _anthropic_body_for(
+        [{**_WEATHER_TOOL, "cache_control": {"type": "ephemeral"}}, _TIME_TOOL],
+        [{"location": "tool_config"}],
+        respx_mock,
+        monkeypatch,
+    )
+
+    assert body["tools"] == [
+        {
+            "name": "get_weather",
+            "description": "Get the weather in a city",
+            "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
+            "type": "custom",
+            "cache_control": {"type": "ephemeral"},
+        },
+        {
+            "name": "get_time",
+            "description": "Get the time in a city",
+            "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
+            "type": "custom",
+        },
+    ]
+
+
+async def test_tool_config_point_walks_back_off_a_tool_search_tool(
+    respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """This transform drops a cache_control written on a tool-search tool."""
+    body: Final = await _anthropic_body_for(
+        [_WEATHER_TOOL, {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}],
+        [{"location": "tool_config"}],
+        respx_mock,
+        monkeypatch,
+    )
+
+    assert body["tools"] == [
+        {
+            "name": "get_weather",
+            "description": "Get the weather in a city",
+            "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
+            "type": "custom",
+            "cache_control": {"type": "ephemeral"},
+        },
+        {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},
+    ]
+
+
+async def test_tool_config_point_writes_the_control_the_point_carries(
+    respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """A point naming a one-hour cache must not be written as the five-minute default."""
+    body: Final = await _anthropic_body_for(
+        [_WEATHER_TOOL],
+        [{"location": "tool_config", "control": {"type": "ephemeral", "ttl": "1h"}}],
+        respx_mock,
+        monkeypatch,
+    )
+
+    assert body["tools"] == [
+        {
+            "name": "get_weather",
+            "description": "Get the weather in a city",
+            "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
+            "type": "custom",
+            "cache_control": {"type": "ephemeral", "ttl": "1h"},
+        }
+    ]
+
+
+async def test_tool_config_point_with_no_tool_to_mark_sends_no_injection_points(
+    respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """The key is litellm's own routing input; Anthropic rejects fields it does not define."""
+    body: Final = await _anthropic_body_for([], [{"location": "tool_config"}], respx_mock, monkeypatch)
+
+    assert "cache_control_injection_points" not in body
+    assert body["messages"] == [
+        {"role": "user", "content": [{"type": "text", "text": "What is the weather in Hanoi?"}]}
+    ]
+
+
+async def test_tool_config_point_records_the_gateway_injection(
+    respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Spend attribution credits litellm only for breakpoints it placed itself."""
+    metadata: Final = {}
+
+    await _anthropic_body_for([_WEATHER_TOOL], [{"location": "tool_config"}], respx_mock, monkeypatch, metadata)
+
+    assert metadata[GATEWAY_INJECTED_CACHE_METADATA_KEY] == GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT
+
+
+async def test_a_point_that_is_not_a_tool_config_point_leaves_the_tools_unmarked(
+    respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Every point this pass does not place reaches here, not only the tool_config one."""
+    body: Final = await _anthropic_body_for([_WEATHER_TOOL], [{"location": "system"}], respx_mock, monkeypatch)
+
+    assert body["tools"] == [
+        {
+            "name": "get_weather",
+            "description": "Get the weather in a city",
+            "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
+            "type": "custom",
+        }
+    ]
+
+
+async def test_tool_config_point_moves_past_a_control_that_is_not_an_object(
+    respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Giving up the tools breakpoint over one unusable line answers the same config two ways."""
+    body: Final = await _anthropic_body_for(
+        [_WEATHER_TOOL],
+        [{"location": "tool_config", "control": "ephemeral"}, {"location": "tool_config"}],
+        respx_mock,
+        monkeypatch,
+    )
+
+    assert body["tools"] == [
+        {
+            "name": "get_weather",
+            "description": "Get the weather in a city",
+            "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
+            "type": "custom",
+            "cache_control": {"type": "ephemeral"},
+        }
+    ]
+
+
+def test_tool_config_point_leaves_the_caller_tools_alone() -> None:
+    """A tool already in Anthropic shape reaches the tools list as the caller's own object."""
+    caller_tools: Final = [{"name": "get_weather", "description": "d", "input_schema": {"type": "object"}}]
+
+    data: Final = AnthropicConfig().transform_request(
+        model="claude-sonnet-4-5",
+        messages=[{"role": "user", "content": "What is the weather in Hanoi?"}],
+        optional_params={
+            "tools": caller_tools,
+            "cache_control_injection_points": [{"location": "tool_config"}],
+            "max_tokens": 16,
+        },
+        litellm_params={},
+        headers={},
+    )
+
+    assert data["tools"] == [
+        {
+            "name": "get_weather",
+            "description": "d",
+            "input_schema": {"type": "object"},
+            "cache_control": {"type": "ephemeral"},
+        }
+    ]
+    assert caller_tools == [{"name": "get_weather", "description": "d", "input_schema": {"type": "object"}}]
+
+
+async def test_tool_config_point_refuses_a_control_that_is_not_an_object(
+    respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """`control: ephemeral` in a config file reads back as a string, which Anthropic rejects."""
+    body: Final = await _anthropic_body_for(
+        [_WEATHER_TOOL], [{"location": "tool_config", "control": "ephemeral"}], respx_mock, monkeypatch
+    )
+
+    assert body["tools"] == [
+        {
+            "name": "get_weather",
+            "description": "Get the weather in a city",
+            "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
+            "type": "custom",
+        }
+    ]