diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 06767613d4d..3ef4cc51a1c 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1964,17 +1964,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) @staticmethod - def _mark_the_last_tool_for_a_tool_config_point( - optional_params: dict, points: Sequence[CacheControlInjectionPoint], litellm_params: dict - ) -> None: - """Write a ``tool_config`` point's marker on the last tool that keeps one. + def _tools_marked_for_a_tool_config_point( + tools: Sequence[object], points: Sequence[CacheControlInjectionPoint] + ) -> tuple[object, ...] | None: + """The tools with a ``tool_config`` point's marker on the last tool that keeps one. Anthropic caches the tools block up to and including the marked tool, and this transform drops a marker written on a tool-search tool. A caller that marked a - tool itself already spent the breakpoint, so the point stands down. + tool itself already spent the breakpoint, so the point stands down and this + returns None, as it does when no tool can keep a marker. """ - from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook - control: Final = next( ( point.get("control") or ChatCompletionCachedContent(type="ephemeral") @@ -1984,23 +1983,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ), None, ) - tools: Final[Sequence[object]] = cast( # cast-ok: optional_params is untyped; this key holds the tools list - Sequence[object], optional_params.get("tools") or () - ) if control is None or any(isinstance(tool, dict) and "cache_control" in tool for tool in tools): - return + return None for position in range(len(tools) - 1, -1, -1): tool = tools[position] - if not isinstance(tool, dict) or tool.get("type") in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES: + if not isinstance(tool, dict): continue - marked: list[object] = list(tools) # mutable-ok: a copy, so the caller's own tools list is left alone - marked[position] = {**tool, "cache_control": control} # mutable-ok: a copy of the one tool being marked - optional_params["tools"] = marked - AnthropicCacheControlHook.record_gateway_injection( - cast(Mapping[str, object], litellm_params), # cast-ok: litellm_params is an untyped request dict - 1, - ) - return + fields = cast(dict[str, object], tool) # cast-ok: a runtime dict whose value types are not known here + if fields.get("type") in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES: + continue + return (*tools[:position], {**fields, "cache_control": control}, *tools[position + 1 :]) + return None def transform_request( self, @@ -2133,13 +2126,23 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if len(tools) > 1: optional_params["tools"] = tools + request_params: Final = cast(dict[str, object], optional_params) # cast-ok: the untyped request params cache_control_points: Final = cast( # cast-ok: this key only holds the documented injection-point list - list[CacheControlInjectionPoint] | None, optional_params.pop("cache_control_injection_points", None) + list[CacheControlInjectionPoint] | None, request_params.pop("cache_control_injection_points", None) ) if cache_control_points: - AnthropicConfig._mark_the_last_tool_for_a_tool_config_point( - optional_params=optional_params, points=cache_control_points, litellm_params=litellm_params + marked_tools: Final = AnthropicConfig._tools_marked_for_a_tool_config_point( + cast(Sequence[object], request_params.get("tools") or ()), # cast-ok: this key holds the tools list + cache_control_points, ) + if marked_tools is not None: + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook + + optional_params["tools"] = list(marked_tools) # rebind-ok: the outgoing tools, written as above + AnthropicCacheControlHook.record_gateway_injection( + cast(Mapping[str, object], litellm_params), # cast-ok: litellm_params is an untyped request dict + 1, + ) ## Load Config config: Final = litellm.AnthropicConfig.get_config(model=model)