From 0a64f22249ece9af13faf28d2043bbe944f7eac1 Mon Sep 17 00:00:00 2001 From: JeongCheolLee Date: Mon, 26 Oct 2026 15:26:50 +0900 Subject: [PATCH 1/9] fix(anthropic): do not surface server_tool_use as client tool_calls Anthropic server-executed tools (web search, tool search, code execution) were returned in chat completions tool_calls with srvtoolu_ ids, so OpenAI-protocol agent clients treated them as pending work and re-invoked the model until their recursion limit. Move them to provider_specific_fields.server_tool_calls on both the non-streaming and streaming paths. History replay, the /v1/responses bridge and the /v1/messages streaming guardrail read that field, so multi-turn reconstruction, web_search_call output items and the guardrail's fail-closed check on server tool blocks keep working. --- .../prompt_templates/factory.py | 25 ++- .../chat/guardrail_translation/handler.py | 9 +- litellm/llms/anthropic/chat/handler.py | 46 ++++-- litellm/llms/anthropic/chat/transformation.py | 13 +- litellm/llms/anthropic/common_utils.py | 20 ++- .../streaming_iterator.py | 8 +- .../transformation.py | 15 +- litellm/tool_loop.py | 7 +- .../chat/test_anthropic_chat_handler.py | 118 +++++++++++-- .../test_anthropic_chat_transformation.py | 156 ++++++++++++++---- 10 files changed, 341 insertions(+), 76 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 0aed2956b0a..a81b387d14c 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1893,12 +1893,31 @@ def _tool_call_ids_with_a_kept_server_result(message: object) -> frozenset[str]: return frozenset() return frozenset( tool_call_id - for tool_call in parts_of(message_field(message, "tool_calls")) + for tool_call in ( + *parts_of(message_field(message, "tool_calls")), + *parts_of(message_field(provider_specific_fields, "server_tool_calls")), + ) if isinstance(tool_call_id := message_field(tool_call, "id"), str) and tool_call_is_rebuilt_as_server_tool_use(tool_call_id, provider_specific_fields) ) +def _assistant_tool_calls_with_server_tool_calls( + message: ChatCompletionAssistantMessage, +) -> Sequence[ChatCompletionAssistantToolCall] | None: + from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params + + tool_calls: Final = message.get("tool_calls") + server_tool_calls: Final = anthropic_server_tool_call_params(message) + if not server_tool_calls: + return tool_calls + client_tool_call_ids: Final = frozenset(tool_call["id"] for tool_call in tool_calls or ()) + return ( + *(tool_call for tool_call in server_tool_calls if tool_call["id"] not in client_tool_call_ids), + *(tool_calls or ()), + ) + + def _server_tool_answers_replaced_after(messages: Sequence[object], index: int) -> frozenset[str]: kept_server_result_ids: Final = _tool_call_ids_with_a_kept_server_result(messages[index]) if not kept_server_result_ids: @@ -1958,7 +1977,7 @@ _AnthropicToolInvokeItem: TypeAlias = AnthropicMessagesToolUseParam | dict[str, def convert_to_anthropic_tool_invoke( - tool_calls: list[ChatCompletionAssistantToolCall], + tool_calls: Sequence[ChatCompletionAssistantToolCall], web_search_results: Sequence[object] | None = None, tool_results: Sequence[object] | None = None, answered_tool_ids: Container[str] = frozenset(), @@ -2998,7 +3017,7 @@ def anthropic_messages_pt( # If so, we need to interleave thinking blocks with tool call groups # to preserve the original content block ordering. # Fixes: https://github.com/BerriAI/litellm/issues/23047 - assistant_tool_calls = assistant_content_block.get("tool_calls") + assistant_tool_calls = _assistant_tool_calls_with_server_tool_calls(assistant_content_block) _has_server_tool_calls = False if assistant_tool_calls is not None: for _tc in assistant_tool_calls: diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 3ab21099d3a..0b7075d23e1 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -26,6 +26,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls from litellm.llms.anthropic.pass_through.adapters.transformation import ( OPENAI_MAX_TOOL_NAME_LENGTH, LiteLLMAnthropicMessagesAdapter, @@ -1314,9 +1315,9 @@ class AnthropicMessagesHandler(BaseTranslation): if built_response is not None and hasattr(built_response, "choices") and built_response.choices: model_response: Final = cast(ModelResponse, built_response) first_choice: Final = cast(Choices, model_response.choices[0]) - tool_calls_list: Final = cast( - list[ChatCompletionMessageToolCall] | None, - first_choice.message.tool_calls, + tool_calls_list: Final = ( + *anthropic_server_tool_calls(first_choice.message), + *cast(list[ChatCompletionMessageToolCall], first_choice.message.tool_calls or []), ) string_so_far = first_choice.message.content pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ()) @@ -1324,7 +1325,7 @@ class AnthropicMessagesHandler(BaseTranslation): if string_so_far: guardrail_inputs["texts"] = [string_so_far] if tool_calls_list: - guardrail_inputs["tool_calls"] = tool_calls_list + guardrail_inputs["tool_calls"] = list(tool_calls_list) try: prepared_request_data = self._prepare_request_data( diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 13ceea3e31b..c577721b130 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -743,6 +743,8 @@ class ModelResponseIterator: self._server_tool_inputs: dict[str, object] = {} self.tool_results: list[dict[str, Any]] = [] self._current_server_tool_id: str | None = None + self._current_server_tool_name: str = "" + self._server_tool_calls: tuple[ChatCompletionToolCallChunk, ...] = () self._container_id: str | None = None @property @@ -813,10 +815,10 @@ class ModelResponseIterator: return text, tool_use, thinking_blocks, provider_specific_fields, reasoning_content self.content_blocks.append(content_block) if "partial_json" in content_block["delta"]: - # Only emit tool calls if we're in a tool_use or server_tool_use block + # Only emit tool calls if we're in a client tool_use block, server_tool_use runs on Anthropic's side # web_search_tool_result blocks also have input_json_delta but should not be treated as tool calls # See: https://github.com/BerriAI/litellm/issues/17254 - if self.current_content_block_type in ("tool_use", "server_tool_use"): + if self.current_content_block_type == "tool_use": tool_use = cast( ChatCompletionToolCallChunk, { @@ -1002,7 +1004,6 @@ class ModelResponseIterator: content_block_start["content_block"]["type"] == "tool_use" or content_block_start["content_block"]["type"] == "server_tool_use" ): - self.tool_index += 1 # Reverse-map the (sanitized) tool name back to the # caller's original. No-op when the map is empty. _stream_tool_name = content_block_start["content_block"]["name"] @@ -1011,21 +1012,24 @@ class ModelResponseIterator: # Use empty string for arguments in content_block_start - actual arguments # come in subsequent content_block_delta chunks and get accumulated. # Using str(input) here would prepend '{}' causing invalid JSON accumulation. - tool_use = ChatCompletionToolCallChunk( - id=content_block_start["content_block"]["id"], - type="function", - function=ChatCompletionToolCallFunctionChunk( - name=_stream_tool_name, - arguments="", - ), - index=self.tool_index, - ) + if content_block_start["content_block"]["type"] == "tool_use": + self.tool_index += 1 + tool_use = ChatCompletionToolCallChunk( + id=content_block_start["content_block"]["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=_stream_tool_name, + arguments="", + ), + index=self.tool_index, + ) # Track server tool use inputs for code_interpreter_results. # The initial input in content_block_start is typically {} # for streaming; the full input arrives via input_json_delta # and is assembled at content_block_stop. if content_block_start["content_block"]["type"] == "server_tool_use": self._current_server_tool_id = content_block_start["content_block"]["id"] + self._current_server_tool_name = _stream_tool_name tool_input: Final = content_block_start["content_block"].get("input", {}) self._server_tool_inputs[self._current_server_tool_id] = tool_input if _stream_tool_name == "web_search": @@ -1039,7 +1043,7 @@ class ModelResponseIterator: # Include caller information if present (for programmatic tool calling) if "caller" in content_block_start["content_block"]: caller_data: Final = content_block_start["content_block"]["caller"] - if caller_data: + if caller_data and tool_use is not None: tool_use["caller"] = cast(dict[str, Any], caller_data) elif content_block_start["content_block"]["type"] == "redacted_thinking": ( @@ -1093,7 +1097,7 @@ class ModelResponseIterator: # check if tool call content block - only for tool_use and server_tool_use blocks if self.current_content_block_type in ("tool_use", "server_tool_use"): is_empty: Final = self.check_empty_tool_call_args() - if is_empty: + if is_empty and self.current_content_block_type == "tool_use": tool_use = ChatCompletionToolCallChunk( id=None, type="function", @@ -1117,6 +1121,20 @@ class ModelResponseIterator: self._server_tool_inputs[self._current_server_tool_id] = json.loads(args) except (json.JSONDecodeError, TypeError): pass + server_tool_input: Final = self._server_tool_inputs.get(self._current_server_tool_id, {}) + self._server_tool_calls = ( + *self._server_tool_calls, + ChatCompletionToolCallChunk( + id=self._current_server_tool_id, + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=self._current_server_tool_name, + arguments=json.dumps(server_tool_input), + ), + index=len(self._server_tool_calls), + ), + ) + provider_specific_fields["server_tool_calls"] = list(self._server_tool_calls) self._current_server_tool_id = None if self.is_response_format_tool: text = self._flush_response_format_buffer() diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 92e7c84031c..6508edfbe09 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -61,6 +61,7 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_HOSTED_TOOLS, + ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AllAnthropicPassThroughMessageValues, AllAnthropicToolsValues, @@ -2849,6 +2850,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return usage + @staticmethod + def _is_server_tool_call(tool_call: ChatCompletionToolCallChunk) -> bool: + tool_call_id: Final = tool_call["id"] + return tool_call_id is not None and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) + def _build_code_by_id_map(self, tool_calls: list[ChatCompletionToolCallChunk]) -> dict[str, str]: code_by_id: Final[dict[str, str]] = {} for tc in tool_calls: @@ -2964,6 +2970,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if compaction_blocks is not None: provider_specific_fields["compaction_blocks"] = compaction_blocks + server_tool_calls: Final = [tc for tc in tool_calls if self._is_server_tool_call(tc)] + if server_tool_calls: + provider_specific_fields["server_tool_calls"] = server_tool_calls + refusal_fields: Final = refusal_provider_specific_fields( REFUSAL_STOP_FIELDS.validate_python(completion_response) ) @@ -3023,6 +3033,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ): text_content = prefix_prompt + text_content + client_tool_calls: Final = [tc for tc in tool_calls if not self._is_server_tool_call(tc)] provider_specific_fields: Final = self._build_provider_specific_fields( completion_response, citations, @@ -3035,7 +3046,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): json_mode_message, tool_calls_for_message, json_extra_content = self._resolve_json_mode_non_streaming( json_mode=json_mode, - tool_calls=tool_calls, + tool_calls=client_tool_calls, response_format_schema=response_format_schema, ) merged_text = text_content or "" diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 1f5e4e7de39..73dbb8d8eed 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -59,10 +59,10 @@ from litellm.types.llms.anthropic import ( AnthropicThinkingParam, ) from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantToolCall from litellm.types.proxy.auth.special_headers import SpecialHeaders from litellm.types.proxy.model_listing import ModelInfoResponse -from litellm.types.utils import LlmProviders +from litellm.types.utils import ChatCompletionMessageToolCall, LlmProviders _MessageT = TypeVar("_MessageT") @@ -115,6 +115,7 @@ _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)") _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) +_SERVER_TOOL_CALLS_ADAPTER: Final = TypeAdapter(tuple[ChatCompletionAssistantToolCall, ...]) _DECODED_JSON: Final = TypeAdapter(object) @@ -2014,6 +2015,21 @@ def _replayed_server_tool_use(block: object) -> _ReplayedServerToolUse | None: return None +def anthropic_server_tool_call_params(message: object) -> tuple[ChatCompletionAssistantToolCall, ...]: + fields: Final = validated_object_mapping(message_field(message, "provider_specific_fields")) + try: + return _SERVER_TOOL_CALLS_ADAPTER.validate_python(fields.get("server_tool_calls", ()) if fields else ()) + except ValidationError: + return () + + +def anthropic_server_tool_calls(message: object) -> tuple[ChatCompletionMessageToolCall, ...]: + return tuple( + ChatCompletionMessageToolCall(id=call["id"], type=call["type"], function=dict(call["function"])) + for call in anthropic_server_tool_call_params(message) + ) + + def tool_call_is_rebuilt_as_server_tool_use( tool_call_id: object, provider_specific_fields: object, answered_tool_ids: Container[str] = frozenset() ) -> bool: diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index b5ef9e95741..bc48f59fff7 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -8,6 +8,7 @@ import litellm from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend +from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls from litellm.main import stream_chunk_builder from litellm.responses.litellm_completion_transformation.compaction_items import compaction_output_item from litellm.responses.litellm_completion_transformation.custom_tools import ( @@ -459,11 +460,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): try: message: Final = litellm_complete_object.choices[0].message - tool_calls = getattr(message, "tool_calls", None) + tool_calls = [ + *anthropic_server_tool_calls(message), + *(getattr(message, "tool_calls", None) or ()), + ] except Exception: tool_calls = None - if not tool_calls or not isinstance(tool_calls, list): + if not tool_calls: return for tc in tool_calls: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index a9c7818b577..e72d9f5b948 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -56,7 +56,7 @@ from litellm.litellm_core_utils.hidden_params import ( set_hidden_params, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control +from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls, supports_anthropic_cache_control from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control @@ -2892,13 +2892,12 @@ class LiteLLMCompletionResponsesConfig: all_chat_completion_tools: Final[list[ChatCompletionMessageToolCall]] = [] for choice in chat_completion_response.choices: if isinstance(choice, Choices): - if choice.message.tool_calls: - all_chat_completion_tools.extend(choice.message.tool_calls) - for tool_call in choice.message.tool_calls: - TOOL_CALLS_CACHE.set_cache( - key=tool_call.id, - value=tool_call, - ) + for tool_call in (*anthropic_server_tool_calls(choice.message), *(choice.message.tool_calls or ())): + all_chat_completion_tools.append(tool_call) + TOOL_CALLS_CACHE.set_cache( + key=tool_call.id, + value=tool_call, + ) request_tools: Final = responses_api_request.get("tools") if responses_api_request is not None else None custom_tool_names: Final = extract_custom_tool_names(request_tools) diff --git a/litellm/tool_loop.py b/litellm/tool_loop.py index aaa0d62eea7..2f86130e834 100644 --- a/litellm/tool_loop.py +++ b/litellm/tool_loop.py @@ -118,9 +118,12 @@ def _replays_provider_specific_fields( client_tool_calls: tuple[ChatCompletionMessageToolCall, ...], ) -> bool: from litellm.completion_extras.litellm_responses_transformation.transformation import carries_tool_search_items + from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params - return client_tool_calls != tool_calls or carries_tool_search_items( - getattr(message, "provider_specific_fields", None) + return ( + client_tool_calls != tool_calls + or bool(anthropic_server_tool_call_params(message)) + or carries_tool_search_items(getattr(message, "provider_specific_fields", None)) ) diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index 1f4a5aaa256..f95fe655881 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -15,7 +15,9 @@ import litellm from litellm._uuid import uuid from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of from litellm.llms.anthropic.chat.handler import ModelResponseIterator, _envelope_verdict, make_call +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params from litellm.llms.base_llm.base_utils import response_format_schema, unwrap_response_format_envelope from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.llms.openai import ( @@ -23,7 +25,7 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallFunctionChunk, ) from litellm.types.responses.main import OutputCodeInterpreterCall -from litellm.types.utils import ChatCompletionDeltaToolCall, ModelResponseStream +from litellm.types.utils import ChatCompletionDeltaToolCall, ChatCompletionMessageToolCall, Choices, ModelResponseStream from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE @@ -1729,19 +1731,113 @@ def test_web_search_tool_result_no_extra_tool_calls(): for tc in parsed.choices[0].delta.tool_calls: tool_calls_emitted.append(tc) - # Should have exactly 2 tool calls: - # 1. From content_block_start (server_tool_use) with id and name - # 2. From content_block_delta with the actual query - assert len(tool_calls_emitted) == 2, f"Expected 2 tool calls, got {len(tool_calls_emitted)}" + assert tool_calls_emitted == [] - # First tool call should have the id and name - assert tool_calls_emitted[0]["id"] == "srvtoolu_01ABC123" - assert tool_calls_emitted[0]["function"]["name"] == "web_search" - # Second tool call should have the query arguments - assert tool_calls_emitted[1]["function"]["arguments"] == '{"query": "test"}' +def _server_tool_input_delta(fragment: str) -> dict[str, object]: + return {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": fragment}} - # The {} chunks from web_search_tool_result should NOT have been emitted as tool calls + +def _server_search_then_answer(*partial_json: str) -> tuple[dict[str, object], ...]: + return ( + { + "type": "message_start", + "message": {"id": "msg_01", "model": "claude-opus-5-5", "usage": {"input_tokens": 9}}, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_01Web", "name": "web_search", "input": {}}, + }, + *(_server_tool_input_delta(fragment) for fragment in partial_json), + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_01Web", "content": []}, + }, + {"type": "content_block_stop", "index": 1}, + {"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "Paris is in France."}}, + {"type": "content_block_stop", "index": 2}, + ) + + +_SERVER_SEARCH_THEN_ANSWER: Final = _server_search_then_answer('{"query": "Paris"}') +_CLIENT_TOOL_CALL: Final[tuple[dict[str, object], ...]] = ( + { + "type": "content_block_start", + "index": 3, + "content_block": {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {}}, + }, + { + "type": "content_block_delta", + "index": 3, + "delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'}, + }, + {"type": "content_block_stop", "index": 3}, +) + + +def _message_delta(stop_reason: str) -> dict[str, object]: + return {"type": "message_delta", "delta": {"stop_reason": stop_reason}, "usage": {"output_tokens": 20}} + + +_PARIS_SEARCH: Final = ("srvtoolu_01Web", "web_search", '{"query": "Paris"}') + + +@pytest.mark.parametrize( + ("events", "expected_tool_calls", "expected_finish_reason", "expected_server_tool_call"), + [ + pytest.param( + (*_SERVER_SEARCH_THEN_ANSWER, _message_delta("end_turn")), [], "stop", _PARIS_SEARCH, id="server_tool_only" + ), + pytest.param( + (*_SERVER_SEARCH_THEN_ANSWER, *_CLIENT_TOOL_CALL, _message_delta("tool_use")), + [("toolu_01Weather", "get_weather", '{"city": "Paris"}')], + "tool_calls", + _PARIS_SEARCH, + id="server_tool_then_client_tool", + ), + pytest.param( + (*_server_search_then_answer('{"que', 'ry": "Pa', 'ris"}'), _message_delta("end_turn")), + [], + "stop", + _PARIS_SEARCH, + id="server_tool_input_split_across_deltas", + ), + pytest.param( + (*_server_search_then_answer(), _message_delta("end_turn")), + [], + "stop", + ("srvtoolu_01Web", "web_search", "{}"), + id="server_tool_with_empty_input", + ), + ], +) +def test_streamed_server_tool_use_is_not_returned_as_a_client_tool_call( + events: tuple[dict[str, object], ...], + expected_tool_calls: list[tuple[str, str, str]], + expected_finish_reason: str, + expected_server_tool_call: tuple[str, str, str], +): + choice: Final = _assemble_stream(events).choices[0] + assert isinstance(choice, Choices) + + assert (choice.message.tool_calls is None) == (not expected_tool_calls) + assert [ + (call.id, call.function.name, call.function.arguments) + for call in choice.message.tool_calls or () + if isinstance(call, ChatCompletionMessageToolCall) + ] == expected_tool_calls + assert choice.finish_reason == expected_finish_reason + assert choice.message.content == "Paris is in France." + assert [ + (call["id"], call["function"].get("name"), call["function"].get("arguments")) + for call in anthropic_server_tool_call_params(choice.message) + ] == [expected_server_tool_call] + web_search_results: Final = message_field(choice.message.provider_specific_fields, "web_search_results") + assert [message_field(result, "tool_use_id") for result in parts_of(web_search_results)] == ["srvtoolu_01Web"] def test_current_content_block_type_tracking(): diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 32f18b952ac..320600d3dcc 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,7 +1,8 @@ import copy import json +from collections.abc import Mapping from itertools import chain -from typing import Final +from typing import Final, cast from unittest.mock import MagicMock, patch import httpx @@ -22,8 +23,13 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of from litellm.llms.anthropic.chat import ModelResponseIterator -from litellm.llms.anthropic.common_utils import process_anthropic_headers +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + process_anthropic_headers, + validated_object_mapping, +) from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema from litellm.llms.anthropic.pass_through.messages.transformation import ( @@ -38,8 +44,8 @@ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation im VertexAIAnthropicConfig, ) from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES -from litellm.types.llms.openai import ChatCompletionToolCallFunctionChunk -from litellm.types.utils import ChatCompletionToolCallChunk, ModelResponse, ServerToolUse, Usage +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallFunctionChunk +from litellm.types.utils import ChatCompletionToolCallChunk, Choices, Message, ModelResponse, ServerToolUse, Usage from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE @@ -3519,21 +3525,17 @@ def test_code_execution_tool_results_extraction(): prefix_prompt=None, ) - # Verify tool calls are present - assert transformed_response.choices[0].message.tool_calls is not None - assert len(transformed_response.choices[0].message.tool_calls) == 2 + assert transformed_response.choices[0].message.tool_calls is None - # Verify first tool call - assert transformed_response.choices[0].message.tool_calls[0].id == "srvtoolu_01ABC" - assert transformed_response.choices[0].message.tool_calls[0].function.name == "bash_code_execution" - - # Verify second tool call - assert transformed_response.choices[0].message.tool_calls[1].id == "srvtoolu_01DEF" - assert transformed_response.choices[0].message.tool_calls[1].function.name == "text_editor_code_execution" - - # Verify tool results are in provider_specific_fields provider_fields = transformed_response.choices[0].message.provider_specific_fields assert provider_fields is not None + server_tool_calls: Final = anthropic_server_tool_call_params(transformed_response.choices[0].message) + assert [(call["id"], call["function"].get("name")) for call in server_tool_calls] == [ + ("srvtoolu_01ABC", "bash_code_execution"), + ("srvtoolu_01DEF", "text_editor_code_execution"), + ] + + # Verify tool results are in provider_specific_fields assert "tool_results" in provider_fields assert provider_fields["tool_results"] is not None assert len(provider_fields["tool_results"]) == 2 @@ -3643,7 +3645,7 @@ def _tool_search_turn(tool_references: list[str], last_block: dict[str, object]) ] -def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str): +def _parsed_choice(content: list[dict[str, object]], stop_reason: str) -> Choices: anthropic_response: Final = { "id": "msg_01XYZ", "type": "message", @@ -3658,7 +3660,7 @@ def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str raw_response.json.return_value = anthropic_response raw_response.status_code = 200 raw_response.headers = {} - return ( + choice: Final = ( AnthropicConfig() .transform_parsed_response( completion_response=anthropic_response, @@ -3668,14 +3670,19 @@ def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str prefix_prompt=None, ) .choices[0] - .message ) + assert isinstance(choice, Choices) + return choice + + +def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str) -> Message: + return _parsed_choice(content, stop_reason).message _WEATHER_CALL: Final = {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}} -def _web_search_turn(last_block: dict[str, object]) -> list[dict[str, object]]: +def _web_search_turn(last_block: Mapping[str, object]) -> list[dict[str, object]]: return [ {"type": "server_tool_use", "id": "srvtoolu_01Web", "name": "web_search", "input": {"query": "Paris facts"}}, { @@ -3691,7 +3698,7 @@ def _web_search_turn(last_block: dict[str, object]) -> list[dict[str, object]]: } ], }, - last_block, + dict(last_block), ] @@ -3738,8 +3745,87 @@ def test_server_tool_result_round_trips_right_after_its_call( assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"] -def _block_ids(message: dict[str, object], block_type: str, id_key: str) -> list[object]: - return sorted(block[id_key] for block in message["content"] if block["type"] == block_type) +_PARIS_TEXT: Final = "Paris is the capital of France." +_PARIS_ANSWER: Final[Mapping[str, str]] = {"type": "text", "text": _PARIS_TEXT} + + +@pytest.mark.parametrize( + ("content", "stop_reason", "expected_tool_call_ids", "expected_finish_reason"), + [ + pytest.param(_web_search_turn(_PARIS_ANSWER), "end_turn", [], "stop", id="web_search_then_answer"), + pytest.param( + _web_search_turn(_WEATHER_CALL), "tool_use", ["toolu_01Weather"], "tool_calls", id="web_search_then_tool" + ), + pytest.param([_WEATHER_CALL], "tool_use", ["toolu_01Weather"], "tool_calls", id="client_tool_only"), + ], +) +def test_server_tool_use_is_not_returned_as_a_client_tool_call( + content: list[dict[str, object]], + stop_reason: str, + expected_tool_call_ids: list[str], + expected_finish_reason: str, +): + choice: Final = _parsed_choice(content, stop_reason) + + assert (choice.message.tool_calls is None) == (not expected_tool_call_ids) + assert [call.id for call in choice.message.tool_calls or ()] == expected_tool_call_ids + assert choice.finish_reason == expected_finish_reason + assert [ + (call["id"], call["function"].get("name"), call["function"].get("arguments")) + for call in anthropic_server_tool_call_params(choice.message) + ] == [ + (block["id"], block["name"], json.dumps(block["input"])) + for block in content + if block["type"] == "server_tool_use" + ] + if not expected_tool_call_ids: + assert choice.message.content == _PARIS_TEXT + + +def _replayed(*messages: object): + return anthropic_messages_pt( + messages=cast(list[AllMessageValues], list(messages)), # cast-ok: replays history the way clients send it back + model="claude-opus-5-5", + llm_provider="anthropic", + ) + + +def test_web_search_answer_round_trips_without_any_client_tool_call(): + message: Final = _parsed_assistant_message(_web_search_turn(_PARIS_ANSWER), "end_turn") + + replayed: Final = _replayed( + {"role": "user", "content": "Tell me about Paris"}, + message.model_dump(), + {"role": "user", "content": "And Lyon?"}, + ) + + assert _block_ids(replayed[1], "server_tool_use", "id") == ["srvtoolu_01Web"] + assert _block_ids(replayed[1], "web_search_tool_result", "tool_use_id") == ["srvtoolu_01Web"] + assert [ + message_field(block, "text") + for block in parts_of(replayed[1]["content"]) + if message_field(block, "type") == "text" + ] == [_PARIS_TEXT] + + +def _block_ids(message: Mapping[str, object], block_type: str, id_key: str) -> list[str]: + return sorted( + block_id + for block in parts_of(message["content"]) + if message_field(block, "type") == block_type and isinstance(block_id := message_field(block, id_key), str) + ) + + +def _as_returned_before_server_tool_calls_moved(message: Message) -> Mapping[str, object]: + dumped: Final[Mapping[str, object]] = message.model_dump() + provider_fields: Final = validated_object_mapping(dumped.get("provider_specific_fields")) or {} + return { + **dumped, + "tool_calls": [*anthropic_server_tool_call_params(message), *parts_of(dumped.get("tool_calls"))], + "provider_specific_fields": { + key: value for key, value in provider_fields.items() if key != "server_tool_calls" + }, + } _SEARCH_THAT_FOUND_NOTHING: Final = _tool_search_turn( @@ -3774,21 +3860,33 @@ def test_replayed_search_that_found_nothing_answers_every_tool_use_and_ends_on_a server_tool_use_ids: list[str], monkeypatch: pytest.MonkeyPatch, ): - from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt - message: Final = _parsed_assistant_message(_SEARCH_THAT_FOUND_NOTHING, "end_turn") monkeypatch.setattr(litellm, "modify_params", modify_params) - replayed: Final = anthropic_messages_pt( - messages=[{"role": "user", "content": "Find a tool and use it"}, message.model_dump(), *follow_ups], - model="claude-opus-5-5", - llm_provider="anthropic", + replayed: Final = _replayed( + {"role": "user", "content": "Find a tool and use it"}, + _as_returned_before_server_tool_calls_moved(message), + *follow_ups, ) assert [turn["role"] for turn in replayed] == ["user", "assistant", "user"] assert _block_ids(replayed[2], "tool_result", "tool_use_id") == _block_ids(replayed[1], "tool_use", "id") assert _block_ids(replayed[1], "server_tool_use", "id") == server_tool_use_ids +def test_replayed_search_that_found_nothing_keeps_its_result_when_the_call_never_reached_the_client( + monkeypatch: pytest.MonkeyPatch, +): + message: Final = _parsed_assistant_message(_SEARCH_THAT_FOUND_NOTHING, "end_turn") + monkeypatch.setattr(litellm, "modify_params", True) + + replayed: Final = _replayed({"role": "user", "content": "Find a tool and use it"}, message.model_dump()) + + assert [turn["role"] for turn in replayed] == ["user", "assistant"] + assert _block_ids(replayed[1], "server_tool_use", "id") == ["srvtoolu_01Search"] + assert _block_ids(replayed[1], "tool_search_tool_result", "tool_use_id") == ["srvtoolu_01Search"] + assert _block_ids(replayed[1], "tool_use", "id") == [] + + _CODE_EXECUTION_BLOCKS: Final = [ { "type": "server_tool_use", From 8dbf05bc7402425a25ebbc04962b781530d99c77 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:14:05 -0700 Subject: [PATCH 2/9] fix(responses): return Claude web_fetch calls as web_search_call open_page items --- litellm/llms/anthropic/chat/handler.py | 11 ++- litellm/llms/anthropic/chat/transformation.py | 32 ++++--- litellm/types/responses/main.py | 68 ++++++++++---- .../test_litellm_completion_responses.py | 73 ++++++++++++++ .../test_streaming_iterator_transformation.py | 94 +++++++++++++++++++ 5 files changed, 248 insertions(+), 30 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 9e0331d902c..93c01ec6ae0 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -53,6 +53,7 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallFunctionChunk, ) from litellm.types.responses.main import ( + HOSTED_WEB_TOOL_NAMES, OutputCodeInterpreterCall, build_code_interpreter_log_outputs, build_web_search_call, @@ -918,7 +919,7 @@ class ModelResponseIterator: def _web_search_call_snapshot(self) -> dict[str, object]: return dict(self._web_search_calls) - def _complete_web_search_call(self, result: dict[str, object]) -> None: + def _complete_web_search_call(self, result: dict[str, object], tool_name: str) -> None: tool_use_id: Final = result.get("tool_use_id") if not isinstance(tool_use_id, str) or tool_use_id not in self._web_search_calls: return @@ -926,6 +927,7 @@ class ModelResponseIterator: tool_id=tool_use_id, tool_input=self._server_tool_inputs.get(tool_use_id, {}), result=result, + tool_name=tool_name, ) def _build_code_interpreter_results(self) -> list: @@ -1033,12 +1035,13 @@ class ModelResponseIterator: self._current_server_tool_name = _stream_tool_name tool_input: Final = content_block_start["content_block"].get("input", {}) self._server_tool_inputs[self._current_server_tool_id] = tool_input - if _stream_tool_name == "web_search": + if _stream_tool_name in HOSTED_WEB_TOOL_NAMES: self._web_search_calls[self._current_server_tool_id] = build_web_search_call( self._current_server_tool_id, tool_input, {"content": []}, status="in_progress", + tool_name=_stream_tool_name, ) provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot() # Include caller information if present (for programmatic tool calling) @@ -1075,7 +1078,7 @@ class ModelResponseIterator: # The full content comes in content_block_start, not in deltas # See: https://github.com/BerriAI/litellm/issues/17737 self.web_search_results.append(content_block_start["content_block"]) - self._complete_web_search_call(content_block_start["content_block"]) + self._complete_web_search_call(content_block_start["content_block"], "web_search") provider_specific_fields["web_search_results"] = self.web_search_results provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot() elif content_type == "web_fetch_tool_result": @@ -1083,7 +1086,9 @@ class ModelResponseIterator: # The full content comes in content_block_start, not in deltas # Fixes: https://github.com/BerriAI/litellm/issues/18137 self.web_search_results.append(content_block_start["content_block"]) + self._complete_web_search_call(content_block_start["content_block"], "web_fetch") provider_specific_fields["web_search_results"] = self.web_search_results + provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot() else: self.tool_results.append(content_block_start["content_block"]) provider_specific_fields["tool_results"] = self.tool_results diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 6508edfbe09..2eb9d06f1c5 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -94,6 +94,7 @@ from litellm.types.llms.openai import ( OpenAIWebSearchOptions, ) from litellm.types.responses.main import ( + HOSTED_WEB_TOOL_NAMES, OutputCodeInterpreterCall, build_code_interpreter_log_outputs, build_web_search_call, @@ -251,6 +252,19 @@ def _validated_defer_loading(value: object) -> bool: raise ValueError("defer_loading must be a boolean") from exc +def _hosted_web_tool_use(block: object) -> tuple[str, str, object] | None: + match block: + case { + "type": "server_tool_use", + "name": str() as name, + "id": str() as call_id, + "input": object() as tool_input, + } if name in HOSTED_WEB_TOOL_NAMES: + return call_id, name, tool_input + case _: + return None + + class _AnthropicUsageIteration(TypedDict, total=False): """One entry of the ``usage.iterations`` array on an Anthropic response.""" @@ -2902,26 +2916,22 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) -> list[object]: content: Final = completion_response.get("content") blocks: Final = content if isinstance(content, Sequence) else () - inputs: Final = { - call_id: tool_input - for block in blocks - if isinstance(block, Mapping) - and block.get("type") == "server_tool_use" - and block.get("name") == "web_search" - and isinstance((call_id := block.get("id")), str) - and isinstance((tool_input := block.get("input")), Mapping) + server_tool_uses: Final = { + call_id: (tool_name, tool_input) + for call_id, tool_name, tool_input in filter(None, map(_hosted_web_tool_use, blocks)) } return [ build_web_search_call( tool_id=tool_use_id, - tool_input=inputs.get(tool_use_id, {}), + tool_input=server_tool_uses[tool_use_id][1], result=result, + tool_name=server_tool_uses[tool_use_id][0], ) for result in web_search_results if isinstance(result, dict) - and result.get("type") == "web_search_tool_result" and isinstance((tool_use_id := result.get("tool_use_id")), str) - and tool_use_id in inputs + and tool_use_id in server_tool_uses + and result.get("type") == f"{server_tool_uses[tool_use_id][0]}_tool_result" ] def _build_provider_specific_fields( diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 72dc63a581f..7b1fb0130d8 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -3,7 +3,12 @@ from collections.abc import Mapping, Sequence from typing import Final, Literal, Optional, Union from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall -from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch +from openai.types.responses.response_function_web_search import ( + ActionOpenPage, + ActionSearch, + ActionSearchSource, + ResponseFunctionWebSearch, +) from pydantic import ConfigDict, PrivateAttr from typing_extensions import TypedDict @@ -42,14 +47,11 @@ class OutputFunctionToolCall(BaseLiteLLMOpenAIResponseObject): phase: Phase = None -def build_web_search_call( - tool_id: str, - tool_input: object, - result: object, - status: Literal["in_progress", "searching", "completed", "failed"] | None = None, -) -> ResponseFunctionWebSearch: +HOSTED_WEB_TOOL_NAMES: Final = frozenset({"web_search", "web_fetch"}) + + +def _web_search_action(tool_input: object, content: object) -> ActionSearch: query: Final = tool_input.get("query", "") if isinstance(tool_input, Mapping) else "" - content: Final = result.get("content") if isinstance(result, Mapping) else None result_items: Final = content if isinstance(content, Sequence) and not isinstance(content, (str, bytes)) else () sources: Final = [ ActionSearchSource(type="url", url=url) @@ -58,17 +60,51 @@ def build_web_search_call( and item.get("type") == "web_search_result" and isinstance((url := item.get("url")), str) ] - failed: Final = isinstance(content, Mapping) and content.get("type") == "web_search_tool_result_error" + return ActionSearch( + type="search", + query=query if isinstance(query, str) else "", + queries=[query] if isinstance(query, str) and query else [], + sources=sources, + ) + + +def _web_fetch_action(tool_input: object) -> ActionOpenPage: + match tool_input: + case {"url": str() as url}: + return ActionOpenPage(type="open_page", url=url) + case _: + return ActionOpenPage(type="open_page", url=None) + + +def _result_content(result: object) -> object: + match result: + case {"content": object() as content}: + return content + case _: + return None + + +def _is_tool_result_error(content: object, tool_name: str) -> bool: + match content: + case {"type": str() as content_type}: + return content_type == f"{tool_name}_tool_result_error" + case _: + return False + + +def build_web_search_call( + tool_id: str, + tool_input: object, + result: object, + status: Literal["in_progress", "searching", "completed", "failed"] | None = None, + tool_name: str = "web_search", +) -> ResponseFunctionWebSearch: + content: Final = _result_content(result) return ResponseFunctionWebSearch( id=f"ws_{tool_id}", type="web_search_call", - status=status or ("failed" if failed else "completed"), - action={ - "type": "search", - "query": query if isinstance(query, str) else "", - "queries": [query] if isinstance(query, str) and query else [], - "sources": sources, - }, + status=status or ("failed" if _is_tool_result_error(content, tool_name) else "completed"), + action=_web_fetch_action(tool_input) if tool_name == "web_fetch" else _web_search_action(tool_input, content), ) diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 3fd4be6f722..06ffa721cd4 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -4952,6 +4952,79 @@ def test_function_call_tool_id_falls_back_to_unique_id_for_degenerate_call_id(): assert convert(openai)["id"] == "call_tokyo" +_FETCHED_URL: Final = "https://example.com/" +_BLOCKED_URL: Final = "https://blocked.invalid/" +_ANTHROPIC_WEB_FETCH_TURN: Final[tuple[dict[str, object], ...]] = ( + {"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch", "input": {"url": _FETCHED_URL}}, + { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_01Fetch", + "content": { + "type": "web_fetch_result", + "url": _FETCHED_URL, + "content": {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Example"}}, + }, + }, + {"type": "server_tool_use", "id": "srvtoolu_02Fetch", "name": "web_fetch", "input": {"url": _BLOCKED_URL}}, + { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_02Fetch", + "content": {"type": "web_fetch_tool_result_error", "error_code": "url_not_allowed"}, + }, + {"type": "text", "text": "The page is titled Example."}, + {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}}, +) + + +def test_anthropic_web_fetch_is_a_hosted_open_page_call_on_responses() -> None: + from unittest.mock import MagicMock + + import httpx + + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + anthropic_response: Final = { + "id": "msg_01Fetch", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": list(_ANTHROPIC_WEB_FETCH_TURN), + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 100, "output_tokens": 50, "server_tool_use": {"web_fetch_requests": 2}}, + } + raw_response: Final = MagicMock(spec=httpx.Response) + raw_response.json.return_value = anthropic_response + raw_response.status_code = 200 + raw_response.headers = {} + chat_response: Final = AnthropicConfig().transform_parsed_response( + completion_response=anthropic_response, + raw_response=raw_response, + model_response=ModelResponse(), + json_mode=False, + prefix_prompt=None, + ) + + response: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Fetch two pages and check the weather in Paris.", + responses_api_request={ + "tools": [ + {"type": "web_fetch_20250910", "name": "web_fetch"}, + {"type": "function", "name": "get_weather", "parameters": {"type": "object"}}, + ] + }, + chat_completion_response=chat_response, + ) + + assert [ + (item.id, item.status, item.action.model_dump()) for item in response.output if item.type == "web_search_call" + ] == [ + ("ws_srvtoolu_01Fetch", "completed", {"type": "open_page", "url": _FETCHED_URL}), + ("ws_srvtoolu_02Fetch", "failed", {"type": "open_page", "url": _BLOCKED_URL}), + ] + assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"] + + class TestHostedWebSearchReplay: def test_emitted_hosted_search_output_round_trips_with_client_tool_result(self) -> None: search_result: Final = { diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index d9a0033b179..db7c54c52ca 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -355,6 +355,100 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to ) +def _anthropic_web_fetch_stream_events() -> tuple[dict[str, object], ...]: + return ( + { + "type": "message_start", + "message": { + "id": "msg_01Fetch", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch"}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com/"}'}, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_01Fetch", + "content": {"type": "web_fetch_result", "url": "https://example.com/"}, + }, + }, + {"type": "content_block_stop", "index": 1}, + {"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "Fetched."}}, + {"type": "content_block_stop", "index": 2}, + { + "type": "content_block_start", + "index": 3, + "content_block": {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {}}, + }, + { + "type": "content_block_delta", + "index": 3, + "delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'}, + }, + {"type": "content_block_stop", "index": 3}, + { + "type": "message_delta", + "delta": {"stop_reason": "tool_use", "stop_sequence": None}, + "usage": {"output_tokens": 20}, + }, + ) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call(sync_mode: bool) -> None: + from litellm.llms.anthropic.chat.handler import ModelResponseIterator + + parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False) + chunks: Final = [parser.chunk_parser(event) for event in _anthropic_web_fetch_stream_events()] + iterator: Final = LiteLLMCompletionStreamingIterator( + model="claude-sonnet-5-5", + litellm_custom_stream_wrapper=_FakeStreamWrapper(chunks), + request_input="Fetch example.com and check the weather in Paris.", + responses_api_request={ + "tools": [ + {"type": "web_fetch_20250910", "name": "web_fetch"}, + {"type": "function", "name": "get_weather", "parameters": {"type": "object"}}, + ] + }, + custom_llm_provider="anthropic", + ) + + events: Final = await _collect_events(iterator, sync_mode) + + completed: Final = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + assert [ + (item.id, item.status, item.action.model_dump()) + for item in completed.response.output + if item.type == "web_search_call" + ] == [("ws_srvtoolu_01Fetch", "completed", {"type": "open_page", "url": "https://example.com/"})] + assert [item.name for item in completed.response.output if item.type == "function_call"] == ["get_weather"] + assert not any( + event.item.name == "web_fetch" + for event in events + if getattr(event, "type", None) == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + and event.item.type == "function_call" + ) + + def test_tool_calls_present_only_in_final_response_are_emitted_before_completed(): iterator = LiteLLMCompletionStreamingIterator( model="test-model", From 8fedc17566e01d45c7babdf560d971b061965c96 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:33:21 -0700 Subject: [PATCH 3/9] fix(responses): keep Claude server web calls without a result as hosted items A non-streamed Claude turn that calls web_search or web_fetch next to a client tool can return the server call before its result. Build an in_progress web_search_call for it, so the Responses bridge no longer returns it as a client function_call --- litellm/llms/anthropic/chat/transformation.py | 41 +++++++------- .../test_litellm_completion_responses.py | 55 ++++++++++++++++--- 2 files changed, 69 insertions(+), 27 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 2eb9d06f1c5..7c6ccca7a21 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -7,6 +7,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast import httpx +from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -265,6 +266,16 @@ def _hosted_web_tool_use(block: object) -> tuple[str, str, object] | None: return None +def _hosted_web_call(call_id: str, tool_name: str, tool_input: object, result: object) -> ResponseFunctionWebSearch: + return build_web_search_call( + tool_id=call_id, + tool_input=tool_input, + result=result, + status=None if result is not None else "in_progress", + tool_name=tool_name, + ) + + class _AnthropicUsageIteration(TypedDict, total=False): """One entry of the ``usage.iterations`` array on an Anthropic response.""" @@ -2913,25 +2924,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self, web_search_results: Sequence[object], completion_response: Mapping[str, object], - ) -> list[object]: + ) -> list[ResponseFunctionWebSearch]: content: Final = completion_response.get("content") blocks: Final = content if isinstance(content, Sequence) else () - server_tool_uses: Final = { - call_id: (tool_name, tool_input) - for call_id, tool_name, tool_input in filter(None, map(_hosted_web_tool_use, blocks)) + results_by_call: Final = { + (tool_use_id, result.get("type")): result + for result in web_search_results + if isinstance(result, dict) and isinstance((tool_use_id := result.get("tool_use_id")), str) } return [ - build_web_search_call( - tool_id=tool_use_id, - tool_input=server_tool_uses[tool_use_id][1], - result=result, - tool_name=server_tool_uses[tool_use_id][0], - ) - for result in web_search_results - if isinstance(result, dict) - and isinstance((tool_use_id := result.get("tool_use_id")), str) - and tool_use_id in server_tool_uses - and result.get("type") == f"{server_tool_uses[tool_use_id][0]}_tool_result" + _hosted_web_call(call_id, tool_name, tool_input, results_by_call.get((call_id, f"{tool_name}_tool_result"))) + for call_id, tool_name, tool_input in filter(None, map(_hosted_web_tool_use, blocks)) ] def _build_provider_specific_fields( @@ -2955,10 +2958,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if web_search_results is not None: provider_specific_fields["web_search_results"] = web_search_results - provider_specific_fields["web_search_calls"] = self._build_web_search_calls( - web_search_results, - completion_response, - ) + + web_search_calls: Final = self._build_web_search_calls(web_search_results or (), completion_response) + if web_search_calls: + provider_specific_fields["web_search_calls"] = web_search_calls if tool_results is not None: provider_specific_fields["tool_results"] = tool_results diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 06ffa721cd4..2c827588668 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -31,7 +31,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.types.llms.base import HiddenParams -from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse from litellm.types.responses.main import OutputToolSearchCall, build_web_search_call from litellm.types.utils import ( ChatCompletionMessageToolCall, @@ -4952,6 +4952,12 @@ def test_function_call_tool_id_falls_back_to_unique_id_for_degenerate_call_id(): assert convert(openai)["id"] == "call_tokyo" +_WEATHER_TOOL_USE: Final = { + "type": "tool_use", + "id": "toolu_01Weather", + "name": "get_weather", + "input": {"city": "Paris"}, +} _FETCHED_URL: Final = "https://example.com/" _BLOCKED_URL: Final = "https://blocked.invalid/" _ANTHROPIC_WEB_FETCH_TURN: Final[tuple[dict[str, object], ...]] = ( @@ -4972,11 +4978,11 @@ _ANTHROPIC_WEB_FETCH_TURN: Final[tuple[dict[str, object], ...]] = ( "content": {"type": "web_fetch_tool_result_error", "error_code": "url_not_allowed"}, }, {"type": "text", "text": "The page is titled Example."}, - {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}}, + _WEATHER_TOOL_USE, ) -def test_anthropic_web_fetch_is_a_hosted_open_page_call_on_responses() -> None: +def _anthropic_turn_on_responses(content: Sequence[Mapping[str, object]]) -> ResponsesAPIResponse: from unittest.mock import MagicMock import httpx @@ -4988,10 +4994,10 @@ def test_anthropic_web_fetch_is_a_hosted_open_page_call_on_responses() -> None: "type": "message", "role": "assistant", "model": "claude-sonnet-5-5", - "content": list(_ANTHROPIC_WEB_FETCH_TURN), + "content": list(content), "stop_reason": "tool_use", "stop_sequence": None, - "usage": {"input_tokens": 100, "output_tokens": 50, "server_tool_use": {"web_fetch_requests": 2}}, + "usage": {"input_tokens": 100, "output_tokens": 50}, } raw_response: Final = MagicMock(spec=httpx.Response) raw_response.json.return_value = anthropic_response @@ -5004,11 +5010,11 @@ def test_anthropic_web_fetch_is_a_hosted_open_page_call_on_responses() -> None: json_mode=False, prefix_prompt=None, ) - - response: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( - request_input="Fetch two pages and check the weather in Paris.", + return LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Use the web and check the weather in Paris.", responses_api_request={ "tools": [ + {"type": "web_search_20250305", "name": "web_search"}, {"type": "web_fetch_20250910", "name": "web_fetch"}, {"type": "function", "name": "get_weather", "parameters": {"type": "object"}}, ] @@ -5016,6 +5022,10 @@ def test_anthropic_web_fetch_is_a_hosted_open_page_call_on_responses() -> None: chat_completion_response=chat_response, ) + +def test_anthropic_web_fetch_is_a_hosted_open_page_call_on_responses() -> None: + response: Final = _anthropic_turn_on_responses(_ANTHROPIC_WEB_FETCH_TURN) + assert [ (item.id, item.status, item.action.model_dump()) for item in response.output if item.type == "web_search_call" ] == [ @@ -5025,6 +5035,35 @@ def test_anthropic_web_fetch_is_a_hosted_open_page_call_on_responses() -> None: assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"] +@pytest.mark.parametrize( + ("server_tool_use", "expected_action"), + [ + ( + { + "type": "server_tool_use", + "id": "srvtoolu_01Pending", + "name": "web_fetch", + "input": {"url": _FETCHED_URL}, + }, + {"type": "open_page", "url": _FETCHED_URL}, + ), + ( + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}}, + {"type": "search", "query": "python"}, + ), + ], +) +def test_anthropic_server_web_call_without_a_result_is_an_in_progress_hosted_call_on_responses( + server_tool_use: Mapping[str, object], expected_action: Mapping[str, object] +) -> None: + response: Final = _anthropic_turn_on_responses((server_tool_use, _WEATHER_TOOL_USE)) + + web_calls: Final = [item for item in response.output if item.type == "web_search_call"] + assert [(item.id, item.status) for item in web_calls] == [("ws_srvtoolu_01Pending", "in_progress")] + assert expected_action.items() <= web_calls[0].action.model_dump().items() + assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"] + + class TestHostedWebSearchReplay: def test_emitted_hosted_search_output_round_trips_with_client_tool_result(self) -> None: search_result: Final = { From d93d26ddecc823221ca6c4fa64168bda16b7536a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:22:13 -0700 Subject: [PATCH 4/9] fix(anthropic): keep pending server calls native and hide server calls from client tool paths Replaying a server call that has no result yet now sends it back as server_tool_use, so Anthropic accepts the next turn. The tool permission guardrail checks server calls again and rewrite mode removes a denied one. The /v1/messages adapter keeps Claude's server blocks, and the Responses bridge never returns a server call as a client function_call. --- .../prompt_templates/factory.py | 15 ++ litellm/llms/anthropic/chat/handler.py | 9 + .../messages/adapters/transformation.py | 22 +++ .../guardrail_hooks/tool_permission.py | 20 +- .../streaming_iterator.py | 19 +- .../transformation.py | 17 +- .../test_anthropic_chat_transformation.py | 27 +++ ...hropic_messages_adapters_transformation.py | 53 ++++++ .../guardrail_hooks/test_tool_permission.py | 75 ++++++++ .../test_litellm_completion_responses.py | 41 ++++ .../test_streaming_iterator_transformation.py | 180 ++++++++++++++---- 11 files changed, 429 insertions(+), 49 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index e500c5f426f..80ab0ca5a9f 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2029,6 +2029,14 @@ def _assistant_tool_calls_with_server_tool_calls( ) +def _server_tool_call_ids(message: object) -> frozenset[str]: + from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params + + return frozenset( + tool_call["id"] for tool_call in anthropic_server_tool_call_params(message) if tool_call["id"] is not None + ) + + def _server_tool_answers_replaced_after(messages: Sequence[object], index: int) -> frozenset[str]: kept_server_result_ids: Final = _tool_call_ids_with_a_kept_server_result(messages[index]) if not kept_server_result_ids: @@ -2092,6 +2100,7 @@ def convert_to_anthropic_tool_invoke( web_search_results: Sequence[object] | None = None, tool_results: Sequence[object] | None = None, answered_tool_ids: Container[str] = frozenset(), + server_tool_call_ids: Container[str] = frozenset(), ) -> list[_AnthropicToolInvokeItem]: """ OpenAI tool invokes: @@ -2169,6 +2178,10 @@ def convert_to_anthropic_tool_invoke( anthropic_tool_invoke.append(server_tool_result) elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids: continue + elif tool_id in server_tool_call_ids and tool_id not in answered_tool_ids: + anthropic_tool_invoke.append( + {"type": "server_tool_use", "id": tool_id, "name": tool_name, "input": tool_input} + ) else: sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id) _anthropic_tool_use_param = AnthropicMessagesToolUseParam( @@ -3161,6 +3174,7 @@ def anthropic_messages_pt( web_search_results=_web_search_results_tc, tool_results=_tool_results_tc, answered_tool_ids=client_answered_tool_ids, + server_tool_call_ids=_server_tool_call_ids(assistant_content_block), ) # Group tool invoke results into (server_tool_use, result) pairs @@ -3326,6 +3340,7 @@ def anthropic_messages_pt( web_search_results=_web_search_results, tool_results=_tool_results, answered_tool_ids=client_answered_tool_ids, + server_tool_call_ids=_server_tool_call_ids(assistant_content_block), ) # Prevent "tool_use ids must be unique" errors by filtering duplicates diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 93c01ec6ae0..e1caa7c4b4a 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -1128,6 +1128,15 @@ class ModelResponseIterator: except (json.JSONDecodeError, TypeError): pass server_tool_input: Final = self._server_tool_inputs.get(self._current_server_tool_id, {}) + if self._current_server_tool_id in self._web_search_calls: + self._web_search_calls[self._current_server_tool_id] = build_web_search_call( + self._current_server_tool_id, + server_tool_input, + {"content": []}, + status="in_progress", + tool_name=self._current_server_tool_name, + ) + provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot() self._server_tool_calls = ( *self._server_tool_calls, ChatCompletionToolCallChunk( diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 1ed552d46e3..e924934637a 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -45,6 +45,25 @@ def _thought_signature(provider_specific_fields: object) -> str | None: return signature if isinstance(signature, str) else None +def _object_sequence(value: object) -> Sequence[object] | None: + return value if isinstance(value, (list, tuple)) else None + + +def _server_tool_blocks(message: object) -> tuple[Mapping[str, object], ...]: + server_tool_calls: Final = anthropic_server_tool_call_params(message) + if not server_tool_calls: + return () + fields: Final = _as_string_mapping(_optional_attr(message, "provider_specific_fields")) or {} + return tuple( + convert_to_anthropic_tool_invoke( + server_tool_calls, + web_search_results=_object_sequence(fields.get("web_search_results")), + tool_results=_object_sequence(fields.get("tool_results")), + server_tool_call_ids=frozenset(tool_call["id"] for tool_call in server_tool_calls), + ) + ) + + def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]: fields: Final = _as_string_mapping(provider_specific_fields) raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None @@ -129,11 +148,13 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, + convert_to_anthropic_tool_invoke, ) from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, ) from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, eager_input_streaming_flag, is_empty_unsigned_thinking_block, normalize_anthropic_tool_use_id, @@ -1410,6 +1431,7 @@ class LiteLLMAnthropicMessagesAdapter: ).model_dump() ) + new_content.extend(dict(block) for block in _server_tool_blocks(choice.message)) # Handle text content if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks): new_content.append( diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index fa92a0c90e0..541e49c6751 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -19,6 +19,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params, anthropic_server_tool_calls from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import ( add_guardrail_to_applied_guardrails_header, @@ -515,7 +516,7 @@ class ToolPermissionGuardrail(CustomGuardrail): for choice_index, choice in enumerate(response.choices): if isinstance(choice, Choices): - for tool in choice.message.tool_calls or []: + for tool in (*(choice.message.tool_calls or ()), *anthropic_server_tool_calls(choice.message)): tool_calls.append(tool) legacy_tool_call = self._legacy_function_call_to_tool_call( getattr(choice.message, "function_call", None), choice_index @@ -784,6 +785,23 @@ class ToolPermissionGuardrail(CustomGuardrail): choice.message.tool_calls = filtered_tool_calls if filtered_tool_calls else None + if (server_tool_calls := anthropic_server_tool_call_params(choice.message)) and any( + server_tool_call["id"] in error_results for server_tool_call in server_tool_calls + ): + error_messages.extend( + error_results[server_tool_call["id"]].content + for server_tool_call in server_tool_calls + if server_tool_call["id"] in error_results + ) + choice.message.provider_specific_fields = { + **(choice.message.provider_specific_fields or {}), + "server_tool_calls": [ + server_tool_call + for server_tool_call in server_tool_calls + if server_tool_call["id"] not in error_results + ], + } + legacy_tool_call = self._legacy_function_call_to_tool_call( getattr(choice.message, "function_call", None), choice_index ) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index c9b398b054a..6d0cbec8929 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -471,16 +471,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): try: message: Final = litellm_complete_object.choices[0].message - tool_calls = [ - *anthropic_server_tool_calls(message), - *(getattr(message, "tool_calls", None) or ()), - ] + server_tool_calls: Final = anthropic_server_tool_calls(message) + tool_calls: Final = [*server_tool_calls, *(getattr(message, "tool_calls", None) or ())] except Exception: - tool_calls = None + return if not tool_calls: return + server_tool_call_ids: Final = frozenset(tool_call.id for tool_call in server_tool_calls) + for tc in tool_calls: call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None) if not call_id_raw: @@ -506,6 +506,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if self._is_tool_search_call(call_id, fn_name): self._queue_tool_search_call_done(call_id, output_index, fn_args) continue + if call_id in server_tool_call_ids: + continue # Track if this is a new tool call that wasn't streamed is_new_tool_call = call_id not in self._tool_item_id_by_call_id @@ -580,10 +582,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): } ), ) + finished_events: Final = ( + () + if item.status == "in_progress" + else ((ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),) + ) for event_type, event_class in ( (ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent), (ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent), - (ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent), + *finished_events, ): self._sequence_number += 1 event = event_class(type=event_type, output_index=output_index, item_id=item.id) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 03a1411bba1..7c6d99b9253 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -3370,7 +3370,22 @@ class LiteLLMCompletionResponsesConfig: for item in responses_output ] - return responses_output + server_tool_call_ids: Final = LiteLLMCompletionResponsesConfig._server_tool_call_ids(choices) + return [ + item + for item in responses_output + if not ( + getattr(item, "type", None) == "function_call" + and getattr(item, "call_id", None) in server_tool_call_ids + ) + ] + + @staticmethod + def _server_tool_call_ids(choices: Sequence[Choices]) -> frozenset[str]: + server_tool_calls: Final = chain.from_iterable( + anthropic_server_tool_calls(choice.message) for choice in choices + ) + return frozenset(tool_call.id for tool_call in server_tool_calls) @staticmethod def _extract_tool_result_output_items( diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index e274a998533..255162b3cd9 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -3735,6 +3735,33 @@ def test_server_tool_result_round_trips_right_after_its_call( assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"] +_FETCH_STILL_WAITING_ON_ITS_RESULT: Final = { + "type": "server_tool_use", + "id": "srvtoolu_01Fetch", + "name": "web_fetch", + "input": {"url": "https://example.com/"}, +} + + +@pytest.mark.parametrize("modify_params", [False, True]) +def test_server_call_still_waiting_on_its_result_is_replayed_as_a_server_call( + modify_params: bool, monkeypatch: pytest.MonkeyPatch +): + content: Final = [_FETCH_STILL_WAITING_ON_ITS_RESULT, _WEATHER_CALL] + message: Final = _parsed_assistant_message(content, "tool_use") + monkeypatch.setattr(litellm, "modify_params", modify_params) + + replayed: Final = _replayed( + {"role": "user", "content": "Fetch example.com, then tell me the weather in Paris"}, + message.model_dump(), + {"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C"}, + ) + + assert replayed[1]["content"] == content + assert [turn["role"] for turn in replayed] == ["user", "assistant", "user"] + assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"] + + _PARIS_TEXT: Final = "Paris is the capital of France." _PARIS_ANSWER: Final[Mapping[str, str]] = {"type": "text", "text": _PARIS_TEXT} diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index 4e4b89b764b..ed7e9c8772b 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -80,6 +80,59 @@ def test_native_compaction_response_roundtrip(text: str | None, count: int, expe assert result["stop_reason"] == expected_stop +_SERVER_AND_CLIENT_TOOL_TURN: Final[tuple[dict[str, object], ...]] = ( + { + "type": "server_tool_use", + "id": "srvtoolu_01Fetch", + "name": "web_fetch", + "input": {"url": "https://example.com/"}, + }, + { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_01Fetch", + "content": {"type": "web_fetch_result", "url": "https://example.com/"}, + }, + {"type": "server_tool_use", "id": "srvtoolu_02Search", "name": "web_search", "input": {"query": "Paris"}}, + {"type": "text", "text": "The page is titled Example."}, + {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}}, +) + + +def test_claude_server_tool_blocks_survive_the_round_trip_through_chat_completions() -> None: + from unittest.mock import MagicMock + + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + anthropic_response: Final = { + "id": "msg_01Fetch", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": list(_SERVER_AND_CLIENT_TOOL_TURN), + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 100, "output_tokens": 50}, + } + raw_response: Final = MagicMock(spec=httpx.Response) + raw_response.json.return_value = anthropic_response + raw_response.status_code = 200 + raw_response.headers = {} + chat_response: Final = AnthropicConfig().transform_parsed_response( + completion_response=anthropic_response, + raw_response=raw_response, + model_response=ModelResponse(), + json_mode=False, + prefix_prompt=None, + ) + + result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(chat_response) + + assert [(block["type"], block.get("id") or block.get("tool_use_id")) for block in result["content"]] == [ + (block["type"], block.get("id") or block.get("tool_use_id")) for block in _SERVER_AND_CLIENT_TOOL_TURN + ] + assert result["stop_reason"] == "tool_use" + + def test_translate_chat_refusal_to_anthropic_response(): response = ModelResponse( id="chatcmpl-refusal", diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py index 2ee4c8c72c0..1fac7e7a4c9 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py @@ -27,11 +27,21 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( from litellm.types.utils import ( ChatCompletionMessageToolCall, Choices, + Delta, ModelResponse, ModelResponseStream, + StreamingChoices, ) +_SERVER_WEB_SEARCH_CALL: Final = { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "weather in Paris"}'}, +} +_ALLOWED_BASH_CALL: Final = {"id": "toolu_01Bash", "type": "function", "function": {"name": "Bash", "arguments": "{}"}} + + class TestToolPermissionGuardrail: """Test class for Tool Permission Guardrail functionality""" @@ -952,6 +962,71 @@ class TestToolPermissionGuardrail: assert "Permission denied" in choice.message.content + def _server_search_response(self) -> ModelResponse: + return ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message={ + "content": "Searching.", + "tool_calls": [_ALLOWED_BASH_CALL], + "provider_specific_fields": {"server_tool_calls": [_SERVER_WEB_SEARCH_CALL]}, + }, + ) + ] + ) + + @pytest.mark.asyncio + async def test_denied_server_tool_call_is_blocked(self): + with patch.object(self.guardrail, "should_run_guardrail", return_value=True): + with pytest.raises(GuardrailRaisedException): + await self.guardrail.async_post_call_success_hook( + data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response() + ) + + @pytest.mark.asyncio + async def test_denied_server_tool_call_in_a_chat_stream_is_blocked(self): + async def _stream(): + yield ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta( + content="Searching.", + provider_specific_fields={"server_tool_calls": [{**_SERVER_WEB_SEARCH_CALL, "index": 0}]}, + ), + finish_reason="stop", + ) + ] + ) + + with patch.object(self.guardrail, "should_run_guardrail", return_value=True): + with pytest.raises(GuardrailRaisedException): + async for _ in self.guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_stream(), request_data={} + ): + pass + + @pytest.mark.asyncio + async def test_rewrite_mode_removes_the_denied_server_tool_call(self): + rewriting: Final = ToolPermissionGuardrail( + guardrail_name="test-tool-permission", + rules=self.test_rules, + default_action="deny", + on_disallowed_action="rewrite", + ) + + with patch.object(rewriting, "should_run_guardrail", return_value=True): + result: Final = await rewriting.async_post_call_success_hook( + data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response() + ) + + message: Final = result.choices[0].message + assert message.provider_specific_fields["server_tool_calls"] == [] + assert [tool_call.id for tool_call in message.tool_calls] == [_ALLOWED_BASH_CALL["id"]] + assert "Permission denied" in message.content + + class TestToolPermissionGuardrailIntegration: """Integration tests for Tool Permission Guardrail""" diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 2c827588668..618a33339bf 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -5064,6 +5064,47 @@ def test_anthropic_server_web_call_without_a_result_is_an_in_progress_hosted_cal assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"] +def test_anthropic_code_execution_is_never_a_client_function_call_on_responses() -> None: + response: Final = _anthropic_turn_on_responses( + ( + { + "type": "server_tool_use", + "id": "srvtoolu_01Bash", + "name": "bash_code_execution", + "input": {"command": "ls"}, + }, + { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01Bash", + "content": { + "type": "bash_code_execution_result", + "stdout": "notes.txt\n", + "stderr": "", + "return_code": 0, + }, + }, + { + "type": "server_tool_use", + "id": "srvtoolu_02Edit", + "name": "text_editor_code_execution", + "input": {"command": "view", "path": "notes.txt"}, + }, + { + "type": "server_tool_use", + "id": "srvtoolu_03Bash", + "name": "bash_code_execution", + "input": {"command": "pwd"}, + }, + _WEATHER_TOOL_USE, + ) + ) + + assert [(item.type, getattr(item, "call_id", item.id)) for item in response.output if item.type != "message"] == [ + ("code_interpreter_call", "srvtoolu_01Bash"), + ("function_call", "toolu_01Weather"), + ] + + class TestHostedWebSearchReplay: def test_emitted_hosted_search_output_round_trips_with_client_tool_result(self) -> None: search_result: Final = { diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index db7c54c52ca..f1f6597130b 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -355,53 +355,59 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to ) -def _anthropic_web_fetch_stream_events() -> tuple[dict[str, object], ...]: +_STREAMED_WEB_FETCH_CALL: Final[tuple[dict[str, object], ...]] = ( + { + "type": "message_start", + "message": { + "id": "msg_01Fetch", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch"}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com/"}'}, + }, + {"type": "content_block_stop", "index": 0}, +) +_STREAMED_WEB_FETCH_RESULT: Final[tuple[dict[str, object], ...]] = ( + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_01Fetch", + "content": {"type": "web_fetch_result", "url": "https://example.com/"}, + }, + }, + {"type": "content_block_stop", "index": 1}, +) + + +def _streamed_text_then_weather_call(first_index: int) -> tuple[dict[str, object], ...]: return ( - { - "type": "message_start", - "message": { - "id": "msg_01Fetch", - "type": "message", - "role": "assistant", - "content": [], - "usage": {"input_tokens": 10, "output_tokens": 1}, - }, - }, + {"type": "content_block_start", "index": first_index, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": first_index, "delta": {"type": "text_delta", "text": "Fetched."}}, + {"type": "content_block_stop", "index": first_index}, { "type": "content_block_start", - "index": 0, - "content_block": {"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch"}, - }, - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com/"}'}, - }, - {"type": "content_block_stop", "index": 0}, - { - "type": "content_block_start", - "index": 1, - "content_block": { - "type": "web_fetch_tool_result", - "tool_use_id": "srvtoolu_01Fetch", - "content": {"type": "web_fetch_result", "url": "https://example.com/"}, - }, - }, - {"type": "content_block_stop", "index": 1}, - {"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}}, - {"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "Fetched."}}, - {"type": "content_block_stop", "index": 2}, - { - "type": "content_block_start", - "index": 3, + "index": first_index + 1, "content_block": {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {}}, }, { "type": "content_block_delta", - "index": 3, + "index": first_index + 1, "delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'}, }, - {"type": "content_block_stop", "index": 3}, + {"type": "content_block_stop", "index": first_index + 1}, { "type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": None}, @@ -410,13 +416,52 @@ def _anthropic_web_fetch_stream_events() -> tuple[dict[str, object], ...]: ) +def _anthropic_web_fetch_stream_events(fetch_finished: bool) -> tuple[dict[str, object], ...]: + if fetch_finished: + return (*_STREAMED_WEB_FETCH_CALL, *_STREAMED_WEB_FETCH_RESULT, *_streamed_text_then_weather_call(2)) + return (*_STREAMED_WEB_FETCH_CALL, *_streamed_text_then_weather_call(1)) + + @pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize( + ("fetch_finished", "expected_status", "expected_search_events"), + [ + pytest.param( + True, + "completed", + [ + ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, + ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + ], + id="fetch_finished_in_this_turn", + ), + pytest.param( + False, + "in_progress", + [ + ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, + ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + ], + id="fetch_still_waiting_on_its_result", + ), + ], +) @pytest.mark.asyncio -async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call(sync_mode: bool) -> None: +async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call( + sync_mode: bool, + fetch_finished: bool, + expected_status: str, + expected_search_events: list[ResponsesAPIStreamEvents], +) -> None: from litellm.llms.anthropic.chat.handler import ModelResponseIterator parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False) - chunks: Final = [parser.chunk_parser(event) for event in _anthropic_web_fetch_stream_events()] + chunks: Final = [parser.chunk_parser(event) for event in _anthropic_web_fetch_stream_events(fetch_finished)] iterator: Final = LiteLLMCompletionStreamingIterator( model="claude-sonnet-5-5", litellm_custom_stream_wrapper=_FakeStreamWrapper(chunks), @@ -439,7 +484,12 @@ async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call(sync_mode (item.id, item.status, item.action.model_dump()) for item in completed.response.output if item.type == "web_search_call" - ] == [("ws_srvtoolu_01Fetch", "completed", {"type": "open_page", "url": "https://example.com/"})] + ] == [("ws_srvtoolu_01Fetch", expected_status, {"type": "open_page", "url": "https://example.com/"})] + assert [ + event.type + for event in events + if getattr(event, "item_id", getattr(getattr(event, "item", None), "id", None)) == "ws_srvtoolu_01Fetch" + ] == expected_search_events assert [item.name for item in completed.response.output if item.type == "function_call"] == ["get_weather"] assert not any( event.item.name == "web_fetch" @@ -449,6 +499,54 @@ async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call(sync_mode ) +_STREAMED_PENDING_CODE_EXECUTION: Final[tuple[dict[str, object], ...]] = ( + _STREAMED_WEB_FETCH_CALL[0], + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_01Bash", "name": "bash_code_execution"}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"command": "ls"}'}, + }, + {"type": "content_block_stop", "index": 0}, + *_streamed_text_then_weather_call(1), +) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_streamed_anthropic_code_execution_is_never_a_client_function_call(sync_mode: bool) -> None: + from litellm.llms.anthropic.chat.handler import ModelResponseIterator + + parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False) + iterator: Final = LiteLLMCompletionStreamingIterator( + model="claude-sonnet-5-5", + litellm_custom_stream_wrapper=_FakeStreamWrapper( + [parser.chunk_parser(event) for event in _STREAMED_PENDING_CODE_EXECUTION] + ), + request_input="List the files, then check the weather in Paris.", + responses_api_request={ + "tools": [ + {"type": "code_execution_20250825", "name": "code_execution"}, + {"type": "function", "name": "get_weather", "parameters": {"type": "object"}}, + ] + }, + custom_llm_provider="anthropic", + ) + + events: Final = await _collect_events(iterator, sync_mode) + + item_events: Final = [event for event in events if getattr(event, "item", None) is not None] + assert {event.item.call_id for event in item_events if event.item.type == "function_call"} == {"toolu_01Weather"} + completed: Final = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + assert [item.call_id for item in completed.response.output if item.type == "function_call"] == ["toolu_01Weather"] + + def test_tool_calls_present_only_in_final_response_are_emitted_before_completed(): iterator = LiteLLMCompletionStreamingIterator( model="test-model", From 1a30066945348d5e92583ea05e0f3fae4f60693c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:38:45 -0700 Subject: [PATCH 5/9] fix(anthropic): let guardrails, tool usage, tracing, and Responses replay see Claude's server calls Claude's server tool calls moved out of tool_calls into provider_specific_fields.server_tool_calls, so every reader of tool_calls stopped seeing them. The chat guardrail handler now passes them to output guardrails and writes rewrites back to server_tool_calls (stream and non-stream), tool usage and registry discovery count them, otel, Datadog and the Logs Tools panel list them, and a pending Claude web search or fetch item replayed to /v1/responses goes back to Anthropic as its own server_tool_use instead of running again --- .../integrations/datadog/datadog_llm_obs.py | 6 +- litellm/integrations/otel/mappers/utils.py | 11 +- litellm/llms/anthropic/chat/handler.py | 1 - .../chat/guardrail_translation/handler.py | 120 ++++++++++- litellm/proxy/db/spend_log_tool_index.py | 17 +- .../transformation.py | 56 ++++- litellm/types/responses/main.py | 10 +- .../datadog/test_datadog_llm_obs.py | 35 +++ .../otel/test_otel_v2_vendor_mappers.py | 39 ++++ .../test_openai_guardrail_handler.py | 200 ++++++++++++++++++ .../proxy/db/test_spend_log_tool_index.py | 31 +++ .../test_litellm_completion_responses.py | 69 ++++++ .../sections/ToolsSection/utils.test.ts | 46 ++++ .../detail/sections/ToolsSection/utils.ts | 25 ++- 14 files changed, 643 insertions(+), 23 deletions(-) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index a72023eaedf..79788d35854 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -44,6 +44,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.redact_messages import should_redact_message_logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -315,7 +316,10 @@ def _to_dd_tool_calls(message: Mapping[str, object]) -> tuple[ToolCall, ...]: tool_id=_text_field(tool_call, "id"), type=_text_field(tool_call, "type", "function"), ) - for tool_call in (raw_tool_calls if isinstance(raw_tool_calls, list) else ()) + for tool_call in ( + *anthropic_server_tool_call_params(message), + *(raw_tool_calls if isinstance(raw_tool_calls, list) else ()), + ) if isinstance(tool_call, dict) for function in [_mapping_field(tool_call, "function")] ) diff --git a/litellm/integrations/otel/mappers/utils.py b/litellm/integrations/otel/mappers/utils.py index 80ef2dd553a..bfd50fea5d5 100644 --- a/litellm/integrations/otel/mappers/utils.py +++ b/litellm/integrations/otel/mappers/utils.py @@ -12,6 +12,8 @@ from typing import Final from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue from litellm.integrations.otel.model.payloads import LLMCallSpanData, ToolDefinition +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params @dataclass(frozen=True, slots=True) @@ -141,12 +143,13 @@ def output_messages(data: LLMCallSpanData) -> list: def message_tool_calls(message: object) -> tuple[MessageToolCall, ...]: + tool_call_values: Final = ( + *anthropic_server_tool_call_params(message), + *parts_of(message_field(message, "tool_calls")), + ) if not isinstance(message, dict): return () - tool_calls: Final = message.get("tool_calls") - if not isinstance(tool_calls, (list, tuple)) or not tool_calls: - return () - return tuple(tool_call for value in tool_calls if (tool_call := _message_tool_call(value)) is not None) + return tuple(tool_call for value in tool_call_values if (tool_call := _message_tool_call(value)) is not None) def _message_tool_call(value: object) -> MessageToolCall | None: diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index bb02b9aab85..e414e333684 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -827,7 +827,6 @@ class ModelResponseIterator: return text, tool_use, thinking_blocks, provider_specific_fields, reasoning_content self.content_blocks.append(content_block) if "partial_json" in content_block["delta"]: - # Only emit tool calls if we're in a client tool_use block, server_tool_use runs on Anthropic's side # web_search_tool_result blocks also have input_json_delta but should not be treated as tool calls # See: https://github.com/BerriAI/litellm/issues/17254 if self.current_content_block_type == "tool_use": diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 3070bedd30d..d7c3aeca354 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -21,6 +21,7 @@ from collections.abc import Iterator, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Union, cast +from pydantic import TypeAdapter, ValidationError from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm @@ -29,6 +30,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i replayed_tool_search_tool_names, ) from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, StreamingScanKey, @@ -50,7 +52,12 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( stream_item_items, unappliable_request_rewrite, ) -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionAssistantToolCall, + ChatCompletionToolCallFunctionChunk, + ChatCompletionToolParam, +) from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( coerce_stream_holdback_value, ) @@ -436,9 +443,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation): text_task_mappings=text_task_mappings, tool_call_task_mappings=tool_call_task_mappings, ) + server_tool_call_tasks: Final = tuple(_server_tool_call_tasks(response.choices)) + tool_calls_with_server_calls: Final = [ + *tool_calls_to_check, + *(_server_tool_call_input(call) for _, call in server_tool_call_tasks), + ] # Step 2: Apply guardrail to all texts and tool calls in batch - if texts_to_check or tool_calls_to_check: + if texts_to_check or tool_calls_with_server_calls: # Use the real request_data if provided (proxy path), otherwise # create a standalone dict (SDK / direct-call path). if request_data is None: @@ -452,8 +464,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) if images_to_check: inputs["images"] = images_to_check - if tool_calls_to_check: - inputs["tool_calls"] = tool_calls_to_check + if tool_calls_with_server_calls: + inputs["tool_calls"] = tool_calls_with_server_calls # Include model information from the response if available if hasattr(response, "model") and response.model: inputs["model"] = response.model @@ -469,8 +481,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation): returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") guardrailed_tool_calls: Final[list[dict[str, object]]] = ( cast(list[dict[str, object]], returned_tool_calls) - if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check) - else tool_calls_to_check + if isinstance(returned_tool_calls, list) + and len(returned_tool_calls) == len(tool_calls_with_server_calls) + else tool_calls_with_server_calls ) # Step 3: Map guardrail responses back to original response structure @@ -488,6 +501,11 @@ class OpenAIChatCompletionsHandler(BaseTranslation): tool_calls=guardrailed_tool_calls, task_mappings=tool_call_task_mappings, ) + self._apply_guardrail_responses_to_output_server_tool_calls( + response=response, + guardrailed_tool_calls=guardrailed_tool_calls[len(tool_calls_to_check) :], + task_mappings=tuple(mapping for mapping, _ in server_tool_call_tasks), + ) verbose_proxy_logger.debug("OpenAI Chat Completions: Processed output response: %s", response) @@ -681,6 +699,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): pre_guardrail_tool_calls=pre_guardrail_tool_calls, guardrail_name=guardrail_to_apply.guardrail_name or "unknown", ) + self._write_ended_stream_server_tool_call_rewrites( + responses_so_far=responses_so_far, + guardrailed_response=model_response, + ) @staticmethod def _rebuild_ended_stream_per_choice( @@ -939,6 +961,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if choice.message.tool_calls and isinstance(choice.message.tool_calls, list): if len(choice.message.tool_calls) > 0: return True + if anthropic_server_tool_call_params(choice.message): + return True elif isinstance(response, ModelResponseStream): for streaming_choice in response.choices: if isinstance(streaming_choice, litellm.StreamingChoices): @@ -949,6 +973,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if streaming_choice.delta.tool_calls and isinstance(streaming_choice.delta.tool_calls, list): if len(streaming_choice.delta.tool_calls) > 0: return True + if anthropic_server_tool_call_params(streaming_choice.delta): + return True return False def _extract_output_text_images_and_tool_calls( @@ -1099,6 +1125,25 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if "name" in func_dict: existing_tool_call.function.name = func_dict["name"] + @staticmethod + def _apply_guardrail_responses_to_output_server_tool_calls( + response: "ModelResponse", + guardrailed_tool_calls: Sequence[object], + task_mappings: Sequence[tuple[int, int]], + ) -> None: + guardrailed_by_task: Final = MappingProxyType(dict(zip(task_mappings, guardrailed_tool_calls))) + for choice_idx, choice in enumerate(response.choices): + server_tool_calls = anthropic_server_tool_call_params(choice.message) + rewritten = tuple( + _rewritten_server_tool_call(server_tool_call, guardrailed_by_task.get((choice_idx, server_idx))) + for server_idx, server_tool_call in enumerate(server_tool_calls) + ) + if rewritten != server_tool_calls: + choice.message.provider_specific_fields = { + **(choice.message.provider_specific_fields or {}), + "server_tool_calls": list(rewritten), + } + @staticmethod def _string_choice_contents(response: "ModelResponse") -> tuple[str | None, ...]: return tuple( @@ -1211,6 +1256,32 @@ class OpenAIChatCompletionsHandler(BaseTranslation): for fragment in tail: fragment.function.arguments = "" + @staticmethod + def _write_ended_stream_server_tool_call_rewrites( + responses_so_far: Sequence["ModelResponseStream"], + guardrailed_response: "ModelResponse", + ) -> None: + guardrailed_by_id: Final = MappingProxyType( + { + (guardrailed_response.choices[choice_position].index, server_tool_call["id"]): server_tool_call + for (choice_position, _), server_tool_call in _server_tool_call_tasks(guardrailed_response.choices) + } + ) + if not guardrailed_by_id: + return + for response in responses_so_far: + for choice in response.choices: + streamed = anthropic_server_tool_call_params(choice.delta) + rewritten = tuple( + guardrailed_by_id.get((choice.index, server_tool_call["id"]), server_tool_call) + for server_tool_call in streamed + ) + if rewritten != streamed: + choice.delta.provider_specific_fields = { + **(choice.delta.provider_specific_fields or {}), + "server_tool_calls": list(rewritten), + } + async def _apply_guardrail_responses_to_output_streaming( self, responses: list["ModelResponseStream"], @@ -1400,7 +1471,42 @@ def _stream_chunk_choices(item: object) -> Sequence[object]: def _streamed_delta_tool_calls(delta: object) -> tuple[object, ...]: function_call: Final = stream_item_field(delta, "function_call") legacy: Final = () if function_call is None else (function_call,) - return stream_item_items(delta, "tool_calls") + legacy + return anthropic_server_tool_call_params(delta) + stream_item_items(delta, "tool_calls") + legacy + + +def _server_tool_call_tasks( + choices: Sequence[object], +) -> Iterator[tuple[tuple[int, int], ChatCompletionAssistantToolCall]]: + for choice_idx, choice in enumerate(choices): + if not isinstance(choice, Choices): + continue + for server_idx, server_tool_call in enumerate(anthropic_server_tool_call_params(choice.message)): + yield (choice_idx, server_idx), server_tool_call + + +def _server_tool_call_input(server_tool_call: ChatCompletionAssistantToolCall) -> dict[str, object]: + return { + "id": server_tool_call["id"], + "type": server_tool_call["type"], + "function": dict(server_tool_call["function"]), + } + + +_TOOL_CALL_FUNCTION_ADAPTER: Final = TypeAdapter(ChatCompletionToolCallFunctionChunk) + + +def _rewritten_server_tool_call( + server_tool_call: ChatCompletionAssistantToolCall, guardrailed: object +) -> ChatCompletionAssistantToolCall: + try: + function: Final = _TOOL_CALL_FUNCTION_ADAPTER.validate_python(stream_item_field(guardrailed, "function")) + except ValidationError: + return server_tool_call + return ChatCompletionAssistantToolCall( + id=server_tool_call["id"], + type=server_tool_call["type"], + function={**server_tool_call["function"], **function}, + ) def _released_choices(responses_so_far: Sequence[object]) -> Iterator[object]: diff --git a/litellm/proxy/db/spend_log_tool_index.py b/litellm/proxy/db/spend_log_tool_index.py index 591ad81a4e5..92181d546eb 100644 --- a/litellm/proxy/db/spend_log_tool_index.py +++ b/litellm/proxy/db/spend_log_tool_index.py @@ -63,11 +63,26 @@ def response_tool_call_names(completion_response: object) -> tuple[str, ...]: ) return tuple( stripped - for name in chain(called_names, _auto_executed_mcp_tool_names(completion_response)) + for name in chain( + _server_tool_call_names(completion_response), + called_names, + _auto_executed_mcp_tool_names(completion_response), + ) if isinstance(name, str) and (stripped := name.strip()) ) +def _server_tool_call_names(completion_response: object) -> tuple[str | None, ...]: + from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of + from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params + + server_tool_calls: Final = chain.from_iterable( + anthropic_server_tool_call_params(message_field(choice, "message")) + for choice in parts_of(message_field(completion_response, "choices")) + ) + return tuple(server_tool_call["function"].get("name") for server_tool_call in server_tool_calls) + + class _ExecutedTool(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 559ca6b7c23..ad03186c7a5 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -31,6 +31,7 @@ from openai.types.chat.chat_completion_named_tool_choice_param import ( from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch from openai.types.responses.response_compaction_item import ResponseCompactionItem from openai.types.responses.response_create_params import ResponseInputParam +from openai.types.responses.response_function_web_search import ActionOpenPage, ActionSearch from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam from openai.types.responses.tool_param import FunctionToolParam @@ -56,7 +57,12 @@ from litellm.litellm_core_utils.hidden_params import ( set_hidden_params, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls, supports_anthropic_cache_control +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + anthropic_server_tool_calls, + supports_anthropic_cache_control, + validated_object_mapping, +) from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control @@ -66,11 +72,12 @@ from litellm.responses.litellm_completion_transformation.session_handler import ResponsesSessionHandler, StoredTurns, ) -from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex +from litellm.types.llms.anthropic import ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, AnthropicToolSearchToolRegex from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, + ChatCompletionAssistantToolCall, ChatCompletionAudioObject, ChatCompletionCachedContent, ChatCompletionFileObject, @@ -114,6 +121,7 @@ from litellm.types.responses.main import ( OutputImageGenerationCall, OutputText, OutputToolSearchCall, + web_search_call_tool_id, ) from litellm.types.utils import ( ChatCompletionAnnotation, @@ -873,6 +881,29 @@ def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, obj return {key: {**_STR_KEY_DICT_ADAPTER.validate_python(metadata), CALLER_SESSION_ID_METADATA_KEY: chain_session}} +def _pending_claude_server_tool_call(search: ResponseFunctionWebSearch) -> ChatCompletionAssistantToolCall | None: + tool_id: Final = web_search_call_tool_id(search.id) + if search.status != "in_progress" or tool_id is None: + return None + if not tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX): + return None + match search.action: + case ActionSearch(query=str() as query): + return _server_tool_call(tool_id, "web_search", json.dumps({"query": query})) + case ActionOpenPage(url=str() as url): + return _server_tool_call(tool_id, "web_fetch", json.dumps({"url": url})) + case _: + return None + + +def _server_tool_call(tool_id: str, name: str, arguments: str) -> ChatCompletionAssistantToolCall: + return ChatCompletionAssistantToolCall( + id=tool_id, + type="function", + function=ChatCompletionToolCallFunctionChunk(name=name, arguments=arguments), + ) + + class LiteLLMCompletionResponsesConfig: @staticmethod def get_supported_openai_params(model: str) -> list: @@ -1778,10 +1809,22 @@ class LiteLLMCompletionResponsesConfig: message_cache_control: Final = ( None if isinstance(content, list) else new_cache_control or previous_cache_control ) + new_server_tool_calls: Final = anthropic_server_tool_call_params(new_message) + server_tool_call_fields: Final = ( + { + "provider_specific_fields": { + **(validated_object_mapping(last_message.get("provider_specific_fields")) or {}), + "server_tool_calls": [*anthropic_server_tool_call_params(last_message), *new_server_tool_calls], + } + } + if new_server_tool_calls + else {} + ) merged: Final = { **{key: value for key, value in last_message.items() if key != "cache_control"}, "content": content, **({} if message_cache_control is None else {"cache_control": message_cache_control}), + **server_tool_call_fields, } return cast( # cast-ok: preserves the assistant fields and content blocks ChatCompletionAssistantMessage, merged @@ -2223,10 +2266,15 @@ class LiteLLMCompletionResponsesConfig: content="Hosted web search: " + replay.model_dump_json(exclude_none=True), ) ] + hosted_search_text: Final = "Hosted web search: " + search.model_dump_json(exclude_none=True) + pending_server_tool_call: Final = _pending_claude_server_tool_call(search) + if pending_server_tool_call is None: + return [GenericChatCompletionMessage(role="assistant", content=hosted_search_text)] return [ - GenericChatCompletionMessage( + ChatCompletionResponseMessage( role="assistant", - content="Hosted web search: " + search.model_dump_json(exclude_none=True), + content=hosted_search_text, + provider_specific_fields={"server_tool_calls": [pending_server_tool_call]}, ) ] if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(input_item): diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 7b1fb0130d8..e9111eb8c99 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -92,6 +92,9 @@ def _is_tool_result_error(content: object, tool_name: str) -> bool: return False +WEB_SEARCH_CALL_ID_PREFIX: Final = "ws_" + + def build_web_search_call( tool_id: str, tool_input: object, @@ -101,13 +104,18 @@ def build_web_search_call( ) -> ResponseFunctionWebSearch: content: Final = _result_content(result) return ResponseFunctionWebSearch( - id=f"ws_{tool_id}", + id=f"{WEB_SEARCH_CALL_ID_PREFIX}{tool_id}", type="web_search_call", status=status or ("failed" if _is_tool_result_error(content, tool_name) else "completed"), action=_web_fetch_action(tool_input) if tool_name == "web_fetch" else _web_search_action(tool_input, content), ) +def web_search_call_tool_id(call_id: str) -> str | None: + tool_id: Final = call_id.removeprefix(WEB_SEARCH_CALL_ID_PREFIX) + return None if tool_id == call_id else tool_id + + class OutputImageGenerationCall(BaseLiteLLMOpenAIResponseObject): """An image generation call output""" diff --git a/tests/unit/integrations/datadog/test_datadog_llm_obs.py b/tests/unit/integrations/datadog/test_datadog_llm_obs.py index a7826e11483..dcc1567f5f8 100644 --- a/tests/unit/integrations/datadog/test_datadog_llm_obs.py +++ b/tests/unit/integrations/datadog/test_datadog_llm_obs.py @@ -110,6 +110,41 @@ def test_output_tool_calls_use_the_datadog_tool_call_schema(logger: DataDogLLMOb assert "function" not in message["tool_calls"][0] + +def test_claude_server_tool_calls_are_reported_before_client_tool_calls(logger: DataDogLLMObsLogger) -> None: + payload = build( + logger, + response_message={ + "role": "assistant", + "content": None, + "tool_calls": [ASSISTANT_TOOL_CALL], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "paris weather"}'}, + } + ] + }, + }, + ) + + assert payload["meta"]["output"]["messages"][0]["tool_calls"] == [ + { + "name": "web_search", + "arguments": {"query": "paris weather"}, + "tool_id": "srvtoolu_01Search", + "type": "function", + }, + { + "name": "get_weather", + "arguments": {"city": "Paris", "unit": "c"}, + "tool_id": "call_abc123", + "type": "function", + }, + ] + def test_tool_call_identifiers_that_are_not_strings_are_blanked_not_stringified(logger: DataDogLLMObsLogger) -> None: payload = build( logger, diff --git a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py index 57764bf902b..611e828cf84 100644 --- a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py @@ -174,6 +174,45 @@ def test_openinference_output_tool_calls_preserve_calls_in_attributes_and_value( ] +def test_openinference_output_lists_claude_server_tool_calls_before_client_tool_calls(): + data: Final = _llm_call( + choices_out=( + { + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_paris", + "type": "function", + "function": {"name": "lookup_weather", "arguments": '{"city": "Paris"}'}, + } + ], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "paris weather"}'}, + } + ] + }, + }, + }, + ) + ) + attrs: Final = OpenInferenceMapper().map(data) + assert {key: value for key, value in attrs.items() if ".tool_calls." in key} == { + "llm.output_messages.0.message.tool_calls.0.tool_call.id": "srvtoolu_01Search", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.name": "web_search", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": '{"query": "paris weather"}', + "llm.output_messages.0.message.tool_calls.1.tool_call.id": "call_paris", + "llm.output_messages.0.message.tool_calls.1.tool_call.function.name": "lookup_weather", + "llm.output_messages.0.message.tool_calls.1.tool_call.function.arguments": '{"city": "Paris"}', + } + + def test_openinference_responses_tool_calls_are_emitted_as_output_attributes(): data: Final = LLMCallSpanData.from_standard_logging_payload( _responses_payload( diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 5d71cbe9afc..fcc3b1e2b3c 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -144,6 +144,42 @@ class MockMisalignedToolCallGuardrail(CustomGuardrail): ) +def _server_tool_call(call_id: str, name: str, arguments: Mapping[str, object]) -> dict[str, object]: + return {"id": call_id, "type": "function", "function": {"name": name, "arguments": json.dumps(arguments)}} + + +def _tool_call_ids_names_and_arguments(tool_calls) -> list[tuple[str, str, str]]: + return [ + (tool_call["id"], tool_call["function"]["name"], tool_call["function"]["arguments"]) + for tool_call in tool_calls + ] + + +class QueryRedactingGuardrail(CustomGuardrail): + """Returns copied tool calls with any ``query`` argument redacted, leaving other calls as they came.""" + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + def redacted(tool_call): + arguments = json.loads(tool_call["function"]["arguments"]) + if "query" not in arguments: + return tool_call + return { + **tool_call, + "function": {**tool_call["function"], "arguments": json.dumps({**arguments, "query": "[REDACTED]"})}, + } + + return GenericGuardrailAPIInputs( + texts=inputs.get("texts", []), + tool_calls=[redacted(tool_call) for tool_call in inputs.get("tool_calls", [])], + ) + + class TestOpenAIChatCompletionsHandlerToolsInput: """Test input processing with tools (function definitions)""" @@ -906,6 +942,92 @@ class TestOpenAIChatCompletionsHandlerToolCallsOutput: assert tool_calls[0].function.arguments == first_args assert tool_calls[1].function.arguments == second_args + @pytest.mark.asyncio + async def test_output_guardrail_sees_a_response_that_only_carries_server_tool_calls(self): + handler = OpenAIChatCompletionsHandler() + guardrail = TestOpenAIChatCompletionsHandlerStreamingOutput._recording_guardrail() + response = ModelResponse( + model="claude-sonnet-4-5", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + content=None, + role="assistant", + provider_specific_fields={ + "server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})] + }, + ), + ) + ], + ) + + await handler.process_output_response(response, guardrail) + + assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "x"}') + ] + + @pytest.mark.asyncio + async def test_output_guardrail_rewrite_of_a_server_tool_call_lands_in_server_tool_calls_only(self): + handler = OpenAIChatCompletionsHandler() + weather_args = json.dumps({"city": "Paris", "query": "forecast"}) + response = ModelResponse( + model="claude-sonnet-4-5", + choices=[ + Choices( + finish_reason="tool_calls", + index=0, + message=Message( + content="Looking it up", + role="assistant", + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_1", + type="function", + function=Function(name="get_weather", arguments=json.dumps({"city": "Paris"})), + ) + ], + provider_specific_fields={ + "server_tool_calls": [ + _server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"}) + ] + }, + ), + ), + Choices( + finish_reason="tool_calls", + index=1, + message=Message( + content=None, + role="assistant", + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_2", + type="function", + function=Function(name="get_weather", arguments=weather_args), + ) + ], + ), + ), + ], + ) + + await handler.process_output_response(response, QueryRedactingGuardrail(guardrail_name="redact")) + + first, second = response.choices + assert _tool_call_ids_names_and_arguments(first.message.provider_specific_fields["server_tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}') + ] + assert [(call.id, call.function.arguments) for call in first.message.tool_calls] == [ + ("call_1", '{"city": "Paris"}') + ] + assert [(call.id, call.function.arguments) for call in second.message.tool_calls] == [ + ("call_2", '{"city": "Paris", "query": "[REDACTED]"}') + ] + assert not (second.message.provider_specific_fields or {}).get("server_tool_calls") + class MockPassThroughGuardrail(CustomGuardrail): """Mock guardrail that passes through without blocking - for testing streaming fallback behavior""" @@ -1462,6 +1584,61 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert chunks[0].choices[0].delta.content == "hello [MASKED]" assert chunks[1].choices[0].delta.content in (None, "") + @staticmethod + def _ended_server_tool_call_stream_chunks() -> list: + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + def chunk(delta: Delta, finish_reason: Optional[str] = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-123", + created=1234567890, + model="claude-sonnet-4-5", + object="chat.completion.chunk", + choices=[StreamingChoices(index=0, delta=delta, finish_reason=finish_reason)], + ) + + search = {**_server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"}), "index": 0} + return [ + chunk(Delta(content="Searching")), + chunk(Delta(content=None, provider_specific_fields={"server_tool_calls": [search]})), + chunk(Delta(content=None), finish_reason="stop"), + ] + + @pytest.mark.asyncio + async def test_ended_stream_guardrail_sees_streamed_server_tool_calls(self): + handler = OpenAIChatCompletionsHandler() + guardrail = self._recording_guardrail() + + await handler.process_output_streaming_response( + responses_so_far=self._ended_server_tool_call_stream_chunks(), + guardrail_to_apply=guardrail, + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "jane@example.com"}') + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_writes_server_tool_call_arguments_back_into_chunks(self): + handler = OpenAIChatCompletionsHandler() + chunks = self._ended_server_tool_call_stream_chunks() + + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=QueryRedactingGuardrail(guardrail_name="redact"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + delta = chunks[1].choices[0].delta + assert _tool_call_ids_names_and_arguments(delta.provider_specific_fields["server_tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}') + ] + assert delta.tool_calls is None + assert chunks[0].choices[0].delta.content == "Searching" + class TestUndecoratedGuardrailIsRecorded: """LIT-5983 regression: the handler calls apply_guardrail bare, so a custom guardrail @@ -2323,6 +2500,29 @@ class TestStreamingScanKey: assert len(ended_key.tool_calls) == 1 and "run_shell" in ended_key.tool_calls[0] assert ended_key.tool_calls_in_flight is False + def test_server_tool_calls_hold_the_window_and_enter_the_key_once_the_stream_has_ended(self): + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + handler = OpenAIChatCompletionsHandler() + server_chunk = ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta( + content=None, + provider_specific_fields={ + "server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})] + }, + ), + ) + ] + ) + open_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk]) + ended_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk, self._chunk(None, "stop")]) + assert open_key.tool_calls_in_flight is True + assert len(ended_key.tool_calls) == 1 and "web_search" in ended_key.tool_calls[0] + assert ended_key != open_key + def test_text_after_the_first_choice_finishes_still_changes_the_key(self): handler = OpenAIChatCompletionsHandler() first_done = [self._chunk("a", index=0), self._chunk("b", finish_reason="stop", index=0)] diff --git a/tests/unit/proxy/db/test_spend_log_tool_index.py b/tests/unit/proxy/db/test_spend_log_tool_index.py index af3268a7cc9..a3f400bbc15 100644 --- a/tests/unit/proxy/db/test_spend_log_tool_index.py +++ b/tests/unit/proxy/db/test_spend_log_tool_index.py @@ -23,6 +23,7 @@ from litellm.proxy.db.spend_log_tool_index import ( ) from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import ModelResponse from tests.unit.proxy.db.fake_prisma_engine import engine_call @@ -204,6 +205,36 @@ class TestResponseToolCallNames: def test_blank_names_are_dropped(self): assert response_tool_call_names(_response_with_tool_calls(" ", "real_tool")) == ("real_tool",) + @pytest.mark.parametrize("as_object", [False, True], ids=["dict", "model_response"]) + def test_server_tool_calls_of_every_choice_are_named_before_client_tool_calls(self, as_object: bool): + def tool_call(call_id: str, name: str) -> dict[str, object]: + return {"id": call_id, "type": "function", "function": {"name": name, "arguments": "{}"}} + + response: dict[str, object] = { + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [tool_call("call_1", "get_weather")], + "provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_01", "web_search")]}, + }, + }, + { + "index": 1, + "message": { + "role": "assistant", + "content": "done", + "provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_02", "web_fetch")]}, + }, + }, + ] + } + completion_response = ModelResponse(**response) if as_object else response + + assert response_tool_call_names(completion_response) == ("web_search", "web_fetch", "get_weather") + def test_responses_api_output_function_calls(self): # Regression: /v1/responses carries invocations in output[] items of # type function_call, not in choices; they must reach the rollup. diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 20c7d3076c1..aabf2e1d7cc 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -5216,6 +5216,75 @@ class TestHostedWebSearchReplay: assert messages[2]["content"] == "Paris is sunny." assert history == original + @pytest.mark.parametrize( + "leading_text", ((), ({"type": "text", "text": "Let me look that up."},)), ids=("alone", "after_text") + ) + @pytest.mark.parametrize( + ("server_tool_use", "expected_input"), + ( + ( + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_fetch", "input": {"url": _FETCHED_URL}}, + {"url": _FETCHED_URL}, + ), + ( + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}}, + {"query": "python"}, + ), + ), + ids=("web_fetch", "web_search"), + ) + def test_pending_claude_web_call_replays_to_anthropic_as_its_own_server_tool_use( + self, + leading_text: tuple[Mapping[str, object], ...], + server_tool_use: Mapping[str, object], + expected_input: Mapping[str, object], + ) -> None: + response: Final = _anthropic_turn_on_responses((*leading_text, server_tool_use, _WEATHER_TOOL_USE)) + history: Final = [ + {"role": "user", "content": "Use the web and check the weather in Paris."}, + *(item.model_dump(exclude_none=True) for item in response.output), + {"type": "function_call_output", "call_id": "toolu_01Weather", "output": "Paris is sunny."}, + ] + + anthropic_turns: Final = anthropic_messages_pt( + messages=LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=history, responses_api_request={} + ), + model="claude-sonnet-5-5", + llm_provider="anthropic", + ) + + assert [turn["role"] for turn in anthropic_turns] == ["user", "assistant", "user"] + assert [block for block in anthropic_turns[1]["content"] if block["type"] != "text"] == [ + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": server_tool_use["name"], "input": expected_input}, + {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}}, + ] + assert [(block["type"], block["tool_use_id"]) for block in anthropic_turns[2]["content"]] == [ + ("tool_result", "toolu_01Weather") + ] + + @pytest.mark.parametrize( + ("call_id", "status"), + (("ws_srvtoolu_01Fetch", "completed"), ("ws_68a1b2c3d4", "in_progress"), ("srvtoolu_01Fetch", "in_progress")), + ids=("completed-claude-call", "pending-non-claude-call", "pending-unprefixed-id"), + ) + def test_web_call_that_is_not_a_pending_claude_call_replays_as_text_only( + self, call_id: str, status: Literal["completed", "in_progress"] + ) -> None: + search: Final = ResponseFunctionWebSearch( + id=call_id, type="web_search_call", status=status, action=ActionOpenPage(type="open_page", url=_FETCHED_URL) + ) + + messages: Final = ( + LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=search.model_dump(exclude_none=True) + ) + ) + + assert messages == [ + {"role": "assistant", "content": "Hosted web search: " + search.model_dump_json(exclude_none=True)} + ] + @pytest.mark.parametrize( "action", ( diff --git a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts index 78372fe56d9..a6693794df6 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts @@ -131,6 +131,52 @@ describe("ToolsSection utils", () => { expect(result[1].callData).toBeUndefined(); }); + it("should mark Claude server tools as called from provider_specific_fields.server_tool_calls", () => { + const log: Partial = { + request_id: "test-server-tools", + proxy_server_request: { + tools: [ + { type: "web_search_20250305", name: "web_search", max_uses: 3 }, + { type: "function", function: { name: "get_weather", description: "Get weather" } }, + { type: "function", function: { name: "send_email", description: "Send email" } }, + ], + }, + response: { + choices: [ + { + message: { + tool_calls: [ + { + id: "toolu_01Weather", + type: "function", + function: { name: "get_weather", arguments: '{"location": "Paris"}' }, + }, + ], + provider_specific_fields: { + server_tool_calls: [ + { + id: "srvtoolu_01Search", + type: "function", + function: { name: "web_search", arguments: '{"query": "paris weather"}' }, + }, + ], + }, + }, + }, + ], + }, + }; + + const result = parseToolsFromLog(log as LogEntry); + + expect(result.map((tool) => [tool.name, tool.called, tool.callData?.id])).toEqual([ + ["web_search", true, "srvtoolu_01Search"], + ["get_weather", true, "toolu_01Weather"], + ["send_email", false, undefined], + ]); + expect(result[0].callData?.arguments).toEqual({ query: "paris weather" }); + }); + it("should handle string format request and response", () => { const log: Partial = { request_id: "test-5", diff --git a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts index 5e76a09c155..b1b2e6feb99 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts @@ -42,6 +42,24 @@ function extractToolsFromRequest(log: LogEntry): ToolDefinition[] { return []; } +interface ChatMessageToolCallFields { + tool_calls?: unknown; + provider_specific_fields?: { server_tool_calls?: unknown } | null; +} + +function toolCallList(value: unknown): ToolCall[] { + return Array.isArray(value) ? value : []; +} + +function extractChatMessageToolCalls(message: ChatMessageToolCallFields | undefined): ToolCall[] | null { + const serverToolCalls = message?.provider_specific_fields?.server_tool_calls; + const clientToolCalls = message?.tool_calls; + if (!Array.isArray(serverToolCalls) && !Array.isArray(clientToolCalls)) { + return null; + } + return [...toolCallList(serverToolCalls), ...toolCallList(clientToolCalls)]; +} + /** * Extract tool calls from response data */ @@ -53,10 +71,9 @@ function extractToolCallsFromResponse(log: LogEntry): ToolCall[] { // OpenAI format: response.choices[0].message.tool_calls const choices = responseData.choices; if (Array.isArray(choices) && choices.length > 0) { - const firstChoice = choices[0]; - const message = firstChoice.message; - if (message && Array.isArray(message.tool_calls)) { - return message.tool_calls; + const chatToolCalls = extractChatMessageToolCalls(choices[0].message); + if (chatToolCalls) { + return chatToolCalls; } } From 54756ae0552b1f7b3a619777f43d73253a1bc6d4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:37:50 -0700 Subject: [PATCH 6/9] fix(anthropic): let guardrails and loggers read Claude's server tool calls again Guardrails, tool policies, tracing loggers and the Responses guardrail translation read only client tool calls after server calls moved to provider_specific_fields. They now read server calls first, then client calls, through one shared helper, and the Responses guardrail treats Claude's web_search_call items as web_search or web_fetch calls. Non-streaming splits server calls by block type, as streaming does. --- litellm/integrations/arize/_utils.py | 10 +- litellm/integrations/helicone.py | 36 ++-- .../integrations/langfuse/langfuse_otel.py | 5 +- litellm/integrations/lunary.py | 5 +- litellm/integrations/newrelic/newrelic.py | 7 +- litellm/integrations/opentelemetry.py | 6 +- litellm/integrations/shadow_eval_logger.py | 8 +- .../prompt_templates/common_utils.py | 7 +- litellm/llms/anthropic/chat/transformation.py | 33 ++-- litellm/llms/anthropic/common_utils.py | 78 ++++++++- .../chat/guardrail_translation/handler.py | 35 ++-- .../guardrail_translation/handler.py | 91 ++++++++-- .../cato_networks/cato_networks.py | 50 ++++-- .../cisco_ai_defense/cisco_ai_defense.py | 29 +++- .../guardrails/guardrail_hooks/lasso/lasso.py | 27 ++- .../llm_shield_proxy/llm_shield_proxy.py | 20 ++- .../guardrail_hooks/microsoft_purview/base.py | 5 +- .../panw_prisma_airs/panw_prisma_airs.py | 21 ++- .../guardrails/guardrail_hooks/presidio.py | 88 +++++----- .../guardrail_hooks/repelloai/repelloai.py | 13 +- .../transformation.py | 24 ++- litellm/types/responses/main.py | 38 ++++- .../integrations/arize/test_arize_utils.py | 67 ++++++++ .../integrations/newrelic/test_newrelic.py | 20 +++ tests/unit/integrations/test_helicone.py | 45 ++++- tests/unit/integrations/test_langfuse_otel.py | 49 ++++++ tests/unit/integrations/test_lunary.py | 46 ++++- tests/unit/integrations/test_opentelemetry.py | 54 ++++++ .../integrations/test_shadow_eval_logger.py | 34 ++++ ...ore_utils_prompt_templates_common_utils.py | 29 ++++ .../test_anthropic_chat_transformation.py | 39 +++++ ...test_openai_responses_guardrail_handler.py | 159 ++++++++++++++++++ .../guardrail_hooks/test_cato_networks.py | 77 +++++++++ .../test_cisco_ai_defense_chat.py | 72 ++++++++ .../guardrails/guardrail_hooks/test_lasso.py | 142 ++++++++++++++++ .../guardrail_hooks/test_llm_shield_proxy.py | 49 ++++++ .../guardrail_hooks/test_microsoft_purview.py | 57 +++++++ .../guardrail_hooks/test_panw_prisma_airs.py | 81 +++++++++ .../guardrail_hooks/test_presidio.py | 42 +++++ .../guardrail_hooks/test_repelloai.py | 53 ++++++ 40 files changed, 1560 insertions(+), 191 deletions(-) diff --git a/litellm/integrations/arize/_utils.py b/litellm/integrations/arize/_utils.py index f719dacd148..fd912839c95 100644 --- a/litellm/integrations/arize/_utils.py +++ b/litellm/integrations/arize/_utils.py @@ -14,6 +14,7 @@ from litellm.litellm_core_utils.redact_messages import ( should_redact_message_logging, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls from litellm.types.utils import CallTypes, StandardLoggingMCPToolCall, StandardLoggingPayload if TYPE_CHECKING: @@ -663,12 +664,11 @@ def _to_plain_dict(value): def _get_tool_calls(message) -> list | None: - """Return ``message.tool_calls`` only when it's a non-empty list. - - Works for dicts and Pydantic message objects via ``_safe_get``. + """Return the message's tool calls, Claude's server tool calls first, or + ``None`` when there are none. Works for dicts and Pydantic message objects. """ - tool_calls: Final = _safe_get(message, "tool_calls") - return tool_calls if isinstance(tool_calls, list) and tool_calls else None + tool_calls: Final = tool_calls_with_server_tool_calls(message) + return list(tool_calls) if tool_calls else None class _NormalizedToolCallFunction(TypedDict): diff --git a/litellm/integrations/helicone.py b/litellm/integrations/helicone.py index 63acc562d98..a0052069271 100644 --- a/litellm/integrations/helicone.py +++ b/litellm/integrations/helicone.py @@ -10,6 +10,23 @@ from litellm.integrations.helicone_mock_client import ( create_mock_helicone_client, should_use_helicone_mock, ) +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls +from litellm.types.utils import ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall + + +def _tool_use_block( + tool_call: ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall, +) -> dict[str, object]: + match tool_call: + case ChatCompletionMessageCustomToolCall(custom=custom): + return {"type": "tool_use", "id": tool_call.id, "name": custom.name, "input": custom.input} + case _: + return { + "type": "tool_use", + "id": tool_call.id, + "name": tool_call.function.name, + "input": tool_call.function.arguments, + } class HeliconeLogger: @@ -57,23 +74,8 @@ class HeliconeLogger: message = choice["message"] content = [] - if "tool_calls" in message and message["tool_calls"]: - for tool_call in message["tool_calls"]: - function = tool_call.get("function") - custom = tool_call.get("custom") - if not function and not custom: - continue - name, tool_input = ( - (function["name"], function["arguments"]) if function else (custom["name"], custom["input"]) - ) - content.append( - { - "type": "tool_use", - "id": tool_call["id"], - "name": name, - "input": tool_input, - } - ) + if tool_calls := tool_calls_with_server_tool_calls(message): + content = [_tool_use_block(tool_call) for tool_call in tool_calls] elif "content" in message and message["content"]: content = [{"type": "text", "text": message["content"]}] diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index ee75e6501e8..0117348da66 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -26,6 +26,7 @@ from litellm.integrations.opentelemetry import ( from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls from litellm.secret_managers.main import str_to_bool from litellm.types.integrations.langfuse_otel import ( LangfuseSpanAttributes, @@ -605,8 +606,8 @@ def _extract_choices_output(response_obj: _Gettable) -> str | None: message: Final = _as_gettable(first_choice.get("message", {})) if message is None: return None - tool_calls: Final = message.get("tool_calls") - if isinstance(tool_calls, list) and tool_calls: + tool_calls: Final = tool_calls_with_server_tool_calls(message) + if tool_calls: transformed_tool_calls: Final = [ entry for tool_call in tool_calls if (entry := _transformed_tool_call(response_obj, tool_call)) is not None ] diff --git a/litellm/integrations/lunary.py b/litellm/integrations/lunary.py index 6c020c78c32..08acb2c80dc 100644 --- a/litellm/integrations/lunary.py +++ b/litellm/integrations/lunary.py @@ -6,6 +6,7 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Final import packaging +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls if TYPE_CHECKING: from openai.types.chat.chat_completion_message_custom_tool_call import Custom @@ -67,8 +68,8 @@ def parse_messages(input): } # Only add tool_calls and function_call to res if they are set - if message.get("tool_calls"): - serialized["tool_calls"] = parse_tool_calls(message.get("tool_calls")) + if tool_calls := tool_calls_with_server_tool_calls(message): + serialized["tool_calls"] = parse_tool_calls(tool_calls) return serialized diff --git a/litellm/integrations/newrelic/newrelic.py b/litellm/integrations/newrelic/newrelic.py index 3623335f599..efc796825e4 100644 --- a/litellm/integrations/newrelic/newrelic.py +++ b/litellm/integrations/newrelic/newrelic.py @@ -56,6 +56,7 @@ from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import has_attributes from litellm.litellm_core_utils.redact_messages import should_redact_message_logging +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus from litellm.types.integrations.newrelic import NewRelicInitParams from litellm.types.utils import Message, ModelResponse, StandardLoggingPayload @@ -479,11 +480,11 @@ class NewRelicLogger(CustomLogger): content: Final = message.get("content") # Handle tool calls - if message.get("tool_calls"): + if tool_calls := tool_calls_with_server_tool_calls(message): try: - return json.dumps(message["tool_calls"]) + return json.dumps([tool_call.model_dump() for tool_call in tool_calls]) except Exception: - return str(message["tool_calls"]) + return str(tool_calls) # Handle None or missing content if content is None: diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 0d0e309034a..cc8ee46d8bb 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -42,6 +42,7 @@ from litellm.litellm_core_utils.service_tier_utils import ( get_requested_service_tier, get_served_service_tier, ) +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls from litellm.secret_managers.main import get_secret_bool, str_to_bool from litellm.types.integrations.otel_span_attributes import SpanAttributes from litellm.types.services import ServiceLoggerPayload @@ -2736,9 +2737,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): for idx, choice in enumerate(response_obj.get("choices")): if choice.get("finish_reason"): message = choice.get("message") - tool_calls = message.get("tool_calls") - if tool_calls: - kv_pairs = OpenTelemetry._tool_calls_kv_pair(tool_calls) + if tool_calls := tool_calls_with_server_tool_calls(message): + kv_pairs = OpenTelemetry._tool_calls_kv_pair(list(tool_calls)) for key, value in kv_pairs.items(): self.safe_set_attribute( span=span, diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index b208ab76ead..b4aea1c6ba5 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -39,6 +39,7 @@ from litellm.litellm_core_utils.llm_judge import ( parse_json_verdict, ) from litellm.litellm_core_utils.redact_messages import should_redact_message_logging +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalDirection @@ -202,7 +203,7 @@ def _chat_final_text(response_obj: object) -> str: if read is None: return "" prose: Final = extract_text_from_content(read("content")) - if not (read("tool_calls") or read("function_call")): + if not _tool_calls_list(read): return prose serialized: Final = _serialize_tool_calls(read) return f"{prose} {serialized}".strip() if prose else serialized @@ -218,8 +219,9 @@ _RESPONSES_TOOL_CALL_TYPES: Final = frozenset(("function_call", "custom_tool_cal def _tool_calls_list(read: Callable[[str], object]) -> tuple[object, ...]: - calls: Final = read("tool_calls") - listed: Final = tuple(calls) if isinstance(calls, Sequence) and not isinstance(calls, str) else () + listed: Final = tool_calls_with_server_tool_calls( + {"tool_calls": read("tool_calls"), "provider_specific_fields": read("provider_specific_fields")} + ) single: Final = read("function_call") return listed if listed else ((single,) if single is not None else ()) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index b3f89b33d1d..b2e890ed557 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -606,6 +606,8 @@ def get_content_from_model_response(response: ModelResponse | dict) -> str: else: new_response = response + from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls + content = "" for choice in new_response.choices: @@ -613,9 +615,8 @@ def get_content_from_model_response(response: ModelResponse | dict) -> str: content += choice.message.content if choice.message.content else "" if choice.message.function_call: content += choice.message.function_call.model_dump_json() - if choice.message.tool_calls: - for tc in choice.message.tool_calls: - content += tc.model_dump_json() + for tc in tool_calls_with_server_tool_calls(choice.message): + content += tc.model_dump_json() elif isinstance(choice, StreamingChoices): content += getattr(choice, "delta", {}).get("content", "") or "" return content diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 77ac29fe756..514044f2886 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -63,7 +63,6 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_HOSTED_TOOLS, - ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AllAnthropicPassThroughMessageValues, AllAnthropicToolsValues, @@ -91,7 +90,6 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParam, - OpenAIChatCompletionFinishReason, OpenAIMcpServerTool, OpenAIWebSearchOptions, ) @@ -168,6 +166,7 @@ _ANTHROPIC_TOOL_NAME_MAX_LEN: Final = 128 _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool) _TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _TOOLS_ADAPTER: Final = TypeAdapter(list[object]) +_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(tuple[object, ...]) _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset( ("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid") @@ -267,6 +266,19 @@ def _hosted_web_tool_use(block: object) -> tuple[str, str, object] | None: return None +def _server_tool_use_id(block: object) -> str | None: + match block: + case {"type": "server_tool_use", "id": str() as call_id}: + return call_id + case _: + return None + + +def _server_tool_use_ids(completion_response: Mapping[str, object]) -> frozenset[str]: + blocks: Final = _CONTENT_BLOCKS_ADAPTER.validate_python(completion_response.get("content") or ()) + return frozenset(filter(None, map(_server_tool_use_id, blocks))) + + def _hosted_web_call(call_id: str, tool_name: str, tool_input: object, result: object) -> ResponseFunctionWebSearch: return build_web_search_call( tool_id=call_id, @@ -2876,11 +2888,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return usage - @staticmethod - def _is_server_tool_call(tool_call: ChatCompletionToolCallChunk) -> bool: - tool_call_id: Final = tool_call["id"] - return tool_call_id is not None and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) - def _build_code_by_id_map(self, tool_calls: list[ChatCompletionToolCallChunk]) -> dict[str, str]: code_by_id: Final[dict[str, str]] = {} for tc in tool_calls: @@ -2947,6 +2954,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): tool_results: Sequence[_AnthropicToolResultBlock] | None, compaction_blocks: Sequence[object] | None, tool_calls: list[ChatCompletionToolCallChunk], + server_tool_use_ids: frozenset[str], ) -> dict[str, object]: provider_specific_fields: Final[dict[str, object]] = { "citations": citations, @@ -2984,7 +2992,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if compaction_blocks is not None: provider_specific_fields["compaction_blocks"] = compaction_blocks - server_tool_calls: Final = [tc for tc in tool_calls if self._is_server_tool_call(tc)] + server_tool_calls: Final = [tc for tc in tool_calls if tc["id"] in server_tool_use_ids] if server_tool_calls: provider_specific_fields["server_tool_calls"] = server_tool_calls @@ -3047,7 +3055,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ): text_content = prefix_prompt + text_content - client_tool_calls: Final = [tc for tc in tool_calls if not self._is_server_tool_call(tc)] + server_tool_use_ids: Final = _server_tool_use_ids(completion_response) + client_tool_calls: Final = [tc for tc in tool_calls if tc["id"] not in server_tool_use_ids] provider_specific_fields: Final = self._build_provider_specific_fields( completion_response, citations, @@ -3056,6 +3065,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): tool_results, compaction_blocks, tool_calls, + server_tool_use_ids, ) json_mode_message, tool_calls_for_message, json_extra_content = self._resolve_json_mode_non_streaming( @@ -3090,10 +3100,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): model_response.choices[0].message = _message model_response.hidden_params["original_response"] = completion_response["content"] - model_response.choices[0].finish_reason = cast( - OpenAIChatCompletionFinishReason, - map_finish_reason(completion_response["stop_reason"]), - ) + model_response.choices[0].finish_reason = map_finish_reason(completion_response["stop_reason"]) usage: Final = self.calculate_usage( usage_object=completion_response["usage"], diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 1fccade3528..925b4009901 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -60,10 +60,21 @@ from litellm.types.llms.anthropic import ( AnthropicThinkingParam, ) from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantToolCall +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionAssistantToolCall, + ChatCompletionToolCallFunctionChunk, +) from litellm.types.proxy.auth.special_headers import SpecialHeaders from litellm.types.proxy.model_listing import ModelInfoResponse -from litellm.types.utils import ChatCompletionMessageToolCall, LlmProviders +from litellm.types.utils import ( + ChatCompletionMessageCustomToolCall, + ChatCompletionMessageToolCall, + Delta, + LlmProviders, + Message, + chat_completion_tool_call_from_dict, +) _MessageT = TypeVar("_MessageT") @@ -117,6 +128,7 @@ _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _SERVER_TOOL_CALLS_ADAPTER: Final = TypeAdapter(tuple[ChatCompletionAssistantToolCall, ...]) +_TOOL_CALL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _DECODED_JSON: Final = TypeAdapter(object) @@ -2084,10 +2096,66 @@ def anthropic_server_tool_call_params(message: object) -> tuple[ChatCompletionAs def anthropic_server_tool_calls(message: object) -> tuple[ChatCompletionMessageToolCall, ...]: - return tuple( - ChatCompletionMessageToolCall(id=call["id"], type=call["type"], function=dict(call["function"])) - for call in anthropic_server_tool_call_params(message) + return tuple(_message_tool_call_from_params(call) for call in anthropic_server_tool_call_params(message)) + + +def tool_calls_with_server_tool_calls( + message: object, +) -> tuple[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall, ...]: + client_tool_calls: Final = tuple( + call + for tool_call in parts_of(message_field(message, "tool_calls")) + if (call := _client_message_tool_call(tool_call)) is not None ) + return (*anthropic_server_tool_calls(message), *client_tool_calls) + + +def with_server_tool_call_arguments( + call: ChatCompletionAssistantToolCall, arguments: str +) -> ChatCompletionAssistantToolCall: + return ChatCompletionAssistantToolCall( + id=call["id"], + type=call["type"], + function=ChatCompletionToolCallFunctionChunk(name=call["function"].get("name"), arguments=arguments), + ) + + +def write_anthropic_server_tool_calls( + holder: Message | Delta, rewritten: tuple[ChatCompletionAssistantToolCall, ...] +) -> None: + if rewritten == anthropic_server_tool_call_params(holder): + return + holder.provider_specific_fields = { # rebind-ok: a guardrail rewrite must land on the message it scanned + **(holder.provider_specific_fields or {}), + "server_tool_calls": list(rewritten), + } + + +def write_anthropic_server_tool_call_arguments(holder: Message | Delta, index: int, arguments: str) -> None: + write_anthropic_server_tool_calls( + holder, + tuple( + with_server_tool_call_arguments(call, arguments) if call_index == index else call + for call_index, call in enumerate(anthropic_server_tool_call_params(holder)) + ), + ) + + +def _message_tool_call_from_params(call: ChatCompletionAssistantToolCall) -> ChatCompletionMessageToolCall: + return ChatCompletionMessageToolCall(id=call["id"], type=call["type"], function=dict(call["function"])) + + +def _client_message_tool_call( + tool_call: object, +) -> ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall | None: + match tool_call: + case ChatCompletionMessageToolCall() | ChatCompletionMessageCustomToolCall(): + return tool_call + case _: + try: + return chat_completion_tool_call_from_dict(_TOOL_CALL_FIELDS_ADAPTER.validate_python(tool_call)) + except (TypeError, ValidationError): + return None def tool_call_is_rebuilt_as_server_tool_use( diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index d7c3aeca354..243a09a0588 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -30,7 +30,10 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i replayed_tool_search_tool_names, ) from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder -from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + write_anthropic_server_tool_calls, +) from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, StreamingScanKey, @@ -1133,16 +1136,13 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) -> None: guardrailed_by_task: Final = MappingProxyType(dict(zip(task_mappings, guardrailed_tool_calls))) for choice_idx, choice in enumerate(response.choices): - server_tool_calls = anthropic_server_tool_call_params(choice.message) - rewritten = tuple( - _rewritten_server_tool_call(server_tool_call, guardrailed_by_task.get((choice_idx, server_idx))) - for server_idx, server_tool_call in enumerate(server_tool_calls) + write_anthropic_server_tool_calls( + choice.message, + tuple( + _rewritten_server_tool_call(server_tool_call, guardrailed_by_task.get((choice_idx, server_idx))) + for server_idx, server_tool_call in enumerate(anthropic_server_tool_call_params(choice.message)) + ), ) - if rewritten != server_tool_calls: - choice.message.provider_specific_fields = { - **(choice.message.provider_specific_fields or {}), - "server_tool_calls": list(rewritten), - } @staticmethod def _string_choice_contents(response: "ModelResponse") -> tuple[str | None, ...]: @@ -1271,16 +1271,13 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return for response in responses_so_far: for choice in response.choices: - streamed = anthropic_server_tool_call_params(choice.delta) - rewritten = tuple( - guardrailed_by_id.get((choice.index, server_tool_call["id"]), server_tool_call) - for server_tool_call in streamed + write_anthropic_server_tool_calls( + choice.delta, + tuple( + guardrailed_by_id.get((choice.index, server_tool_call["id"]), server_tool_call) + for server_tool_call in anthropic_server_tool_call_params(choice.delta) + ), ) - if rewritten != streamed: - choice.delta.provider_specific_fields = { - **(choice.delta.provider_specific_fields or {}), - "server_tool_calls": list(rewritten), - } async def _apply_guardrail_responses_to_output_streaming( self, diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 559daf767c2..9ec8923b4cc 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -38,6 +38,7 @@ from itertools import accumulate, chain, groupby, repeat from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast +from openai.types.responses import ResponseFunctionWebSearch from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -81,12 +82,14 @@ from litellm.responses.litellm_completion_transformation.tool_search import is_s from litellm.responses.litellm_completion_transformation.transformation import ( TOOL_CALL_OUTPUT_ITEM_TYPES, LiteLLMCompletionResponsesConfig, + claude_web_search_call_tool_call, tool_output_to_chat_content, ) from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( AllMessageValues, BaseLiteLLMOpenAIResponseObject, + ChatCompletionAssistantToolCall, ChatCompletionToolCallChunk, ChatCompletionToolParam, ContentPartAddedEvent, @@ -108,6 +111,7 @@ from litellm.types.llms.openai import ( from litellm.types.responses.main import ( GenericResponseOutputItem, OutputText, + with_hosted_web_tool_arguments, ) from litellm.types.utils import GenericGuardrailAPIInputs @@ -260,6 +264,8 @@ _TERMINAL_ENVELOPE_EVENT_TYPES: Final = frozenset( _TOOL_CALL_ITEM_TYPES: Final = frozenset({"function_call", "custom_tool_call"}) _TOOL_SEARCH_CALL_TYPE: Final = "tool_search_call" +_WEB_SEARCH_CALL_TYPE: Final = "web_search_call" +_ITEM_ID_KEYED_TOOL_CALL_TYPES: Final = frozenset({_TOOL_SEARCH_CALL_TYPE, _WEB_SEARCH_CALL_TYPE}) _OUTPUT_TOOL_CALL_ITEM_TYPES: Final = _TOOL_CALL_ITEM_TYPES | {_TOOL_SEARCH_CALL_TYPE} _SCANNED_TOOL_SEARCH_NAME: Final = "tool_search" _TOOL_SEARCH_ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object]) @@ -328,8 +334,38 @@ def _is_tool_call_item(item: object) -> bool: return isinstance(item, Mapping) and item.get("type") in _TOOL_CALL_ITEM_TYPES +def _web_search_call(item: object) -> ResponseFunctionWebSearch | None: + if isinstance(item, ResponseFunctionWebSearch): + return item + if stream_item_field(item, "type") != _WEB_SEARCH_CALL_TYPE: + return None + try: + return ResponseFunctionWebSearch.model_validate(item.model_dump() if isinstance(item, BaseModel) else item) + except ValidationError: + return None + + +def _claude_web_search_tool_call(item: object) -> ChatCompletionAssistantToolCall | None: + search: Final = _web_search_call(item) + return None if search is None else claude_web_search_call_tool_call(search) + + +def _claude_web_tool_call(item: object, index: int) -> ChatCompletionToolCallChunk | None: + tool_call: Final = _claude_web_search_tool_call(item) + if tool_call is None: + return None + return ChatCompletionToolCallChunk(id=tool_call["id"], type="function", function=tool_call["function"], index=index) + + +def _is_output_tool_call_item(item: object) -> bool: + return ( + stream_item_field(item, "type") in _OUTPUT_TOOL_CALL_ITEM_TYPES + or _claude_web_search_tool_call(item) is not None + ) + + def _tool_call_output_item_mapping(item: object) -> Mapping[str, object] | None: - if stream_item_field(item, "type") not in _OUTPUT_TOOL_CALL_ITEM_TYPES: + if not _is_output_tool_call_item(item): return None if isinstance(item, Mapping): return cast("Mapping[str, object]", item) # cast-ok: output items are str-keyed JSON objects @@ -341,7 +377,7 @@ def _tool_call_item_key(item: object) -> str | None: if isinstance(call_id, str) and call_id: return call_id item_id: Final = stream_item_field(item, "id") - if stream_item_field(item, "type") == _TOOL_SEARCH_CALL_TYPE and isinstance(item_id, str) and item_id: + if stream_item_field(item, "type") in _ITEM_ID_KEYED_TOOL_CALL_TYPES and isinstance(item_id, str) and item_id: return item_id return None @@ -361,6 +397,26 @@ def _scanned_tool_call_item(item: Mapping[str, object]) -> Mapping[str, object]: return {**item, "name": _SCANNED_TOOL_SEARCH_NAME, "arguments": json.dumps(item.get("arguments") or {})} +def _scanned_tool_call(item: Mapping[str, object], index: int) -> ChatCompletionToolCallChunk: + web_tool_call: Final = _claude_web_tool_call(item, index) + if web_tool_call is not None: + return web_tool_call + return tool_call_dict_from_output_item(_scanned_tool_call_item(item), index) + + +def _rewritten_web_search_call(item: object, payload: str | None) -> object: + search: Final = _web_search_call(item) + if payload is None or search is None: + return item + action: Final = with_hosted_web_tool_arguments(search.action, payload) + if isinstance(item, ResponseFunctionWebSearch): + return item.model_copy(update={"action": action}) + if isinstance(item, BaseModel): + return item.model_copy(update={"action": action.model_dump()}) + fields: Final = _tool_call_output_item_mapping(item) + return item if fields is None else {**fields, "action": action.model_dump()} + + def _tool_search_rewrite(payload: str) -> Mapping[str, object]: try: return {"arguments": _TOOL_SEARCH_ARGUMENTS_ADAPTER.validate_json(payload)} @@ -1299,17 +1355,22 @@ class OpenAIResponsesHandler(BaseTranslation): # delivered, so a truncated buffer still reports text undeliverable. # # ------------------------------------------------------------------ # if final_chunk.get("type") == "response.output_item.done": - finished_summary: Final = _output_compaction_summary(stream_item_field(final_chunk, "item")) + finished_item: Final = stream_item_field(final_chunk, "item") + finished_summary: Final = _output_compaction_summary(finished_item) model_response_stream: Final = ( OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(final_chunk) ) - tool_calls: Final = model_response_stream.choices[0].delta.tool_calls + finished_web_tool_call: Final = _claude_web_tool_call(finished_item, 0) + tool_calls: Final = [ + *cast(list[ChatCompletionToolCallChunk], model_response_stream.choices[0].delta.tool_calls or []), + *(() if finished_web_tool_call is None else (finished_web_tool_call,)), + ] if tool_calls or finished_summary is not None: inputs = GenericGuardrailAPIInputs() if finished_summary is not None: inputs["texts"] = [finished_summary] if tool_calls: - inputs["tool_calls"] = cast(list[ChatCompletionToolCallChunk], tool_calls) + inputs["tool_calls"] = tool_calls if hasattr(model_response_stream, "model") and model_response_stream.model: inputs["model"] = model_response_stream.model await guardrail_to_apply.apply_guardrail( @@ -1598,7 +1659,7 @@ class OpenAIResponsesHandler(BaseTranslation): if event_type not in _OUTPUT_ITEM_EVENT_TYPES: return None item: Final = stream_item_field(event, "item") - return _tool_call_item_key(item) if stream_item_field(item, "type") in _OUTPUT_TOOL_CALL_ITEM_TYPES else None + return _tool_call_item_key(item) if _is_output_tool_call_item(item) else None @staticmethod def _write_tool_call_item(item: object, name: str | None, payload: str | None) -> object: @@ -1612,6 +1673,8 @@ class OpenAIResponsesHandler(BaseTranslation): return item if item_type == _TOOL_SEARCH_CALL_TYPE: return _rewritten_tool_search_call(item, payload) + if item_type == _WEB_SEARCH_CALL_TYPE: + return _rewritten_web_search_call(item, payload) if name is not None: OpenAIResponsesHandler._write_event_field(item, "name", name) if payload is not None and isinstance(item_type, str) and item_type in _TOOL_CALL_PAYLOAD_FIELDS: @@ -1667,7 +1730,7 @@ class OpenAIResponsesHandler(BaseTranslation): stream_item_field(event, "type") in _TOOL_CALL_PAYLOAD_EVENT_TYPES or ( stream_item_field(event, "type") in _OUTPUT_ITEM_EVENT_TYPES - and stream_item_field(stream_item_field(event, "item"), "type") in _OUTPUT_TOOL_CALL_ITEM_TYPES + and _is_output_tool_call_item(stream_item_field(event, "item")) ) for event in responses_so_far ) @@ -1675,9 +1738,7 @@ class OpenAIResponsesHandler(BaseTranslation): @staticmethod def _completed_response_scan_key(response: object) -> StreamingScanKey: output_items: Final = stream_item_items(response, "output") - message_items: Final = tuple( - item for item in output_items if stream_item_field(item, "type") not in _OUTPUT_TOOL_CALL_ITEM_TYPES - ) + message_items: Final = tuple(item for item in output_items if not _is_output_tool_call_item(item)) message_parts: Final = chain.from_iterable(stream_item_items(item, "content") for item in message_items) part_texts: Final = tuple( text for part in message_parts if isinstance(text := stream_item_field(part, "text"), str) and text @@ -1685,11 +1746,7 @@ class OpenAIResponsesHandler(BaseTranslation): summaries: Final = tuple(summary for item in output_items if (summary := _output_compaction_summary(item))) return StreamingScanKey( texts=(*part_texts, *summaries), - tool_calls=tuple( - stream_item_fingerprint(item) - for item in output_items - if stream_item_field(item, "type") in _OUTPUT_TOOL_CALL_ITEM_TYPES - ), + tool_calls=tuple(stream_item_fingerprint(item) for item in output_items if _is_output_tool_call_item(item)), stream_ended=True, ) @@ -1802,9 +1859,7 @@ class OpenAIResponsesHandler(BaseTranslation): tool_call_item: Final = _tool_call_output_item_mapping(output_item) if tool_call_item is not None: if tool_calls_to_check is not None: - tool_calls_to_check.append( - tool_call_dict_from_output_item(_scanned_tool_call_item(tool_call_item), output_idx) - ) + tool_calls_to_check.append(_scanned_tool_call(tool_call_item, output_idx)) return summary: Final = _output_compaction_summary(output_item) diff --git a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py index 19b9bccd8aa..3526635a9de 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py @@ -24,6 +24,10 @@ from litellm._logging import verbose_proxy_logger from litellm._version import version as litellm_version from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.core_helpers import iter_items +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_calls, + write_anthropic_server_tool_call_arguments, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, get_ssl_configuration, @@ -515,27 +519,37 @@ class CatoNetworksGuardrail(CustomGuardrail): @staticmethod def _output_fragments(message: Message) -> Sequence[tuple[tuple[str, int | None], str]]: """Assistant text the proxy returns to the caller: ``content`` plus every - ``tool_calls[].function.arguments`` string, each tagged with where a - redaction must be written back. ``content`` is only included when present - so a tool-call-only choice keeps its ``None`` content (the text-vs-tool-call - signal downstream consumers rely on) while its arguments are still inspected.""" - fragments: Final[list] = [] - if message.content is not None: - fragments.append((("content", None), message.content)) - for idx, tool_call in enumerate(message.tool_calls or []): - function = getattr(tool_call, "function", None) - arguments = getattr(function, "arguments", None) - if isinstance(arguments, str) and arguments: - fragments.append((("tool_call", idx), arguments)) - return fragments + tool call's ``function.arguments`` string, Claude's server tool calls first, + each tagged with where a redaction must be written back. ``content`` is only + included when present so a tool-call-only choice keeps its ``None`` content + (the text-vs-tool-call signal downstream consumers rely on) while its + arguments are still inspected.""" + content: Final = () if message.content is None else ((("content", None), message.content),) + return ( + *content, + *CatoNetworksGuardrail._tool_call_fragments("server_tool_call", anthropic_server_tool_calls(message)), + *CatoNetworksGuardrail._tool_call_fragments("tool_call", message.tool_calls or ()), + ) + + @staticmethod + def _tool_call_fragments(kind: str, tool_calls: Sequence[object]) -> tuple[tuple[tuple[str, int | None], str], ...]: + return tuple( + ((kind, idx), arguments) + for idx, arguments in enumerate( + getattr(getattr(tool_call, "function", None), "arguments", None) for tool_call in tool_calls + ) + if isinstance(arguments, str) and arguments + ) @staticmethod def _apply_output_fragment(message: Any, target: tuple[str, int | None], redacted: str) -> None: - kind, idx = target - if kind == "content": - message.content = redacted - else: - message.tool_calls[idx].function.arguments = redacted + match target: + case ("content", _): + message.content = redacted + case ("server_tool_call", int() as idx): + write_anthropic_server_tool_call_arguments(message, idx, redacted) + case (_, idx): + message.tool_calls[idx].function.arguments = redacted @staticmethod def _responses_output_field(item: object, key: str) -> str | Sequence[object] | None: diff --git a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py index 45e785fd53c..fb5304d6ecd 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py @@ -36,6 +36,12 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.litellm_core_utils.core_helpers import iter_items +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + tool_calls_with_server_tool_calls, + with_server_tool_call_arguments, + write_anthropic_server_tool_calls, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -47,7 +53,9 @@ from litellm.proxy.common_utils.callback_utils import ( from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import ( Choices, + Delta, LLMResponseTypes, + Message, ModelResponse, ModelResponseStream, TextCompletionResponse, @@ -1633,6 +1641,7 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): @classmethod def _clear_tool_call_arguments(cls, message: object) -> None: """Clear tool-call / function-call arguments after Cisco redaction.""" + cls._clear_server_tool_call_arguments(message) tool_calls = message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None) for tc in tool_calls or []: fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None) @@ -1642,6 +1651,23 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): ) cls._clear_arguments_field(function_call) + @staticmethod + def _clear_server_tool_call_arguments(message: object) -> None: + cleared: Final = tuple( + with_server_tool_call_arguments(server_tool_call, "{}") + for server_tool_call in anthropic_server_tool_call_params(message) + ) + match message: + case Message() | Delta(): + write_anthropic_server_tool_calls(message, cleared) + case dict() if cleared: + message["provider_specific_fields"] = { # rebind-ok: a redaction must land on the message it scanned + **(message.get("provider_specific_fields") or {}), + "server_tool_calls": list(cleared), + } + case _: + pass + def _redact_responses_api_output( self, output_items: list, @@ -2074,8 +2100,7 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): @classmethod def _extract_message_tool_argument_parts(cls, message: object) -> list[str]: parts: Final[list[str]] = [] - tool_calls = message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None) - for tool_call in tool_calls or []: + for tool_call in tool_calls_with_server_tool_calls(message): args = cls._extract_tool_call_arguments(tool_call) if args: parts.append(args) diff --git a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py index 97b35769f63..5902a07dea2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py @@ -40,6 +40,12 @@ from litellm.integrations.custom_guardrail import ( ) from litellm.integrations.custom_guardrail import dc as global_cache from litellm.litellm_core_utils.core_helpers import iter_items +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + anthropic_server_tool_calls, + with_server_tool_call_arguments, + write_anthropic_server_tool_calls, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -50,6 +56,7 @@ from litellm.proxy.guardrails._content_utils import ( has_non_string_content, ) from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.llms.openai import ChatCompletionAssistantToolCall class LassoResponse(TypedDict): @@ -259,7 +266,7 @@ class LassoGuardrail(CustomGuardrail): msg = choice.message if msg.content: response_messages.append({"role": "assistant", "content": msg.content}) - for call in getattr(msg, "tool_calls", None) or []: + for call in (*anthropic_server_tool_calls(msg), *(getattr(msg, "tool_calls", None) or ())): call_id, name, input_data = self._extract_tool_call_fields(call) if not call_id or not name: continue @@ -962,6 +969,14 @@ class LassoGuardrail(CustomGuardrail): text_cursor += 1 verbose_proxy_logger.debug("Applied masked text content to choice %s", text_cursor) + write_anthropic_server_tool_calls( + msg, + tuple( + self._masked_server_tool_call(server_tool_call, masked_tool_use) + for server_tool_call in anthropic_server_tool_call_params(msg) + ), + ) + for call in getattr(msg, "tool_calls", None) or []: call_id = self._get_field(call, "id") if call_id and call_id in masked_tool_use: @@ -977,6 +992,16 @@ class LassoGuardrail(CustomGuardrail): func.arguments = json.dumps(masked_input) verbose_proxy_logger.debug("Applied masked tool_call arguments for call_id=%s", call_id) + @staticmethod + def _masked_server_tool_call( + server_tool_call: ChatCompletionAssistantToolCall, + masked_tool_use: Mapping[object, Mapping[str, object]], + ) -> ChatCompletionAssistantToolCall: + masked_input: Final = masked_tool_use.get(server_tool_call["id"], {}).get("input") + if masked_input is None: + return server_tool_call + return with_server_tool_call_arguments(server_tool_call, json.dumps(masked_input)) + @staticmethod def get_config_model() -> type["GuardrailConfigModel"] | None: from litellm.types.proxy.guardrails.guardrail_hooks.lasso import ( diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py index d2dafb04fe0..5e7a0240545 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py @@ -24,13 +24,17 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.litellm_core_utils.core_helpers import attribute_items, attribute_or_none, iter_items, require_attributes +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + write_anthropic_server_tool_call_arguments, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks -from litellm.types.utils import GenericGuardrailAPIInputs, TextChoices +from litellm.types.utils import GenericGuardrailAPIInputs, Message, TextChoices if TYPE_CHECKING: from litellm.caching.caching import DualCache @@ -87,6 +91,19 @@ _DEFAULT_TIMEOUT_SECONDS: Final = 10.0 _SHIELD_BODY: Final = TypeAdapter(dict[str, object]) +def _server_tool_call_slots(message: object) -> tuple[Slot, ...]: + if not isinstance(message, Message): + return () + return tuple( + (arguments, functools.partial(write_anthropic_server_tool_call_arguments, message, index)) + for index, arguments in enumerate( + server_tool_call["function"].get("arguments") + for server_tool_call in anthropic_server_tool_call_params(message) + ) + if isinstance(arguments, str) and arguments + ) + + def _string_tool_arguments(calls: Sequence[object]) -> Iterator[tuple[object, str]]: for call in calls: function = read_field(call, "function") @@ -382,6 +399,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): content = getattr(message, "content", None) if isinstance(content, str) and content: pending.append((content, functools.partial(setattr, message, "content"))) + pending.extend(_server_tool_call_slots(message)) for tool_call in attribute_items(message, "tool_calls"): function: object = getattr(tool_call, "function", None) arguments = getattr(function, "arguments", None) if function is not None else None diff --git a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py index f83314af548..1956170ba67 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py +++ b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py @@ -13,6 +13,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, ) from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -434,9 +435,7 @@ class PurviewGuardrailBase: args: Final[list[str]] = [] # tool_calls: [{"function": {"arguments": "..."}}] - tool_calls: Final[Sequence[object] | None] = ( - message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None) - ) + tool_calls: Final = tool_calls_with_server_tool_calls(message) if tool_calls: for tc in tool_calls: fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None) diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index 3cd9dc6a7b4..379c2c4b075 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -27,6 +27,12 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + tool_calls_with_server_tool_calls, + with_server_tool_call_arguments, + write_anthropic_server_tool_calls, +) from litellm.llms.base_llm.guardrail_translation.utils import ( effective_scan_only_tool_results_for_guardrail, effective_skip_system_message_for_guardrail, @@ -364,10 +370,9 @@ class PanwPrismaAirsHandler(CustomGuardrail): text_parts.append(str(choice.message.content)) # Extract tool call arguments - if hasattr(choice.message, "tool_calls") and choice.message.tool_calls: - for tool_call in choice.message.tool_calls: - if hasattr(tool_call, "function") and hasattr(tool_call.function, "arguments"): - text_parts.append(str(tool_call.function.arguments)) + for tool_call in tool_calls_with_server_tool_calls(choice.message): + if hasattr(tool_call, "function") and hasattr(tool_call.function, "arguments"): + text_parts.append(str(tool_call.function.arguments)) # Extract function call arguments (legacy) if hasattr(choice.message, "function_call") and choice.message.function_call: @@ -756,6 +761,14 @@ class PanwPrismaAirsHandler(CustomGuardrail): elif isinstance(content, list): choice.message.content = self._mask_content_list(content, masked_text) + write_anthropic_server_tool_calls( + choice.message, + tuple( + with_server_tool_call_arguments(server_tool_call, masked_text) + for server_tool_call in anthropic_server_tool_call_params(choice.message) + ), + ) + # Mask tool call arguments if hasattr(choice.message, "tool_calls") and choice.message.tool_calls: for tool_call in choice.message.tool_calls: diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 7e2f6bfec33..8d0f71b554c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -12,7 +12,7 @@ import asyncio import json import re import threading -from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Iterator, Sequence +from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Callable, Iterator, Sequence from contextlib import asynccontextmanager from dataclasses import dataclass from datetime import datetime @@ -43,6 +43,11 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + with_server_tool_call_arguments, + write_anthropic_server_tool_calls, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames from litellm.proxy.guardrails.anthropic_sse import ( @@ -1392,6 +1397,18 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): verbose_proxy_logger.debug("No pii_tokens found in request_data['metadata'] — nothing to unmask") presidio_config: Final = self.get_presidio_settings_from_request_data(request_data or {}) + async def pii_processed_text(text: str) -> str: + match mode: + case "unmask": + return self._unmask_pii_text(text, pii_tokens) + case "mask": + return await self.check_pii( + text=text, + output_parse_pii=False, + presidio_config=presidio_config, + request_data=request_data, + ) + for choice in response.choices: message: Message | None = getattr(choice, "message", None) if message is None: @@ -1400,15 +1417,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): # 1. Process content content = getattr(message, "content", None) if isinstance(content, str): - if mode == "unmask": - message.content = self._unmask_pii_text(content, pii_tokens) - elif mode == "mask": - message.content = await self.check_pii( - text=content, - output_parse_pii=False, - presidio_config=presidio_config, - request_data=request_data, - ) + message.content = await pii_processed_text(content) elif isinstance(content, list): for item in content: if not isinstance(item, dict): @@ -1416,54 +1425,43 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): text_value = item.get("text") if text_value is None: continue - if mode == "unmask": - item["text"] = self._unmask_pii_text(text_value, pii_tokens) - elif mode == "mask": - item["text"] = await self.check_pii( - text=text_value, - output_parse_pii=False, - presidio_config=presidio_config, - request_data=request_data, - ) + item["text"] = await pii_processed_text(text_value) + + await self._process_server_tool_calls_for_pii(message, pii_processed_text) # 2. Process tool calls for tool_call in attribute_items(message, "tool_calls"): function = attribute_or_none(tool_call, "function") args = attribute_or_none(function, "arguments") if function else None if isinstance(args, str): - if mode == "unmask": - _write_attribute(function, "arguments", self._unmask_pii_text(args, pii_tokens)) - elif mode == "mask": - _write_attribute( - function, - "arguments", - await self.check_pii( - text=args, - output_parse_pii=False, - presidio_config=presidio_config, - request_data=request_data, - ), - ) + _write_attribute(function, "arguments", await pii_processed_text(args)) # 3. Process legacy function calls function_call = attribute_or_none(message, "function_call") legacy_args = attribute_or_none(function_call, "arguments") if function_call else None if isinstance(legacy_args, str): - if mode == "unmask": - _write_attribute(function_call, "arguments", self._unmask_pii_text(legacy_args, pii_tokens)) - elif mode == "mask": - _write_attribute( - function_call, - "arguments", - await self.check_pii( - text=legacy_args, - output_parse_pii=False, - presidio_config=presidio_config, - request_data=request_data, - ), - ) + _write_attribute(function_call, "arguments", await pii_processed_text(legacy_args)) return response + @staticmethod + async def _process_server_tool_calls_for_pii( + message: Message, + pii_processed_text: Callable[[str], Awaitable[str]], + ) -> None: + server_tool_calls: Final = anthropic_server_tool_call_params(message) + if not server_tool_calls: + return + processed_arguments: Final = tuple( + [ + await pii_processed_text(server_tool_call["function"].get("arguments", "")) + for server_tool_call in server_tool_calls + ] + ) + write_anthropic_server_tool_calls( + message, + tuple(map(with_server_tool_call_arguments, server_tool_calls, processed_arguments)), + ) + async def _mask_output_response( self, response: ModelResponse | EmbeddingResponse | ImageResponse, diff --git a/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py b/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py index b1f0f588ade..2e1ac62d850 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py +++ b/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py @@ -12,6 +12,7 @@ from pydantic import BaseModel, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] httpxSpecialProvider, @@ -80,13 +81,11 @@ class RepelloAIGuardrail(CustomGuardrail): def _extract_tool_call_args_from_message(cls, message: object) -> list[str]: args: Final[list[str]] = [] - tool_calls: Final = cls._get_field(message, "tool_calls") - if _is_object_list(tool_calls): - for tool_call in tool_calls: - function = cls._get_field(tool_call, "function") - arguments = cls._get_field(function, "arguments") - if isinstance(arguments, str) and arguments.strip(): - args.append(arguments) + for tool_call in tool_calls_with_server_tool_calls(message): + function = cls._get_field(tool_call, "function") + arguments = cls._get_field(function, "arguments") + if isinstance(arguments, str) and arguments.strip(): + args.append(arguments) function_call: Final = cls._get_field(message, "function_call") arguments = cls._get_field(function_call, "arguments") diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index ad03186c7a5..683e51be6e8 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -31,7 +31,6 @@ from openai.types.chat.chat_completion_named_tool_choice_param import ( from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch from openai.types.responses.response_compaction_item import ResponseCompactionItem from openai.types.responses.response_create_params import ResponseInputParam -from openai.types.responses.response_function_web_search import ActionOpenPage, ActionSearch from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam from openai.types.responses.tool_param import FunctionToolParam @@ -121,6 +120,7 @@ from litellm.types.responses.main import ( OutputImageGenerationCall, OutputText, OutputToolSearchCall, + hosted_web_tool_call, web_search_call_tool_id, ) from litellm.types.utils import ( @@ -881,22 +881,14 @@ def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, obj return {key: {**_STR_KEY_DICT_ADAPTER.validate_python(metadata), CALLER_SESSION_ID_METADATA_KEY: chain_session}} -def _pending_claude_server_tool_call(search: ResponseFunctionWebSearch) -> ChatCompletionAssistantToolCall | None: +def claude_web_search_call_tool_call(search: ResponseFunctionWebSearch) -> ChatCompletionAssistantToolCall | None: tool_id: Final = web_search_call_tool_id(search.id) - if search.status != "in_progress" or tool_id is None: + if tool_id is None or not tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX): return None - if not tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX): + hosted_call: Final = hosted_web_tool_call(search.action) + if hosted_call is None: return None - match search.action: - case ActionSearch(query=str() as query): - return _server_tool_call(tool_id, "web_search", json.dumps({"query": query})) - case ActionOpenPage(url=str() as url): - return _server_tool_call(tool_id, "web_fetch", json.dumps({"url": url})) - case _: - return None - - -def _server_tool_call(tool_id: str, name: str, arguments: str) -> ChatCompletionAssistantToolCall: + name, arguments = hosted_call return ChatCompletionAssistantToolCall( id=tool_id, type="function", @@ -904,6 +896,10 @@ def _server_tool_call(tool_id: str, name: str, arguments: str) -> ChatCompletion ) +def _pending_claude_server_tool_call(search: ResponseFunctionWebSearch) -> ChatCompletionAssistantToolCall | None: + return claude_web_search_call_tool_call(search) if search.status == "in_progress" else None + + class LiteLLMCompletionResponsesConfig: @staticmethod def get_supported_openai_params(model: str) -> list: diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index e9111eb8c99..44f50a9df9c 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -1,15 +1,17 @@ import builtins +import json from collections.abc import Mapping, Sequence from typing import Final, Literal, Optional, Union from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall from openai.types.responses.response_function_web_search import ( + Action, ActionOpenPage, ActionSearch, ActionSearchSource, ResponseFunctionWebSearch, ) -from pydantic import ConfigDict, PrivateAttr +from pydantic import ConfigDict, PrivateAttr, TypeAdapter, ValidationError from typing_extensions import TypedDict from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject @@ -116,6 +118,40 @@ def web_search_call_tool_id(call_id: str) -> str | None: return None if tool_id == call_id else tool_id +_HOSTED_WEB_TOOL_ARGUMENTS: Final = TypeAdapter(dict[str, object]) + + +def hosted_web_tool_call(action: object) -> tuple[str, str] | None: + match action: + case ActionSearch(query=str() as query): + return "web_search", json.dumps({"query": query}) + case ActionOpenPage(url=str() as url): + return "web_fetch", json.dumps({"url": url}) + case _: + return None + + +def _decoded_hosted_web_tool_arguments(arguments: str) -> Mapping[str, object]: + try: + return _HOSTED_WEB_TOOL_ARGUMENTS.validate_json(arguments) + except ValidationError: + return {} + + +def with_hosted_web_tool_arguments(action: Action, arguments: str) -> Action: + match action, _decoded_hosted_web_tool_arguments(arguments): + case ActionSearch(), {"query": str() as query}: + return action.model_copy(update={"query": query, "queries": [query] if query else []}) + case ActionSearch(), _: + return action.model_copy(update={"query": "", "queries": []}) + case ActionOpenPage(), {"url": str() as url}: + return action.model_copy(update={"url": url}) + case ActionOpenPage(), _: + return action.model_copy(update={"url": None}) + case _: + return action + + class OutputImageGenerationCall(BaseLiteLLMOpenAIResponseObject): """An image generation call output""" diff --git a/tests/unit/integrations/arize/test_arize_utils.py b/tests/unit/integrations/arize/test_arize_utils.py index 31699359bbe..db7144b125a 100644 --- a/tests/unit/integrations/arize/test_arize_utils.py +++ b/tests/unit/integrations/arize/test_arize_utils.py @@ -725,6 +725,73 @@ def test_arize_output_value_falls_back_to_tool_calls_summary(): assert "SF" in out +def test_arize_server_tool_calls_lead_the_output_summary_and_tool_call_attributes() -> None: + span: Final = TracerProvider().get_tracer(__name__).start_span("litellm_request") + kwargs: Final = { + "model": "claude-opus-5-5", + "messages": [{"role": "user", "content": "persimmon facts?"}], + "standard_logging_object": {"model_parameters": {}, "metadata": {}, "call_type": "completion"}, + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "anthropic"}, + } + response_obj: Final = ModelResponse( + usage={"total_tokens": 10, "completion_tokens": 4, "prompt_tokens": 6}, + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=litellm.Message( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}, + } + ], + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + ), + ) + ], + model="claude-opus-5-5", + id="r-server-tools", + ) + + ArizeLogger.set_arize_attributes(span, kwargs, response_obj) + + tool_calls_prefix: Final = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_TOOL_CALLS}." + tool_call_attributes: Final = { + key: value for key, value in span.attributes.items() if key.startswith(tool_calls_prefix) + } + assert json.loads(span.attributes[SpanAttributes.OUTPUT_VALUE]) == { + "tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + }, + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}}, + ] + } + assert tool_call_attributes == { + f"{tool_calls_prefix}0.{ToolCallAttributes.TOOL_CALL_ID}": "srvtoolu_01", + f"{tool_calls_prefix}0.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}": "web_search", + f"{tool_calls_prefix}0.{ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON}": '{"query": "persimmon"}', + f"{tool_calls_prefix}1.{ToolCallAttributes.TOOL_CALL_ID}": "call_1", + f"{tool_calls_prefix}1.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}": "get_weather", + f"{tool_calls_prefix}1.{ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON}": '{"city": "SF"}', + } + + def test_arize_output_value_unchanged_when_content_present(): """Regression guard: when content is non-empty, OUTPUT_VALUE must be exactly that content (no summary written).""" diff --git a/tests/unit/integrations/newrelic/test_newrelic.py b/tests/unit/integrations/newrelic/test_newrelic.py index d09732c6eb2..79dde4f5959 100644 --- a/tests/unit/integrations/newrelic/test_newrelic.py +++ b/tests/unit/integrations/newrelic/test_newrelic.py @@ -1,3 +1,4 @@ +import json import os import sys from datetime import datetime, timezone @@ -45,6 +46,12 @@ NR_ENV = { "NEW_RELIC_APP_NAME": "test-app", } +_SERVER_CALL: Final = { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, +} + def make_logger(**kwargs) -> NewRelicLogger: """Instantiate NewRelicLogger with NR agent calls mocked out.""" @@ -361,6 +368,19 @@ class TestExtractMessageContent: assert "get_weather" in result assert "call_1" in result + @pytest.mark.parametrize( + "message", + [ + {"role": "assistant", "content": None, "provider_specific_fields": {"server_tool_calls": [_SERVER_CALL]}}, + litellm.Message( + role="assistant", content=None, provider_specific_fields={"server_tool_calls": [_SERVER_CALL]} + ), + ], + ids=["dict", "message"], + ) + def test_server_tool_call_only_message_serializes_the_server_call(self, message): + assert json.loads(self.logger._extract_message_content(message)) == [_SERVER_CALL] + def test_multimodal_list_serialized_as_json(self): msg = { "content": [ diff --git a/tests/unit/integrations/test_helicone.py b/tests/unit/integrations/test_helicone.py index ce3f61a61eb..3f49d000747 100644 --- a/tests/unit/integrations/test_helicone.py +++ b/tests/unit/integrations/test_helicone.py @@ -17,7 +17,8 @@ def _claude_mapping(messages, response_obj): return logger.claude_mapping(model="gpt-5.6", messages=messages, response_obj=response_obj) -def test_claude_mapping_serializes_custom_tool_calls(monkeypatch): +@pytest.fixture() +def anthropic_prompts(monkeypatch): """ Stub the anthropic module unconditionally: the SDK may be absent (it lives in the proxy-runtime extra), and the tests/unit/llms/anthropic test package can @@ -27,6 +28,10 @@ def test_claude_mapping_serializes_custom_tool_calls(monkeypatch): stub.HUMAN_PROMPT = "\n\nHuman:" stub.AI_PROMPT = "\n\nAssistant:" monkeypatch.setitem(sys.modules, "anthropic", stub) + + +@pytest.mark.usefixtures("anthropic_prompts") +def test_claude_mapping_serializes_custom_tool_calls(): response_obj = { "id": "chatcmpl-1", "choices": [ @@ -58,6 +63,44 @@ def test_claude_mapping_serializes_custom_tool_calls(monkeypatch): assert {"type": "tool_use", "id": "call_f", "name": "read_file", "input": '{"path": "a.py"}'} in tool_use_blocks +@pytest.mark.usefixtures("anthropic_prompts") +def test_claude_mapping_puts_server_tool_calls_first_as_tool_use_blocks(): + response_obj = { + "id": "chatcmpl-1", + "choices": [ + { + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_f", + "type": "function", + "function": {"name": "read_file", "arguments": '{"path": "a.py"}'}, + }, + ], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + }, + "finish_reason": "tool_calls", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 2}, + } + mapped = _claude_mapping([{"role": "user", "content": "hi"}], response_obj) + assert mapped["content"] == [ + {"type": "tool_use", "id": "srvtoolu_01", "name": "web_search", "input": '{"query": "persimmon"}'}, + {"type": "tool_use", "id": "call_f", "name": "read_file", "input": '{"path": "a.py"}'}, + ] + + @pytest.fixture() def _vcr_outcome_gate(request, vcr): install_live_call_probe(request, vcr) diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index a2803719c75..71087b4de51 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -1309,6 +1309,55 @@ class TestDerivedTraceFields: assert json.loads(str(attributes["langfuse.observation.output"])) == {"content": "hi"} + def test_server_tool_calls_lead_the_observation_output(self) -> None: + response_obj: Final = ModelResponse( + id="chatcmpl-server", + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=litellm.Message( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}, + } + ], + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + ), + ) + ], + ) + attributes: Final = _emitted({"call_type": "acompletion", "litellm_params": {"metadata": {}}}, response_obj) + + assert json.loads(str(attributes["langfuse.observation.output"])) == [ + { + "id": "chatcmpl-server", + "name": "web_search", + "call_id": "srvtoolu_01", + "type": "function_call", + "arguments": {"query": "persimmon"}, + }, + { + "id": "chatcmpl-server", + "name": "get_weather", + "call_id": "call_1", + "type": "function_call", + "arguments": {"city": "SF"}, + }, + ] + def test_no_messages_leaves_trace_input_unset_but_emits_output(self) -> None: attributes: Final = _emitted( {"call_type": "acompletion", "litellm_params": {"metadata": {}}}, _chat_response("hi") diff --git a/tests/unit/integrations/test_lunary.py b/tests/unit/integrations/test_lunary.py index 6491f5c8b82..302f3d7e22e 100644 --- a/tests/unit/integrations/test_lunary.py +++ b/tests/unit/integrations/test_lunary.py @@ -1,6 +1,7 @@ -from litellm.integrations.lunary import parse_tool_calls +import litellm +from litellm.integrations.lunary import parse_messages, parse_tool_calls from litellm.types.utils import ( ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall, @@ -35,3 +36,46 @@ def test_parse_tool_calls_serializes_custom_tool_calls(): def test_parse_tool_calls_none_passthrough(): assert parse_tool_calls(None) is None + + +def test_parse_messages_lists_server_tool_calls_before_client_calls(): + response = litellm.ModelResponse( + choices=[ + litellm.Choices( + index=0, + finish_reason="tool_calls", + message=litellm.Message( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}, + } + ], + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + ), + ) + ], + ) + assert parse_messages(response.choices) == { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "type": "function", + "id": "srvtoolu_01", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + }, + {"type": "function", "id": "call_1", "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}}, + ], + } diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index f0fca94c370..0e5f5a0ccdd 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -367,6 +367,60 @@ def test_v1_routing_diagnostics_add_scalar_attributes_and_preserve_legacy_metada assert finished.attributes["metadata.routing_decision"] == str(decision) +def test_v1_server_tool_calls_lead_the_completion_function_call_attributes() -> None: + exporter: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + otel: Final = OpenTelemetry(tracer_provider=provider) + response_obj: Final = litellm.ModelResponse( + choices=[ + litellm.Choices( + index=0, + finish_reason="tool_calls", + message=litellm.Message( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}, + } + ], + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + ), + ) + ], + ) + with provider.get_tracer(__name__).start_as_current_span("inference") as span: + otel.set_attributes( + span, + { + "model": "claude-opus-5-5", + "standard_logging_object": {"id": "server-tools-call", "call_type": "acompletion", "metadata": {}}, + }, + response_obj, + ) + (finished,) = exporter.get_finished_spans() + function_call_attributes: Final = { + key: value for key, value in finished.attributes.items() if ".function_call." in key + } + assert function_call_attributes == { + f"{SpanAttributes.LLM_COMPLETIONS.value}.0.function_call.name": "web_search", + f"{SpanAttributes.LLM_COMPLETIONS.value}.0.function_call.arguments": '{"query": "persimmon"}', + f"{SpanAttributes.LLM_COMPLETIONS.value}.1.function_call.name": "get_weather", + f"{SpanAttributes.LLM_COMPLETIONS.value}.1.function_call.arguments": '{"city": "SF"}', + } + + def test_v1_pod_routing_none_matches_other_none_metadata_attributes() -> None: exporter: Final = InMemorySpanExporter() provider: Final = TracerProvider() diff --git a/tests/unit/integrations/test_shadow_eval_logger.py b/tests/unit/integrations/test_shadow_eval_logger.py index f802dbfdba9..f048821f8b7 100644 --- a/tests/unit/integrations/test_shadow_eval_logger.py +++ b/tests/unit/integrations/test_shadow_eval_logger.py @@ -1905,6 +1905,40 @@ class TestShadowPipeline: assert "Let me look that up. [tool call] Read({})" in judge_prompt + async def test_a_server_tool_call_reaches_the_judge_ahead_of_client_tool_calls(self): + router = _shadow_reply_router( + { + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}, + } + ], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + }, + finish_reason="tool_calls", + ) + await self._judged_shadow_row(router) + + judge_prompt = next( + call.kwargs["messages"][-1]["content"] + for call in router.acompletion.call_args_list + if call.kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != SHADOW_EVAL_ROUTER_CALL_ORIGIN + ) + + shadow_text: Final = '[tool call] web_search({"query": "persimmon"}), [tool call] get_weather({"city": "SF"})' + assert f":\n{shadow_text}\n\n" in judge_prompt, judge_prompt + async def test_an_empty_shadow_reply_names_the_finish_reason_and_the_routed_model(self): """A reply that really carried no text is diagnosable only if the row says what the arm was doing when it produced none: a truncated turn and a model that answers diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 72e72714cd4..e3d58fffdda 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( converse_redacted_content, encrypted_content_from_signature, encrypted_reasoning_signature, + get_content_from_model_response, get_file_ids_from_messages, get_format_from_file_id, get_semantic_cache_prompt_from_messages, @@ -46,6 +47,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( with_prompt_cache_breakpoint, ) from litellm.types.llms.openai import ChatCompletionAudioObject, ChatCompletionFileObject, ChatCompletionImageObject +from litellm.types.utils import ChatCompletionMessageToolCall, Choices, Function, Message, ModelResponse _ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$' _ARTIFACT_DATA_ID_PATTERN: Final = r"^(?!\.\.?(?:\/|$))[A-Za-z0-9_\-.~:@+]{1,200}$" @@ -2675,3 +2677,30 @@ def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None projected, media = split_media_for_tpm(request) assert projected is request assert media == () + + +def test_get_content_from_model_response_includes_server_tool_calls_before_client_calls() -> None: + server_call: Final = ChatCompletionMessageToolCall( + id="srvtoolu_01", type="function", function=Function(name="web_search", arguments='{"query": "persimmon"}') + ) + client_call: Final = ChatCompletionMessageToolCall( + id="call_1", type="function", function=Function(name="get_weather", arguments='{"city": "SF"}') + ) + response: Final = ModelResponse( + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content="Looking it up.", + tool_calls=[client_call.model_dump()], + provider_specific_fields={"server_tool_calls": [server_call.model_dump()]}, + ), + ) + ], + ) + + assert get_content_from_model_response(response) == ( + "Looking it up." + server_call.model_dump_json() + client_call.model_dump_json() + ) diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 255162b3cd9..055b40fbdd6 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -3548,6 +3548,45 @@ def test_code_execution_tool_results_extraction(): assert "Done!" in transformed_response.choices[0].message.content +def test_server_and_client_tool_calls_split_by_block_type_not_id_prefix(): + completion_response: Final = { + "id": "msg_split", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20250929", + "content": [ + {"type": "server_tool_use", "id": "toolu_server", "name": "web_search", "input": {"query": "persimmon"}}, + {"type": "tool_use", "id": "srvtoolu_client", "name": "get_weather", "input": {"city": "Paris"}}, + ], + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + raw_response: Final = MagicMock(spec=httpx.Response) + raw_response.headers = {} + + message: Final = ( + AnthropicConfig() + .transform_parsed_response( + completion_response=completion_response, + raw_response=raw_response, + model_response=ModelResponse(), + json_mode=False, + prefix_prompt=None, + ) + .choices[0] + .message + ) + + assert [ + (call["id"], call["function"].get("name"), call["function"].get("arguments")) + for call in anthropic_server_tool_call_params(message) + ] == [("toolu_server", "web_search", '{"query": "persimmon"}')] + assert [ + (tool_call.id, tool_call.function.name, tool_call.function.arguments) for tool_call in message.tool_calls or () + ] == [("srvtoolu_client", "get_weather", '{"city": "Paris"}')] + + def test_code_execution_tool_results_in_hidden_params(): """ Test that tool_results reaches _hidden_params so the Responses API adapter diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 673e880a1c0..96b19fc36a3 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -20,7 +20,9 @@ import pytest from fastapi import HTTPException from pydantic import BaseModel +from openai.types.responses.response_function_web_search import ActionSearch from openai.types.responses import ( + ResponseFunctionWebSearch, ResponseCustomToolCall, ResponseCustomToolCallInputDeltaEvent, ResponseCustomToolCallInputDoneEvent, @@ -48,6 +50,7 @@ from litellm.types.llms.openai import ResponsesAPIResponse from litellm.responses.litellm_completion_transformation.tool_search import build_tool_search_call from litellm.types.responses.main import ( CustomToolCallOutputItem, + build_web_search_call, GenericResponseOutputItem, OutputText, OutputToolSearchCall, @@ -237,6 +240,30 @@ CUSTOM_TOOL_CALL_ITEM = { } +def _claude_web_calls_and_a_native_search() -> list[ResponseFunctionWebSearch]: + return [ + build_web_search_call("srvtoolu_1", {"query": "persimmon"}, None, "completed"), + build_web_search_call("srvtoolu_2", {"url": "https://persimmon.example"}, None, "in_progress", "web_fetch"), + ResponseFunctionWebSearch( + id="ws_native", + type="web_search_call", + status="completed", + action=ActionSearch(type="search", query="persimmon"), + ), + ] + + +def _web_call_actions(items: list[object]) -> list[dict[str, object]]: + return [ResponseFunctionWebSearch.model_validate(item, from_attributes=True).action.model_dump() for item in items] + + +_MASKED_WEB_CALL_ACTIONS: Final = [ + {"type": "search", "query": "[MASKED]", "queries": ["[MASKED]"], "sources": []}, + {"type": "open_page", "url": "https://[MASKED].example"}, + {"type": "search", "query": "persimmon", "queries": None, "sources": None}, +] + + class TestOpenAIResponsesHandlerDiscovery: """Test that the handler is properly discovered by the guardrail system""" @@ -1804,6 +1831,27 @@ class TestOpenAIResponsesHandlerToolCallExtraction: assert search_call.call_id == "srvtoolu_1" assert "name" not in search_call.model_dump() + @pytest.mark.asyncio + async def test_process_output_response_scans_and_rewrites_claude_web_calls_but_not_native_search(self): + guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask") + response = ResponsesAPIResponse( + id="resp_1", + created_at=1, + model="claude-sonnet-5-5", + object="response", + status="completed", + output=_claude_web_calls_and_a_native_search(), + ) + + result = await OpenAIResponsesHandler().process_output_response(response, guardrail) + + assert guardrail.seen_tool_calls == [ + ("web_search", '{"query": "persimmon"}'), + ("web_fetch", '{"url": "https://persimmon.example"}'), + ] + assert [item.type for item in result.output] == ["web_search_call"] * 3 + assert _web_call_actions(result.output) == _MASKED_WEB_CALL_ACTIONS + @pytest.mark.asyncio async def test_process_output_response_never_returns_original_search_arguments_after_a_non_json_rewrite(self): response = ResponsesAPIResponse( @@ -2739,6 +2787,117 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: assert completed.response.output[0].arguments == {"query": "[MASKED]"} assert "name" not in done.item.model_dump() + @pytest.mark.asyncio + async def test_deliver_ended_stream_scans_and_rewrites_claude_web_calls_but_not_native_search(self): + from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject + from litellm.types.llms.openai import ( + OutputItemAddedEvent, + OutputItemDoneEvent, + ResponseCompletedEvent, + ResponsesAPIStreamEvents, + ) + + items = _claude_web_calls_and_a_native_search() + added = [ + OutputItemAddedEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, + output_index=index, + item=BaseLiteLLMOpenAIResponseObject(**{**item.model_dump(), "action": None}), + ) + for index, item in enumerate(items) + ] + done = [ + OutputItemDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + output_index=index, + sequence_number=index + 3, + item=BaseLiteLLMOpenAIResponseObject(**item.model_dump()), + ) + for index, item in enumerate(items) + ] + completed = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=ResponsesAPIResponse( + id="resp_1", + created_at=1, + model="claude-sonnet-5-5", + object="response", + status="completed", + output=_claude_web_calls_and_a_native_search(), + ), + ) + guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask") + + await OpenAIResponsesHandler().process_output_streaming_response( + responses_so_far=[*added, *done, completed], + guardrail_to_apply=guardrail, + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert sorted(guardrail.seen_tool_calls) == [ + ("web_fetch", '{"url": "https://persimmon.example"}'), + ("web_search", '{"query": "persimmon"}'), + ] + assert [event.item.model_dump()["action"] for event in added] == [None, None, None] + assert _web_call_actions([event.item for event in done]) == _MASKED_WEB_CALL_ACTIONS + assert _web_call_actions(completed.response.output) == _MASKED_WEB_CALL_ACTIONS + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_dict_shaped_claude_web_calls_but_not_native_search(self): + items = [item.model_dump() for item in _claude_web_calls_and_a_native_search()] + done = [ + {"type": "response.output_item.done", "output_index": index, "item": dict(item)} + for index, item in enumerate(items) + ] + completed = { + "type": "response.completed", + "response": { + "id": "resp_1", + "model": "claude-sonnet-5-5", + "status": "completed", + "output": [dict(item) for item in items], + }, + } + guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask") + + await OpenAIResponsesHandler().process_output_streaming_response( + responses_so_far=[*done, completed], + guardrail_to_apply=guardrail, + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert sorted(guardrail.seen_tool_calls) == [ + ("web_fetch", '{"url": "https://persimmon.example"}'), + ("web_search", '{"query": "persimmon"}'), + ] + assert _web_call_actions([event["item"] for event in done]) == _MASKED_WEB_CALL_ACTIONS + assert _web_call_actions(completed["response"]["output"]) == _MASKED_WEB_CALL_ACTIONS + + @pytest.mark.asyncio + async def test_streaming_output_item_done_reports_a_claude_web_search_call(self): + from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject + from litellm.types.llms.openai import OutputItemDoneEvent, ResponsesAPIStreamEvents + + done = OutputItemDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + output_index=0, + sequence_number=1, + item=BaseLiteLLMOpenAIResponseObject( + **build_web_search_call("srvtoolu_1", {"query": "persimmon"}, None, "completed").model_dump() + ), + ) + guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask") + + await OpenAIResponsesHandler().process_output_streaming_response( + responses_so_far=[done], + guardrail_to_apply=guardrail, + litellm_logging_obj=None, + ) + + assert guardrail.seen_tool_calls == [("web_search", '{"query": "persimmon"}')] + @pytest.mark.asyncio async def test_deliver_ended_stream_rewrites_a_native_tool_search_call_that_has_no_call_id(self): def search_call(arguments: dict, status: str) -> dict: diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_cato_networks.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_cato_networks.py index 349030c6c75..7741973c15b 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_cato_networks.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_cato_networks.py @@ -2237,6 +2237,83 @@ async def test_post_call_success_hook_redacts_both_content_and_tool_arguments(): assert message.tool_calls[0].function.arguments == '{"to": "[NAME_1]"}' +@pytest.mark.asyncio +async def test_post_call_success_hook_inspects_and_redacts_server_tool_call_arguments(): + guard = _make_guardrail() + request_data = {"messages": [{"role": "user", "content": "look up persimmon"}]} + + def side_effect(url, *args, **kwargs): + last = kwargs["json"]["messages"][-1]["content"] + return _make_response( + { + "analysis_result": {"policy_drill_down": {"PII": {}}}, + "required_action": {"action_type": "anonymize_action", "policy_name": "PII"}, + "redacted_chat": { + "all_redacted_messages": [ + {"role": "user", "content": "look up persimmon"}, + {"role": "assistant", "content": last.replace("persimmon", "[FRUIT_1]")}, + ] + }, + } + ) + + llm_response = ModelResponse( + choices=[ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": None, + "role": "assistant", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + ], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + }, + } + ] + ) + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + side_effect=side_effect, + ) as mock_post: + result = await guard.async_post_call_success_hook( + data=request_data, + response=llm_response, + user_api_key_dict=UserAPIKeyAuth(), + ) + + inspected = [call.kwargs["json"]["messages"][-1] for call in mock_post.call_args_list] + assert inspected == [ + {"role": "assistant", "content": '{"query": "persimmon"}'}, + {"role": "assistant", "content": '{"city": "Paris"}'}, + ] + message = result.choices[0].message + assert message.provider_specific_fields == { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "[FRUIT_1]"}'}, + } + ] + } + assert message.content is None + assert message.tool_calls[0].function.arguments == '{"city": "Paris"}' + + def _make_responses_api_response(output: list) -> ResponsesAPIResponse: return ResponsesAPIResponse(id="resp-1", created_at=0, output=output) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py index 4d1f254aef9..ae4ce3fe26b 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py @@ -1777,6 +1777,78 @@ class TestCiscoAIDefenseToolCallBypass: f"included in the scanned text. Sent: {sent!r}" ) + @staticmethod + def _server_tool_call_response() -> ModelResponse: + from litellm.types.utils import ChatCompletionMessageToolCall, Function + + return ModelResponse( + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_1", + type="function", + function=Function(name="get_weather", arguments='{"city": "Paris"}'), + ) + ], + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + ), + ) + ] + ) + + @pytest.mark.asyncio + async def test_post_call_scan_sends_server_tool_call_arguments_to_cisco(self): + g = _make_guardrail(event_hook="post_call") + post_mock = AsyncMock(return_value=_safe_response()) + + with _patch_inspection_post(g, post_mock): + await g.async_post_call_success_hook( + data={"messages": [{"role": "user", "content": "look up persimmon"}]}, + user_api_key_dict=UserAPIKeyAuth(), + response=self._server_tool_call_response(), + ) + + assert [call.kwargs["json"]["messages"] for call in post_mock.call_args_list] == [ + [ + {"role": "user", "content": "look up persimmon"}, + {"role": "assistant", "content": '{"query": "persimmon"} {"city": "Paris"}'}, + ] + ] + + @pytest.mark.asyncio + async def test_post_call_redact_clears_server_tool_call_arguments(self): + g = _make_guardrail(event_hook="post_call", on_flagged_action="monitor") + cisco_resp = _redact_response(sanitized_text="[REDACTED]") + + with _patch_inspection_post(g, AsyncMock(return_value=cisco_resp)): + result = await g.async_post_call_success_hook( + data={"messages": [{"role": "user", "content": "look up persimmon"}]}, + user_api_key_dict=UserAPIKeyAuth(), + response=self._server_tool_call_response(), + ) + + message = result.choices[0].message + assert message.provider_specific_fields == { + "server_tool_calls": [ + {"id": "srvtoolu_01", "type": "function", "function": {"name": "web_search", "arguments": "{}"}} + ] + } + assert message.tool_calls[0].function.arguments == "{}" + class TestCiscoAIDefenseToolDefinitionBypass: diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_lasso.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_lasso.py index b8a219ba863..6b6849f4f49 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_lasso.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_lasso.py @@ -1,3 +1,4 @@ +import json import os import uuid from typing import Final @@ -1549,3 +1550,144 @@ async def test_api_error_handling_legacy_messages( assert "Failed to verify request safety with Lasso API" in str(excinfo.value) assert error_message in str(excinfo.value) assert route.called + + +def _reply_with_web_search_call( + arguments: str, tool_calls: list[dict[str, object]] | None = None +) -> litellm.ModelResponse: + return litellm.ModelResponse( + choices=[ + litellm.Choices( + index=0, + finish_reason="tool_calls", + message=litellm.Message( + role="assistant", + content=None, + tool_calls=tool_calls, + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": arguments}, + } + ] + }, + ), + ) + ] + ) + + +@pytest.mark.asyncio +async def test_post_call_scan_sends_server_tool_calls_as_tool_use_entries( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + lasso_guardrail: Final = LassoGuardrail( + lasso_api_key="test-key", + user_id="test-user", + conversation_id="test-conversation", + guardrail_name="test-guard", + event_hook="post_call", + default_on=True, + ) + route: Final = respx_mock.post("https://server.lasso.security/gateway/v3/classify").mock( + return_value=httpx.Response(200, json={"violations_detected": False, "deputies": {}, "findings": {}}) + ) + response: Final = _reply_with_web_search_call( + '{"query": "persimmon"}', + tool_calls=[ + { + "id": "call_01", + "type": "function", + "function": {"name": "lookup_order", "arguments": '{"order_id": "A-17"}'}, + } + ], + ) + + await lasso_guardrail.async_post_call_success_hook( + data={"messages": [{"role": "user", "content": "search persimmons"}]}, + user_api_key_dict=UserAPIKeyAuth(), + response=response, + ) + + assert json.loads(route.calls.last.request.content) == { + "messages": [ + { + "role": "model", + "content": { + "type": "tool_use", + "id": "srvtoolu_01", + "name": "web_search", + "input": {"query": "persimmon"}, + }, + }, + { + "role": "model", + "content": { + "type": "tool_use", + "id": "call_01", + "name": "lookup_order", + "input": {"order_id": "A-17"}, + }, + }, + ], + "messageType": "COMPLETION", + "source": {"type": "litellm"}, + "userId": "test-user", + "sessionId": "test-conversation", + } + + +@pytest.mark.asyncio +async def test_post_call_masking_rewrites_server_tool_call_arguments( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + lasso_guardrail: Final = LassoGuardrail( + lasso_api_key="test-key", + user_id="test-user", + conversation_id="test-conversation", + mask=True, + guardrail_name="test-guard", + event_hook="post_call", + default_on=True, + ) + respx_mock.post("https://server.lasso.security/gateway/v3/classifix").mock( + return_value=httpx.Response( + 200, + json={ + "violations_detected": True, + "deputies": {"pattern-detection": True}, + "findings": {"pattern-detection": [{"action": "AUTO_MASKING", "severity": "HIGH"}]}, + "messages": [ + { + "role": "model", + "content": { + "type": "tool_use", + "id": "srvtoolu_01", + "name": "web_search", + "input": {"query": " persimmon"}, + }, + } + ], + }, + ) + ) + + result: Final = await lasso_guardrail.async_post_call_success_hook( + data={"messages": [{"role": "user", "content": "search persimmons"}]}, + user_api_key_dict=UserAPIKeyAuth(), + response=_reply_with_web_search_call('{"query": "jane@example.com persimmon"}'), + ) + + assert result.choices[0].message.provider_specific_fields == { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": " persimmon"}'}, + } + ] + } diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py index 3da77a8707e..7e11505d0d0 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py @@ -893,6 +893,55 @@ class TestRestoration: assert result.choices[0].message.content == "a@b.com" + @pytest.mark.asyncio + async def test_server_tool_call_arguments_are_restored(self): + guardrail, _ = _shielded({"[EMAIL_1]": "a@b.com", "[URL_1]": "https://b.com/a"}) + response = ModelResponse( + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "[EMAIL_1] persimmon"}'}, + }, + { + "id": "srvtoolu_02", + "type": "function", + "function": {"name": "web_fetch", "arguments": '{"url": "[URL_1]"}'}, + }, + ] + }, + ), + ) + ] + ) + + result = await guardrail.async_post_call_success_hook( + data={"messages": []}, user_api_key_dict=None, response=response + ) + + assert result.choices[0].message.provider_specific_fields == { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "a@b.com persimmon"}'}, + }, + { + "id": "srvtoolu_02", + "type": "function", + "function": {"name": "web_fetch", "arguments": '{"url": "https://b.com/a"}'}, + }, + ] + } + @pytest.mark.asyncio async def test_responses_api_shape_is_restored(self): """The Responses API reply carries output items, not choices. diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py index 25d2556bf26..c7d7cb5da54 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py @@ -2072,6 +2072,63 @@ class TestCompletionResponseTextPartsToolCalls: sent_text = mock_check.call_args.kwargs["text"] assert '{"credit_card": "4111-1111-1111-1111"}' in sent_text + @pytest.mark.asyncio + async def test_post_call_hook_sends_server_tool_call_args_to_purview(self): + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.types.utils import Choices, Message, ModelResponse + + def graph(request: httpx.Request) -> httpx.Response: + if request.url.host == "login.microsoftonline.com": + return httpx.Response(200, json={"access_token": "graph-token", "expires_in": 3600}) + if not request.url.path.endswith("/processContent"): + return httpx.Response(200, json={"value": []}, headers={"ETag": "scope-etag"}) + return httpx.Response(200, json={"protectionScopeState": "unchanged", "policyActions": []}) + + transport: Final = Mock(side_effect=graph) + guardrail: Final = _make_guardrail() + guardrail.async_handler = AsyncHTTPHandler(transport=httpx.MockTransport(transport)) + response: Final = ModelResponse( + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + ], + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + ), + ) + ], + ) + + await guardrail.async_post_call_success_hook( + data={}, + user_api_key_dict=UserAPIKeyAuth(api_key="test", user_id="user-123"), + response=response, + ) + + scanned: Final = [ + json.loads(call.args[0].content)["contentToProcess"]["contentEntries"][0]["content"]["data"] + for call in transport.call_args_list + if call.args[0].url.path.endswith("/processContent") + ] + assert scanned == ['{"query": "persimmon"}\n\n---\n\n{"city": "Paris"}'] + def test_responses_api_function_call_args_included(self): """Function-call arguments in ``ResponsesAPIResponse.output`` must be DLP-scanned.""" from litellm.types.llms.openai import ResponsesAPIResponse diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py index a41331230c2..834bd4ce224 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py @@ -6513,5 +6513,86 @@ class TestPanwAirsToolCallBlockMaskedDataRouting: assert error["scan_id"] == "scan-tool-2" +class TestPanwAirsServerToolCalls: + SERVER_TOOL_CALL: Final = { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + + @classmethod + def _response(cls, *client_tool_calls: ChatCompletionMessageToolCall) -> ModelResponse: + return ModelResponse( + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=list(client_tool_calls) or None, + provider_specific_fields={"server_tool_calls": [copy.deepcopy(cls.SERVER_TOOL_CALL)]}, + ), + ) + ], + model="claude-opus-4-1", + ) + + @staticmethod + def _airs(scan_result: Mapping[str, object]) -> tuple[PanwPrismaAirsHandler, MagicMock]: + transport: Final = MagicMock(side_effect=lambda request: httpx.Response(200, json=scan_result)) + handler: Final = make_handler( + mask_response_content=True, + http_client=AsyncHTTPHandler(transport=httpx.MockTransport(transport)), + ) + return handler, transport + + @pytest.mark.asyncio + async def test_post_call_scan_sends_server_tool_call_query_to_airs(self, user_api_key_dict): + handler, transport = self._airs({"action": "allow", "category": "benign", "scan_id": "s1"}) + response: Final = self._response( + ChatCompletionMessageToolCall( + id="call_1", + type="function", + function=Function(name="get_weather", arguments='{"city": "Paris"}'), + ) + ) + + await handler.async_post_call_success_hook( + data={"model": "claude-opus-4-1", "litellm_call_id": "call-srv", "metadata": {}}, + user_api_key_dict=user_api_key_dict, + response=response, + ) + + sent_contents: Final = [json.loads(call.args[0].content)["contents"] for call in transport.call_args_list] + assert sent_contents == [[{"response": '{"query": "persimmon"} {"city": "Paris"}'}]] + + @pytest.mark.asyncio + async def test_post_call_masking_rewrites_server_tool_call_arguments(self, user_api_key_dict): + masked: Final = '{"query": "*********"}' + handler, _ = self._airs( + { + "action": "block", + "category": "sensitive_data", + "scan_id": "s1", + "response_masked_data": {"data": masked}, + } + ) + + result: Final = await handler.async_post_call_success_hook( + data={"model": "claude-opus-4-1", "litellm_call_id": "call-srv-mask", "metadata": {}}, + user_api_key_dict=user_api_key_dict, + response=self._response(), + ) + + message: Final = result.choices[0].message + assert message.provider_specific_fields == { + "server_tool_calls": [ + {"id": "srvtoolu_01", "type": "function", "function": {"name": "web_search", "arguments": masked}} + ] + } + assert message.tool_calls is None + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py index e312378969a..43ed53cca1d 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -1781,6 +1781,48 @@ async def test_pii_tokens_in_metadata_used_for_unmasking(): assert response.choices[0].message.content == "Hello John, how can I help you?" +@pytest.mark.asyncio +async def test_output_unmasking_restores_server_tool_call_arguments(mock_user_api_key): + guardrail: Final = OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True) + response: Final = ModelResponse( + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": " persimmon"}'}, + } + ] + }, + ), + ) + ] + ) + + result: Final = await guardrail.async_post_call_success_hook( + data={"metadata": {"pii_tokens": {"": "John Smith"}}}, + user_api_key_dict=mock_user_api_key, + response=response, + ) + + assert result.choices[0].message.provider_specific_fields == { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "John Smith persimmon"}'}, + } + ] + } + + @pytest.mark.parametrize( "initial_hook", ["pre_call", "during_call", "pre_mcp_call"], diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_repelloai.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_repelloai.py index 77883e9af0e..246636e016f 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_repelloai.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_repelloai.py @@ -852,6 +852,59 @@ class TestRepelloAIPostCall: in captured["json"]["scan_data"]["response"] ) + @pytest.mark.asyncio + async def test_server_tool_call_arguments_scanned(self, monkeypatch): + guardrail = _guardrail(event_hook="post_call") + response = ModelResponse( + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_01", + "type": "function", + "function": { + "name": "lookup_order", + "arguments": '{"order_id": "A-17"}', + }, + } + ], + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": { + "name": "web_search", + "arguments": '{"query": "persimmon"}', + }, + } + ] + }, + ), + ) + ] + ) + captured = {} + + async def capture(url, headers, json, **_kwargs): + captured["json"] = json + return _verdict_response("passed", url) + + monkeypatch.setattr(guardrail.async_handler, "post", capture) + await guardrail.async_post_call_success_hook( + data={"messages": [{"role": "user", "content": "q"}]}, + user_api_key_dict=UserAPIKeyAuth(), + response=response, + ) + assert captured["json"]["scan_data"] == { + "response": '{"query": "persimmon"}\n{"order_id": "A-17"}' + } + # ---------------------------------------------------------------------- # verdict handling: unknown / malformed responses must not fail open From 87d0edfd0a9b88d3ffe93c785ea598e53cafb4ac Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 07:43:04 -0700 Subject: [PATCH 7/9] fix(anthropic): keep Claude server calls visible to guardrails, redaction and tracing A guardrail rewrite of a Claude server call now also updates its hosted web_search_calls copy, streamed or not. Redaction with message logging off hides server call arguments and hosted web call actions on chat and Responses. Pillar and Pangea scan server calls, transform-mode streaming guardrails keep server call chunks, and OTEL v2 Responses spans list Claude's web calls. --- litellm/integrations/otel/model/payloads.py | 29 +++- litellm/litellm_core_utils/redact_messages.py | 63 ++++++++- litellm/llms/anthropic/common_utils.py | 112 ++++++++++++++- .../chat/guardrail_translation/handler.py | 30 ++++ .../guardrail_hooks/pangea/pangea.py | 25 +++- .../guardrail_hooks/pillar/pillar.py | 5 +- .../unified_guardrail/unified_guardrail.py | 18 ++- .../transformation.py | 13 +- litellm/types/llms/anthropic.py | 11 ++ .../otel/test_otel_v2_sources_of_truth.py | 32 +++++ .../test_redact_messages.py | 114 ++++++++++++++- .../test_openai_guardrail_handler.py | 133 +++++++++++++++++- .../guardrails/guardrail_hooks/test_pangea.py | 66 +++++++++ .../test_unified_guardrail.py | 31 ++++ .../guardrails/test_pillar_guardrails.py | 44 ++++++ 15 files changed, 695 insertions(+), 31 deletions(-) diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index 7a62ed6aed9..4238d72fc5e 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -10,6 +10,8 @@ from types import MappingProxyType from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, cast from urllib.parse import urlsplit +from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch +from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm.integrations.otel.model.metadata import ( @@ -36,6 +38,7 @@ from litellm.integrations.otel.model.utils import ( as_str_tuple, ) from litellm.integrations.otel.routing import RoutingAttributeValue, routing_decision_attributes +from litellm.types.llms.anthropic import claude_hosted_web_tool_call # ``RequestIdentity`` and the request-metadata translation now live in # :mod:`metadata`; re-exported here so existing ``model.payloads`` imports keep @@ -809,6 +812,7 @@ class _Choice(TypedDict): _RESPONSES_TOOL_CALL_TYPES: Final = frozenset({"function_call", "custom_tool_call"}) +_WEB_SEARCH_CALL: Final = TypeAdapter(ResponseFunctionWebSearch) def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: @@ -816,9 +820,13 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: items: Final = _dicts(response.get("output")) messages: Final = tuple(item for item in items if item.get("type") == "message") parts: Final = tuple(part for item in messages for part in _dicts(item.get("content"))) - tool_calls: Final = tuple( + client_tool_calls: Final = tuple( _responses_tool_call(item) for item in items if item.get("type") in _RESPONSES_TOOL_CALL_TYPES ) + tool_calls: Final = ( + *(call for item in items if (call := _claude_web_search_tool_call(item)) is not None), + *client_tool_calls, + ) if not messages and not tool_calls: return () message: Final[_AssistantMessage] = { @@ -827,7 +835,10 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: "refusal": _responses_parts_text(parts, "refusal", "refusal"), "tool_calls": tool_calls or None, } - choice: Final[_Choice] = {"message": message, "finish_reason": _responses_finish_reason(response, bool(tool_calls))} + choice: Final[_Choice] = { + "message": message, + "finish_reason": _responses_finish_reason(response, bool(client_tool_calls)), + } return (choice,) @@ -981,6 +992,20 @@ def _responses_tool_call(item: Mapping[str, object]) -> _ToolCall: return tool_call +def _claude_web_search_tool_call(item: Mapping[str, object]) -> _ToolCall | None: + if item.get("type") != "web_search_call": + return None + try: + search: Final = _WEB_SEARCH_CALL.validate_python(item) + except ValidationError: + return None + claude_call: Final = claude_hosted_web_tool_call(search) + if claude_call is None: + return None + tool_id, name, arguments = claude_call + return {"id": tool_id, "type": "function", "function": {"name": name, "arguments": arguments}} + + def _responses_finish_reason(response: Mapping[str, object], has_tool_calls: bool) -> str | None: status: Final = as_str(response.get("status")) if status == "completed": diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 548d088d833..220e572c659 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -14,7 +14,7 @@ from collections.abc import Mapping, Sequence from dataclasses import replace from typing import TYPE_CHECKING, Any, Final -from pydantic import JsonValue, TypeAdapter, ValidationError +from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError import litellm from litellm.constants import REDACTED_BY_LITELLM @@ -253,9 +253,63 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[ ) if key in fields }, + **( + {"server_tool_calls": _redacted_tool_call_dicts(fields["server_tool_calls"])} + if "server_tool_calls" in fields + else {} + ), + **( + {"web_search_calls": _redacted_web_search_calls(fields["web_search_calls"])} + if "web_search_calls" in fields + else {} + ), } +def _redacted_tool_call_dict(call: object) -> object: + if (mapping := _as_mapping(call)) is None or (function := _as_mapping(mapping.get("function"))) is None: + return call + return {**mapping, "function": {**function, "arguments": REDACTED_BY_LITELLM}} + + +def _redacted_tool_call_dicts(calls: object) -> object: + return [_redacted_tool_call_dict(call) for call in calls] if isinstance(calls, list) else calls + + +_WEB_ACTION_TEXT_FIELDS: Final = ("query", "url", "pattern") +_WEB_ACTION_LIST_FIELDS: Final = ("queries", "sources") + + +def _web_action_redactions(action: Mapping[str, object]) -> Mapping[str, object]: + return { + **{key: REDACTED_BY_LITELLM for key in _WEB_ACTION_TEXT_FIELDS if action.get(key) is not None}, + **{key: None for key in _WEB_ACTION_LIST_FIELDS if action.get(key) is not None}, + } + + +def _redacted_web_search_action(action: object) -> object: + if isinstance(action, BaseModel): + return action.model_copy(update=dict(_web_action_redactions(action.model_dump()))) + mapping: Final = _as_mapping(action) + return action if mapping is None else {**mapping, **_web_action_redactions(mapping)} + + +def _redacted_web_search_call(item: object) -> object: + if isinstance(item, BaseModel): + action: Final = getattr(item, "action", None) + return item if action is None else item.model_copy(update={"action": _redacted_web_search_action(action)}) + mapping: Final = _as_mapping(item) + if mapping is None or mapping.get("action") is None: + return item + return {**mapping, "action": _redacted_web_search_action(mapping["action"])} + + +def _redacted_web_search_calls(calls: object) -> object: + if (mapping := _as_mapping(calls)) is not None: + return {key: _redacted_web_search_call(call) for key, call in mapping.items()} + return [_redacted_web_search_call(call) for call in calls] if isinstance(calls, list) else calls + + def _redact_field_when_present(item: object, field: str) -> None: if getattr(item, field, None) is not None: setattr(item, field, REDACTED_BY_LITELLM) @@ -282,6 +336,8 @@ def _redact_responses_api_output(output_items): output_item.arguments = REDACTED_BY_LITELLM if output_type == "custom_tool_call" and hasattr(output_item, "input"): output_item.input = REDACTED_BY_LITELLM + if output_type == "web_search_call" and getattr(output_item, "action", None) is not None: + output_item.action = _redacted_web_search_action(output_item.action) _RESPONSE_ITEMS: Final = TypeAdapter(tuple[object, ...]) @@ -336,6 +392,11 @@ def _redacted_responses_api_output_item(item: object, redacted_str: str) -> obje if item_mapping.get("type") == "custom_tool_call" and "input" in item_mapping else {} ), + **( + {"action": _redacted_web_search_action(item_mapping["action"])} + if item_mapping.get("type") == "web_search_call" and item_mapping.get("action") is not None + else {} + ), } diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index ad70496300a..fd1645b697f 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -68,6 +68,11 @@ from litellm.types.llms.openai import ( ) from litellm.types.proxy.auth.special_headers import SpecialHeaders from litellm.types.proxy.model_listing import ModelInfoResponse +from litellm.types.responses.main import ( + ResponseFunctionWebSearch, + web_search_call_tool_id, + with_hosted_web_tool_arguments, +) from litellm.types.utils import ( ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall, @@ -129,6 +134,9 @@ _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _SERVER_TOOL_CALLS_ADAPTER: Final = TypeAdapter(tuple[ChatCompletionAssistantToolCall, ...]) +_SERVER_TOOL_CALL_ADAPTER: Final = TypeAdapter(ChatCompletionAssistantToolCall) +_SERVER_CALL_FIELD_KEYS: Final = ("server_tool_calls", "web_search_calls") +_WEB_SEARCH_CALL_ADAPTER: Final = TypeAdapter(ResponseFunctionWebSearch) _TOOL_CALL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _DECODED_JSON: Final = TypeAdapter(object) @@ -2153,6 +2161,53 @@ def tool_calls_with_server_tool_calls( return (*anthropic_server_tool_calls(message), *client_tool_calls) +def anthropic_server_call_fields(holder: object) -> Mapping[str, object]: + fields: Final = validated_object_mapping(message_field(holder, "provider_specific_fields")) or {} + return MappingProxyType({key: fields[key] for key in _SERVER_CALL_FIELD_KEYS if fields.get(key)}) + + +def message_dict_with_server_tool_calls(message: object) -> object: + mapping: Final = validated_object_mapping(message) + server_tool_calls: Final = anthropic_server_tool_call_params(message) + if mapping is None or not server_tool_calls: + return message + return {**mapping, "tool_calls": [*server_tool_calls, *parts_of(mapping.get("tool_calls"))]} + + +def _server_tool_call_param(tool_call: object) -> ChatCompletionAssistantToolCall | None: + try: + call: Final = _SERVER_TOOL_CALL_ADAPTER.validate_python(tool_call) + except ValidationError: + return None + return call if _server_tool_call_id(call).startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) else None + + +def message_dict_with_server_tool_calls_split_out(message: object) -> object: + mapping: Final = validated_object_mapping(message) + tool_calls: Final = parts_of(mapping.get("tool_calls")) if mapping is not None else () + server_tool_calls: Final = tuple( + call for tool_call in tool_calls if (call := _server_tool_call_param(tool_call)) is not None + ) + if mapping is None or not server_tool_calls: + return message + fields: Final = validated_object_mapping(mapping.get("provider_specific_fields")) or {} + changed: Final = changed_server_tool_call_arguments(anthropic_server_tool_call_params(message), server_tool_calls) + client_tool_calls: Final = [tool_call for tool_call in tool_calls if _server_tool_call_param(tool_call) is None] + return { + **mapping, + "tool_calls": client_tool_calls or None, + "provider_specific_fields": { + **fields, + "server_tool_calls": list(server_tool_calls), + **( + {"web_search_calls": _rewritten_web_search_calls(fields["web_search_calls"], changed)} + if changed and fields.get("web_search_calls") is not None + else {} + ), + }, + } + + def with_server_tool_call_arguments( call: ChatCompletionAssistantToolCall, arguments: str ) -> ChatCompletionAssistantToolCall: @@ -2166,12 +2221,67 @@ def with_server_tool_call_arguments( def write_anthropic_server_tool_calls( holder: Message | Delta, rewritten: tuple[ChatCompletionAssistantToolCall, ...] ) -> None: - if rewritten == anthropic_server_tool_call_params(holder): + original: Final = anthropic_server_tool_call_params(holder) + if rewritten == original: return holder.provider_specific_fields = { # rebind-ok: a guardrail rewrite must land on the message it scanned **(holder.provider_specific_fields or {}), "server_tool_calls": list(rewritten), } + write_anthropic_hosted_web_call_arguments(holder, changed_server_tool_call_arguments(original, rewritten)) + + +def _server_tool_call_id(call: ChatCompletionAssistantToolCall) -> str: + return call["id"] or "" + + +def _server_tool_call_arguments(call: ChatCompletionAssistantToolCall) -> str: + return call["function"].get("arguments") or "" + + +def changed_server_tool_call_arguments( + original: Sequence[ChatCompletionAssistantToolCall], rewritten: Sequence[ChatCompletionAssistantToolCall] +) -> Mapping[str, str]: + original_arguments: Final = MappingProxyType( + {_server_tool_call_id(call): _server_tool_call_arguments(call) for call in original} + ) + return MappingProxyType( + { + _server_tool_call_id(call): _server_tool_call_arguments(call) + for call in rewritten + if _server_tool_call_arguments(call) != original_arguments.get(_server_tool_call_id(call)) + } + ) + + +def write_anthropic_hosted_web_call_arguments(holder: Message | Delta, arguments_by_tool_id: Mapping[str, str]) -> None: + fields: Final = holder.provider_specific_fields + if not arguments_by_tool_id or not fields or fields.get("web_search_calls") is None: + return + holder.provider_specific_fields = { # rebind-ok: the hosted copy of a rewritten call must not keep its old input + **fields, + "web_search_calls": _rewritten_web_search_calls(fields["web_search_calls"], arguments_by_tool_id), + } + + +def _rewritten_web_search_calls(calls: object, arguments_by_tool_id: Mapping[str, str]) -> object: + mapping: Final = validated_object_mapping(calls) + if mapping is not None: + return {key: _with_hosted_web_call_arguments(call, arguments_by_tool_id) for key, call in mapping.items()} + if not isinstance(calls, list): + return calls + return [_with_hosted_web_call_arguments(call, arguments_by_tool_id) for call in parts_of(calls)] + + +def _with_hosted_web_call_arguments(item: object, arguments_by_tool_id: Mapping[str, str]) -> object: + try: + call: Final = _WEB_SEARCH_CALL_ADAPTER.validate_python(item) + except ValidationError: + return item + arguments: Final = arguments_by_tool_id.get(web_search_call_tool_id(call.id) or "") + if arguments is None: + return item + return call.model_copy(update={"action": with_hosted_web_tool_arguments(call.action, arguments)}) def write_anthropic_server_tool_call_arguments(holder: Message | Delta, index: int, arguments: str) -> None: diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 92a9a8ad4c2..51dae634ab0 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -32,6 +32,8 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder from litellm.llms.anthropic.common_utils import ( anthropic_server_tool_call_params, + changed_server_tool_call_arguments, + write_anthropic_hosted_web_call_arguments, write_anthropic_server_tool_calls, ) from litellm.llms.base_llm.guardrail_translation.base_translation import ( @@ -1264,6 +1266,16 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) if not guardrailed_by_id: return + original_by_id: Final = MappingProxyType(dict(_streamed_server_tool_calls(responses_so_far))) + changed_by_choice: Final = MappingProxyType( + { + choice_index: changed_server_tool_call_arguments( + _choice_server_tool_calls(original_by_id, choice_index), + _choice_server_tool_calls(guardrailed_by_id, choice_index), + ) + for choice_index in frozenset(index for index, _ in guardrailed_by_id) + } + ) for response in responses_so_far: for choice in response.choices: write_anthropic_server_tool_calls( @@ -1273,6 +1285,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation): for server_tool_call in anthropic_server_tool_call_params(choice.delta) ), ) + write_anthropic_hosted_web_call_arguments( + choice.delta, changed_by_choice.get(choice.index, MappingProxyType({})) + ) async def _apply_guardrail_responses_to_output_streaming( self, @@ -1476,6 +1491,21 @@ def _server_tool_call_tasks( yield (choice_idx, server_idx), server_tool_call +def _streamed_server_tool_calls( + responses: Sequence["ModelResponseStream"], +) -> Iterator[tuple[tuple[int, str | None], ChatCompletionAssistantToolCall]]: + for response in responses: + for choice in response.choices: + for server_tool_call in anthropic_server_tool_call_params(choice.delta): + yield (choice.index, server_tool_call["id"]), server_tool_call + + +def _choice_server_tool_calls( + calls_by_id: Mapping[tuple[int, str | None], ChatCompletionAssistantToolCall], choice_index: int +) -> tuple[ChatCompletionAssistantToolCall, ...]: + return tuple(call for (index, _), call in calls_by_id.items() if index == choice_index) + + def _server_tool_call_input(server_tool_call: ChatCompletionAssistantToolCall) -> dict[str, object]: return { "id": server_tool_call["id"], diff --git a/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py b/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py index 2194238cea0..4ec3ed21359 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py @@ -1,5 +1,6 @@ # litellm/proxy/guardrails/guardrail_hooks/pangea.py import os +from collections.abc import Callable from typing import TYPE_CHECKING, Final from fastapi import HTTPException @@ -10,6 +11,11 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.llms.anthropic.common_utils import ( + message_dict_with_server_tool_calls, + message_dict_with_server_tool_calls_split_out, + validated_object_mapping, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -30,6 +36,13 @@ if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +def _choice_with_message(choice: object, rewrite: Callable[[object], object]) -> object: + mapping: Final = validated_object_mapping(choice) + if mapping is None or "message" not in mapping: + return choice + return {**mapping, "message": rewrite(mapping["message"])} + + class PangeaGuardrailMissingSecrets(Exception): """Custom exception for missing Pangea secrets.""" @@ -242,9 +255,13 @@ class PangeaHandler(CustomGuardrail): for c in choices: if isinstance(c, Choices): try: - serialized_choices.append(c.model_dump()) + serialized_choices.append( + _choice_with_message(c.model_dump(), message_dict_with_server_tool_calls) + ) except Exception: - serialized_choices.append(c.dict()) + serialized_choices.append( + _choice_with_message(c.dict(), message_dict_with_server_tool_calls) + ) else: serialized_choices.append(c) choices = serialized_choices @@ -269,7 +286,9 @@ class PangeaHandler(CustomGuardrail): return output: Final = ai_guard_response.get("result", {}).get("output", {}) - response.choices = output["choices"] + response.choices = [ # pyright: ignore[reportAttributeAccessIssue] # Pangea returns untyped choice dicts + _choice_with_message(choice, message_dict_with_server_tool_calls_split_out) for choice in output["choices"] + ] return response @log_guardrail_information diff --git a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py index 3f025cfc8b3..0facab6ddef 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py @@ -26,6 +26,7 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.anthropic.common_utils import message_dict_with_server_tool_calls from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -430,7 +431,9 @@ class PillarGuardrail(CustomGuardrail): # Extract response messages in the format Pillar expects response_dict = response.model_dump() if hasattr(response, "model_dump") else {} response_messages: Final = [ - choice.get("message") for choice in response_dict.get("choices", []) if choice.get("message") + message_dict_with_server_tool_calls(choice["message"]) + for choice in response_dict.get("choices", []) + if choice.get("message") ] if not response_messages: diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 5ce2dfb3b87..6f6d056423f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -25,6 +25,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket from litellm.llms import get_guardrail_translation_mapping, load_guardrail_translation_mappings +from litellm.llms.anthropic.common_utils import anthropic_server_call_fields from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import ( @@ -144,6 +145,15 @@ def _ends_stream(endpoint_translation: object, item: object) -> bool: return has_ended is not None and has_ended((item,)) +def _passthrough_delta(delta: object) -> Delta: + server_call_fields: Final = anthropic_server_call_fields(delta) + role: Final = getattr(delta, "role", None) + tool_calls: Final = getattr(delta, "tool_calls", None) + if not server_call_fields: + return Delta(content=None, role=role, tool_calls=tool_calls) + return Delta(content=None, role=role, tool_calls=tool_calls, provider_specific_fields=dict(server_call_fields)) + + def _held_choices(held_chars_per_choice: Mapping[int, int]) -> frozenset[int]: return frozenset(idx for idx, held in held_chars_per_choice.items() if held > 0) @@ -974,7 +984,7 @@ class UnifiedLLMGuardrails(CustomLogger): def _chunk_has_tool_calls(item: object) -> bool: for choice in _chunk_choices(item): delta = getattr(choice, "delta", None) - if getattr(delta, "tool_calls", None): + if getattr(delta, "tool_calls", None) or anthropic_server_call_fields(delta): return True return False @@ -1042,11 +1052,7 @@ class UnifiedLLMGuardrails(CustomLogger): synthetic_choices.append( StreamingChoices( index=idx, - delta=Delta( - content=None, - role=getattr(delta, "role", None), - tool_calls=getattr(delta, "tool_calls", None), - ), + delta=_passthrough_delta(delta), finish_reason=passthrough_finish, ) ) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 683e51be6e8..f6dc10a4422 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -71,7 +71,7 @@ from litellm.responses.litellm_completion_transformation.session_handler import ResponsesSessionHandler, StoredTurns, ) -from litellm.types.llms.anthropic import ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, AnthropicToolSearchToolRegex +from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex, claude_hosted_web_tool_call from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( AllMessageValues, @@ -120,8 +120,6 @@ from litellm.types.responses.main import ( OutputImageGenerationCall, OutputText, OutputToolSearchCall, - hosted_web_tool_call, - web_search_call_tool_id, ) from litellm.types.utils import ( ChatCompletionAnnotation, @@ -882,13 +880,10 @@ def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, obj def claude_web_search_call_tool_call(search: ResponseFunctionWebSearch) -> ChatCompletionAssistantToolCall | None: - tool_id: Final = web_search_call_tool_id(search.id) - if tool_id is None or not tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX): + claude_call: Final = claude_hosted_web_tool_call(search) + if claude_call is None: return None - hosted_call: Final = hosted_web_tool_call(search.action) - if hosted_call is None: - return None - name, arguments = hosted_call + tool_id, name, arguments = claude_call return ChatCompletionAssistantToolCall( id=tool_id, type="function", diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 50c4a2340a3..799d63bc8b9 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -2,10 +2,12 @@ from collections.abc import Iterable, Mapping, Sequence from enum import Enum from typing import Final, Literal, TypeAlias +from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch from pydantic import ConfigDict from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.responses.main import hosted_web_tool_call, web_search_call_tool_id from .openai import ( ChatCompletionCachedContent, @@ -826,6 +828,15 @@ ANTHROPIC_TOOL_SEARCH_TOOL_TYPES: Final = frozenset( ANTHROPIC_TOOL_SEARCH_TOOL_NAMES: Final = frozenset({"tool_search_tool_regex", "tool_search_tool_bm25"}) ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX: Final = "srvtoolu_" + +def claude_hosted_web_tool_call(search: ResponseFunctionWebSearch) -> tuple[str, str, str] | None: + tool_id: Final = web_search_call_tool_id(search.id) + if tool_id is None or not tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX): + return None + hosted_call: Final = hosted_web_tool_call(search.action) + return None if hosted_call is None else (tool_id, *hosted_call) + + # Effort beta header constant ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24" diff --git a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py index 2d0b4701c9f..961bfcd3867 100644 --- a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py @@ -48,6 +48,7 @@ from litellm.integrations.otel.model.spans import ( validate_registry, ) from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls +from litellm.types.responses.main import build_web_search_call @pytest.fixture(autouse=True) @@ -798,6 +799,37 @@ def test_responses_tool_calls_fold_into_the_assistant_message_with_tool_calls_fi assert data.finish_reasons == ("tool_calls",) +def test_responses_claude_web_calls_lead_the_tool_calls_without_ending_the_turn_on_tool_calls(): + claude_fetch = build_web_search_call("srvtoolu_01", {"url": "https://example.com/"}, None, tool_name="web_fetch") + openai_search = { + "type": "web_search_call", + "id": "ws_1", + "status": "completed", + "action": {"type": "search", "query": "news"}, + } + client_call = {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": "{}"} + claude_tool_call = { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_fetch", "arguments": '{"url": "https://example.com/"}'}, + } + + hosted_only = LLMCallSpanData.from_standard_logging_payload( + _responses_payload([_RESPONSES_TEXT_ITEM, claude_fetch.model_dump(), openai_search]), capture_content=True + ) + with_client_call = LLMCallSpanData.from_standard_logging_payload( + _responses_payload([_RESPONSES_TEXT_ITEM, claude_fetch.model_dump(), client_call]), capture_content=True + ) + + assert json.loads(json.dumps(hosted_only.choices_out[0]["message"]["tool_calls"])) == [claude_tool_call] + assert hosted_only.finish_reasons == ("stop",) + assert json.loads(json.dumps(with_client_call.choices_out[0]["message"]["tool_calls"])) == [ + claude_tool_call, + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}, + ] + assert with_client_call.finish_reasons == ("tool_calls",) + + def test_responses_tool_call_only_output_has_no_content(): data = LLMCallSpanData.from_standard_logging_payload( _responses_payload([{"type": "function_call", "id": "fc_1", "name": "get_weather", "arguments": "{}"}]), diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 5b64e42d8a8..76207d5fc15 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -20,10 +20,11 @@ from unittest.mock import patch import httpx import pytest import pytest_asyncio -from pydantic import JsonValue +from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch +from pydantic import JsonValue, TypeAdapter import litellm -from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE +from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE, REDACTED_BY_LITELLM from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.redact_messages import ( @@ -37,6 +38,7 @@ from litellm.litellm_core_utils.redact_messages import ( ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.responses.main import mock_responses_api_response +from litellm.types.responses.main import build_web_search_call from litellm.types.router import BaselineRouteStamp from litellm.types.utils import ( ModelResponse, @@ -168,6 +170,114 @@ def test_streaming_redaction_redacts_delta_provider_fields_without_touching_the_ assert fields == _provider_fields() +_SECRET_URL: Final = f"https://example.com/?q={SECRET}" +_WEB_SEARCH_CALL: Final = TypeAdapter(ResponseFunctionWebSearch) + + +def _claude_fetch_call() -> ResponseFunctionWebSearch: + return build_web_search_call( + "srvtoolu_01Fetch", {"url": _SECRET_URL}, {"content": []}, status="in_progress", tool_name="web_fetch" + ) + + +def _web_call_summaries(calls: object) -> list[tuple[str, str | None, dict[str, object]]]: + items: Final = calls.values() if isinstance(calls, dict) else calls + return [ + (call.id, call.status, call.action.model_dump(exclude_none=True)) + for call in (_WEB_SEARCH_CALL.validate_python(item) for item in items) + ] + + +@pytest.mark.parametrize("shape", ["object", "dict"]) +def test_redaction_hides_claude_server_call_inputs_but_keeps_the_calls(shape: str) -> None: + response: Final = ModelResponse( + model="claude-opus-5-5", + choices=[ + { + "message": { + "role": "assistant", + "content": None, + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01Fetch", + "type": "function", + "function": {"name": "web_fetch", "arguments": json.dumps({"url": _SECRET_URL})}, + } + ], + "web_search_calls": [ + _claude_fetch_call(), + build_web_search_call( + "srvtoolu_01Search", {"query": SECRET}, {"content": []}, status="in_progress" + ), + ], + }, + } + } + ], + ) + result: Final = response if shape == "object" else response.model_dump() + + redacted: Final = perform_redaction({}, result) + + message: Final = ( + redacted.choices[0].message if shape == "object" else SimpleNamespace(**redacted["choices"][0]["message"]) + ) + fields: Final = message.provider_specific_fields + assert fields["server_tool_calls"][0]["id"] == "srvtoolu_01Fetch" + assert fields["server_tool_calls"][0]["function"] == {"name": "web_fetch", "arguments": REDACTED_BY_LITELLM} + assert _web_call_summaries(fields["web_search_calls"]) == [ + ("ws_srvtoolu_01Fetch", "in_progress", {"type": "open_page", "url": REDACTED_BY_LITELLM}), + ("ws_srvtoolu_01Search", "in_progress", {"type": "search", "query": REDACTED_BY_LITELLM}), + ] + assert SECRET not in str(redacted), redacted + assert SECRET in str(result) + + +@pytest.mark.parametrize("key", ["complete_streaming_response", "async_complete_streaming_response"]) +def test_streaming_redaction_hides_the_hosted_copy_of_a_claude_web_fetch(key: str) -> None: + response: Final = ModelResponseStream( + choices=[ + litellm.utils.StreamingChoices( + delta=litellm.utils.Delta( + provider_specific_fields={"web_search_calls": {"srvtoolu_01Fetch": _claude_fetch_call()}} + ) + ) + ] + ) + + perform_redaction({"stream": True, key: response}, None) + + assert _web_call_summaries(response.choices[0].delta.provider_specific_fields["web_search_calls"]) == [ + ("ws_srvtoolu_01Fetch", "in_progress", {"type": "open_page", "url": REDACTED_BY_LITELLM}) + ] + + +@pytest.mark.parametrize("shape", ["object", "dict"]) +def test_redaction_hides_web_search_call_actions_in_responses_api_output(shape: str) -> None: + response: Final = ResponsesAPIResponse( + id="resp_1", + created_at=1, + model="claude-opus-5-5", + object="response", + status="completed", + output=[ + build_web_search_call("srvtoolu_01Fetch", {"url": _SECRET_URL}, {"content": []}, tool_name="web_fetch"), + build_web_search_call("srvtoolu_01Search", {"query": SECRET}, {"content": []}), + ], + ) + result: Final = response if shape == "object" else response.model_dump() + + redacted: Final = perform_redaction({}, result) + + output: Final = redacted.output if shape == "object" else redacted["output"] + assert _web_call_summaries(output) == [ + ("ws_srvtoolu_01Fetch", "completed", {"type": "open_page", "url": REDACTED_BY_LITELLM}), + ("ws_srvtoolu_01Search", "completed", {"type": "search", "query": REDACTED_BY_LITELLM}), + ] + assert SECRET in str(result) + + @pytest.mark.parametrize("shape", ["object", "dict"]) def test_redaction_hides_a_compaction_summary_in_responses_api_output(shape: str) -> None: response: Final = ResponsesAPIResponse( diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index fcc3b1e2b3c..bb205c5e251 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -17,6 +17,7 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import Streami from litellm.llms.openai.chat.guardrail_translation.handler import ( OpenAIChatCompletionsHandler, ) +from litellm.types.responses.main import build_web_search_call from litellm.types.utils import ( ChatCompletionMessageToolCall, Choices, @@ -155,8 +156,19 @@ def _tool_call_ids_names_and_arguments(tool_calls) -> list[tuple[str, str, str]] ] +def _hosted_web_call_inputs(web_search_calls) -> list[tuple[str, str, Optional[str]]]: + return [ + (call.id, call.status, call.action.query if call.action.type == "search" else call.action.url) + for call in web_search_calls + ] + + +_SEARCH_INPUT = {"query": "jane@example.com"} +_FETCH_INPUT = {"url": "https://example.com/?email=jane@example.com"} + + class QueryRedactingGuardrail(CustomGuardrail): - """Returns copied tool calls with any ``query`` argument redacted, leaving other calls as they came.""" + """Returns copied tool calls with any ``query`` or ``url`` argument redacted, leaving other calls as they came.""" async def apply_guardrail( self, @@ -167,12 +179,12 @@ class QueryRedactingGuardrail(CustomGuardrail): ) -> GenericGuardrailAPIInputs: def redacted(tool_call): arguments = json.loads(tool_call["function"]["arguments"]) - if "query" not in arguments: - return tool_call - return { - **tool_call, - "function": {**tool_call["function"], "arguments": json.dumps({**arguments, "query": "[REDACTED]"})}, + redacted_arguments = { + key: "[REDACTED]" if key in ("query", "url") else value for key, value in arguments.items() } + if redacted_arguments == arguments: + return tool_call + return {**tool_call, "function": {**tool_call["function"], "arguments": json.dumps(redacted_arguments)}} return GenericGuardrailAPIInputs( texts=inputs.get("texts", []), @@ -1028,6 +1040,46 @@ class TestOpenAIChatCompletionsHandlerToolCallsOutput: ] assert not (second.message.provider_specific_fields or {}).get("server_tool_calls") + @pytest.mark.asyncio + async def test_output_guardrail_rewrite_of_a_server_web_call_reaches_its_hosted_copy(self): + handler = OpenAIChatCompletionsHandler() + response = ModelResponse( + model="claude-sonnet-4-5", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + content=None, + role="assistant", + provider_specific_fields={ + "server_tool_calls": [ + _server_tool_call("srvtoolu_01Search", "web_search", _SEARCH_INPUT), + _server_tool_call("srvtoolu_01Fetch", "web_fetch", _FETCH_INPUT), + ], + "web_search_calls": [ + build_web_search_call("srvtoolu_01Search", _SEARCH_INPUT, {"content": []}), + build_web_search_call( + "srvtoolu_01Fetch", + _FETCH_INPUT, + {"content": []}, + status="in_progress", + tool_name="web_fetch", + ), + ], + }, + ), + ) + ], + ) + + await handler.process_output_response(response, QueryRedactingGuardrail(guardrail_name="redact")) + + assert _hosted_web_call_inputs(response.choices[0].message.provider_specific_fields["web_search_calls"]) == [ + ("ws_srvtoolu_01Search", "completed", "[REDACTED]"), + ("ws_srvtoolu_01Fetch", "in_progress", "[REDACTED]"), + ] + class MockPassThroughGuardrail(CustomGuardrail): """Mock guardrail that passes through without blocking - for testing streaming fallback behavior""" @@ -1604,6 +1656,75 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: chunk(Delta(content=None), finish_reason="stop"), ] + @staticmethod + def _ended_server_fetch_stream_chunks() -> list: + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + def chunk(delta: Delta, finish_reason: Optional[str] = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-123", + created=1234567890, + model="claude-sonnet-4-5", + object="chat.completion.chunk", + choices=[StreamingChoices(index=0, delta=delta, finish_reason=finish_reason)], + ) + + def hosted_fetch(tool_input: dict, result: dict, status: Optional[str]) -> dict: + return { + "srvtoolu_01Fetch": build_web_search_call( + "srvtoolu_01Fetch", tool_input, result, status=status, tool_name="web_fetch" + ) + } + + fetch = {**_server_tool_call("srvtoolu_01Fetch", "web_fetch", _FETCH_INPUT), "index": 0} + return [ + chunk( + Delta( + content=None, + provider_specific_fields={"web_search_calls": hosted_fetch({}, {"content": []}, "in_progress")}, + ) + ), + chunk( + Delta( + content=None, + provider_specific_fields={ + "server_tool_calls": [fetch], + "web_search_calls": hosted_fetch(_FETCH_INPUT, {"content": []}, "in_progress"), + }, + ) + ), + chunk( + Delta( + content=None, + provider_specific_fields={ + "web_search_calls": hosted_fetch(_FETCH_INPUT, {"content": {"type": "web_fetch_result"}}, None) + }, + ) + ), + chunk(Delta(content="Fetched"), finish_reason="stop"), + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_reach_every_hosted_copy_of_a_server_web_call(self): + handler = OpenAIChatCompletionsHandler() + chunks = self._ended_server_fetch_stream_chunks() + + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=QueryRedactingGuardrail(guardrail_name="redact"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert [ + _hosted_web_call_inputs(chunk.choices[0].delta.provider_specific_fields["web_search_calls"].values()) + for chunk in chunks[:3] + ] == [ + [("ws_srvtoolu_01Fetch", "in_progress", "[REDACTED]")], + [("ws_srvtoolu_01Fetch", "in_progress", "[REDACTED]")], + [("ws_srvtoolu_01Fetch", "completed", "[REDACTED]")], + ] + @pytest.mark.asyncio async def test_ended_stream_guardrail_sees_streamed_server_tool_calls(self): handler = OpenAIChatCompletionsHandler() diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_pangea.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_pangea.py index 4e31a3c2055..0c5b8747e46 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_pangea.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_pangea.py @@ -1,5 +1,7 @@ from unittest.mock import AsyncMock, patch +from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch + import httpx import pytest from fastapi import HTTPException @@ -11,6 +13,7 @@ from litellm.proxy.guardrails.guardrail_hooks.pangea.pangea import ( ) from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 from litellm.types.guardrails import GuardrailEventHooks, LitellmParams +from litellm.types.responses.main import build_web_search_call from litellm.types.utils import Choices, Message, ModelResponse @@ -368,3 +371,66 @@ async def test_pangea_ai_guard_response_transformed(pangea_guardrail): ) assert response.choices[0]["message"]["content"] == "Yes, here is an SSN: " + + +def _claude_fetch_call(url): + return { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_fetch", "arguments": '{"url": "%s"}' % url}, + } + + +def _with_masked_tool_calls(choice): + message = choice["message"] + masked_calls = [_claude_fetch_call("") for _ in message["tool_calls"]] + return {**choice, "message": {**message, "tool_calls": masked_calls}} + + +@pytest.mark.asyncio +async def test_pangea_ai_guard_response_masks_claude_server_tool_call_inputs(pangea_guardrail): + data = {"messages": [{"role": "user", "content": "Read https://example.com/"}]} + fetch = build_web_search_call("srvtoolu_01", {"url": "https://example.com/"}, None, tool_name="web_fetch") + sent_choices = [] + + async def _mask_tool_call_urls(_handler, **kwargs): + choices = kwargs["json"]["input"]["choices"] + sent_choices.extend(choices) + return httpx.Response( + status_code=200, + json={ + "result": { + "blocked": False, + "transformed": True, + "output": {"choices": [_with_masked_tool_calls(choice) for choice in choices]}, + } + }, + request=httpx.Request(method="POST", url=kwargs["url"]), + ) + + with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new=_mask_tool_call_urls): + response = await pangea_guardrail.async_post_call_success_hook( + data=data, + user_api_key_dict=None, + response=ModelResponse( + choices=[ + Choices( + message=Message( + role="assistant", + content="Done", + provider_specific_fields={ + "server_tool_calls": [_claude_fetch_call("https://example.com/")], + "web_search_calls": [fetch], + }, + ) + ) + ] + ), + ) + + assert sent_choices[0]["message"]["tool_calls"] == [_claude_fetch_call("https://example.com/")] + message = response.choices[0]["message"] + assert message["tool_calls"] is None + assert message["provider_specific_fields"]["server_tool_calls"] == [_claude_fetch_call("")] + hosted_copy = ResponseFunctionWebSearch.model_validate(message["provider_specific_fields"]["web_search_calls"][0]) + assert hosted_copy.action.url == "" diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 8a3e942e148..1fd2d60d360 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -50,6 +50,7 @@ from litellm.proxy.mcp_server.guardrail_translation.handler import ( ) from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.responses.main import build_web_search_call from litellm.types.utils import ( CallTypes, CallTypesLiteral, @@ -1288,6 +1289,36 @@ class TestStreamingTransform: # end of stream (it can block), matching block_only. assert guardrail.received_tool_calls + @pytest.mark.asyncio + async def test_claude_server_tool_call_chunks_keep_their_provider_fields(self): + server_fields = { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_fetch", "arguments": '{"url": "https://example.com/"}'}, + } + ], + "web_search_calls": { + "srvtoolu_01": build_web_search_call( + "srvtoolu_01", {"url": "https://example.com/"}, None, tool_name="web_fetch" + ).model_dump() + }, + } + server_chunk = ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content=None, provider_specific_fields=server_fields))], + ) + + out = await _drive_stream( + UnifiedLLMGuardrails(), + _StreamingTextGuardrail(), + [server_chunk, _stream_chunk("done", finish_reason="stop")], + ) + + delivered = [getattr(item.choices[0].delta, "provider_specific_fields", None) for item in out if item.choices] + assert server_fields in delivered + assert "".join(_delta_text(item) for item in out) == "DONE" + @pytest.mark.asyncio async def test_per_choice_finish_reason_when_choices_finish_in_different_chunks(self): """n>1: a choice finishing before the stream's last chunk keeps its own diff --git a/tests/unit/proxy/guardrails/test_pillar_guardrails.py b/tests/unit/proxy/guardrails/test_pillar_guardrails.py index c6be433399a..766c7b1b00c 100644 --- a/tests/unit/proxy/guardrails/test_pillar_guardrails.py +++ b/tests/unit/proxy/guardrails/test_pillar_guardrails.py @@ -34,6 +34,7 @@ from litellm.proxy.guardrails.guardrail_hooks.pillar.pillar import ( build_pillar_response_headers, ) from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 +from litellm.types.utils import Choices, Message, ModelResponse # ============================================================================ # FIXTURES @@ -669,6 +670,49 @@ async def test_post_call_hook_with_tool_calls( assert result == mock_llm_response_with_tools +@pytest.mark.asyncio +async def test_post_call_hook_blocks_a_flagged_claude_server_tool_call( + pillar_guardrail_instance, + sample_request_data, + user_api_key_dict, + pillar_clean_response, + pillar_flagged_response, +): + server_call = { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_fetch", "arguments": '{"url": "https://example.com/"}'}, + } + response = ModelResponse( + choices=[ + Choices( + message=Message( + role="assistant", + content="Fetched it", + provider_specific_fields={"server_tool_calls": [server_call]}, + ) + ) + ] + ) + + async def _pillar_flags_the_server_call(*_args, **kwargs): + scanned_tool_calls = kwargs["json"]["messages"][-1].get("tool_calls") or [] + return pillar_flagged_response if server_call in scanned_tool_calls else pillar_clean_response + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + side_effect=_pillar_flags_the_server_call, + ): + with pytest.raises(HTTPException) as excinfo: + await pillar_guardrail_instance.async_post_call_success_hook( + data=sample_request_data, + user_api_key_dict=user_api_key_dict, + response=response, + ) + + assert excinfo.value.status_code == 400 + + # ========================================================================= # HEADER CONFIGURATION TESTS # ========================================================================= From 61da755f273738d99975cbbb99d81f84279319c3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:16:06 -0700 Subject: [PATCH 8/9] fix(anthropic): keep the legacy hook view of streamed /v1/messages web search after the main merge Main's legacy post-call hooks see Claude's server call as tool_use with its search results first. The streamed /v1/messages response now keeps that view while native scans keep server_tool_use. --- .../chat/guardrail_translation/handler.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 8b6d9e2aba7..4188c675dfd 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -16,7 +16,7 @@ import json from collections.abc import Iterable, Mapping, MutableSequence, Sequence from copy import deepcopy from dataclasses import dataclass -from itertools import chain, repeat +from itertools import chain, filterfalse, repeat from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast, overload, runtime_checkable @@ -206,17 +206,15 @@ class _WebSearchString: value: str -def _streamed_web_search_result_blocks(response: ModelResponse) -> tuple[Mapping[str, object], ...]: - choice: Final = response.choices[0] if response.choices else None - fields: Final = as_str_mapping(choice.message.provider_specific_fields) if isinstance(choice, Choices) else None - blocks: Final = ( - (as_str_mapping(item) for item in iter_items(fields.get("web_search_results"))) - if fields is not None and isinstance(fields.get("web_search_results"), list) - else () - ) - return tuple( - deepcopy(block) for block in blocks if block is not None and block.get("type") == "web_search_tool_result" - ) +def _legacy_hook_block(block: object) -> object: + if not isinstance(block, Mapping): + return block + fields: Final = _as_str_mapping(block) + return {**fields, "type": "tool_use"} if fields.get("type") == "server_tool_use" else fields + + +def _is_web_search_result_block(block: object) -> bool: + return isinstance(block, Mapping) and _as_str_mapping(block).get("type") == "web_search_tool_result" def _web_search_results(block: Mapping[str, object]) -> Iterable[object]: @@ -586,11 +584,14 @@ class AnthropicMessagesHandler(BaseTranslation): if not isinstance(response, ModelResponse): return response translated: Final = self.adapter.translate_openai_response_to_anthropic(response) - result_blocks: Final = _streamed_web_search_result_blocks(response) - if not result_blocks: - return translated - content: Final = translated.get("content") or () - return {**translated, "content": [*result_blocks, *content]} + blocks: Final = tuple(map(_legacy_hook_block, translated.get("content") or ())) + return { + **translated, + "content": [ + *filter(_is_web_search_result_block, blocks), + *filterfalse(_is_web_search_result_block, blocks), + ], + } @staticmethod def _build_streaming_usage_response( From b062c49e41b7e8a9a0a0ec27df13e603b11755c5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 21:35:41 -0700 Subject: [PATCH 9/9] fix(guardrails): let Cisco AI Defense scan Claude's web calls on Responses Claude's web search and fetch now come back from /v1/responses as web_search_call items, so Cisco AI Defense reads and clears their query or URL the way it did for the old function_call arguments. The helpers that read and rewrite those items move to the shared guardrail utils so the Responses guardrail handler and Cisco use one copy. --- .../base_llm/guardrail_translation/utils.py | 26 ++++++++ .../guardrail_translation/handler.py | 28 ++------- .../cisco_ai_defense/cisco_ai_defense.py | 23 ++++++- .../test_openai_guardrail_handler.py | 6 +- .../test_cisco_ai_defense_chat.py | 62 +++++++++++++++++++ 5 files changed, 115 insertions(+), 30 deletions(-) diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index fb886e42780..bac95918599 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -4,11 +4,13 @@ import json from collections.abc import Callable, Iterator, Mapping, Sequence from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row has no typed constructor across roles +from openai.types.responses import ResponseFunctionWebSearch from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage +from litellm.types.responses.main import with_hosted_web_tool_arguments _DECODED_JSON: Final = TypeAdapter(object) @@ -135,6 +137,30 @@ def stream_item_field(item: object, field: str) -> object | None: return getattr(item, field, None) +def web_search_call(item: object) -> ResponseFunctionWebSearch | None: + if isinstance(item, ResponseFunctionWebSearch): + return item + if stream_item_field(item, "type") != "web_search_call": + return None + try: + return ResponseFunctionWebSearch.model_validate(item.model_dump() if isinstance(item, BaseModel) else item) + except ValidationError: + return None + + +def with_web_search_call_arguments(item: object, arguments: str) -> object: + search: Final = web_search_call(item) + if search is None: + return item + action: Final = with_hosted_web_tool_arguments(search.action, arguments) + if isinstance(item, ResponseFunctionWebSearch): + return item.model_copy(update={"action": action}) + if isinstance(item, BaseModel): + return item.model_copy(update={"action": action.model_dump()}) + fields: Final = cast("Mapping[str, object]", item) # cast-ok: a validated web_search_call dict is str-keyed JSON + return {**fields, "action": action.model_dump()} + + def stream_item_fingerprint(item: object) -> str: plain: Final = item.model_dump() if isinstance(item, BaseModel) else item return json.dumps(plain, sort_keys=True, default=str) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 9ec8923b4cc..3d30b909da8 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -38,7 +38,6 @@ from itertools import accumulate, chain, groupby, repeat from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast -from openai.types.responses import ResponseFunctionWebSearch from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -67,6 +66,8 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( stream_item_fingerprint, stream_item_items, unappliable_request_rewrite, + web_search_call, + with_web_search_call_arguments, ) from litellm.llms.openai.responses.guardrail_translation.tool_merge import ( append_returned_tools, @@ -111,7 +112,6 @@ from litellm.types.llms.openai import ( from litellm.types.responses.main import ( GenericResponseOutputItem, OutputText, - with_hosted_web_tool_arguments, ) from litellm.types.utils import GenericGuardrailAPIInputs @@ -334,19 +334,8 @@ def _is_tool_call_item(item: object) -> bool: return isinstance(item, Mapping) and item.get("type") in _TOOL_CALL_ITEM_TYPES -def _web_search_call(item: object) -> ResponseFunctionWebSearch | None: - if isinstance(item, ResponseFunctionWebSearch): - return item - if stream_item_field(item, "type") != _WEB_SEARCH_CALL_TYPE: - return None - try: - return ResponseFunctionWebSearch.model_validate(item.model_dump() if isinstance(item, BaseModel) else item) - except ValidationError: - return None - - def _claude_web_search_tool_call(item: object) -> ChatCompletionAssistantToolCall | None: - search: Final = _web_search_call(item) + search: Final = web_search_call(item) return None if search is None else claude_web_search_call_tool_call(search) @@ -405,16 +394,7 @@ def _scanned_tool_call(item: Mapping[str, object], index: int) -> ChatCompletion def _rewritten_web_search_call(item: object, payload: str | None) -> object: - search: Final = _web_search_call(item) - if payload is None or search is None: - return item - action: Final = with_hosted_web_tool_arguments(search.action, payload) - if isinstance(item, ResponseFunctionWebSearch): - return item.model_copy(update={"action": action}) - if isinstance(item, BaseModel): - return item.model_copy(update={"action": action.model_dump()}) - fields: Final = _tool_call_output_item_mapping(item) - return item if fields is None else {**fields, "action": action.model_dump()} + return item if payload is None else with_web_search_call_arguments(item, payload) def _tool_search_rewrite(payload: str) -> Mapping[str, object]: diff --git a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py index 972160bf3ae..8724093a7a3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py @@ -43,6 +43,10 @@ from litellm.llms.anthropic.common_utils import ( with_server_tool_call_arguments, write_anthropic_server_tool_calls, ) +from litellm.llms.base_llm.guardrail_translation.utils import ( + web_search_call, + with_web_search_call_arguments, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -52,6 +56,7 @@ from litellm.proxy.common_utils.callback_utils import ( add_guardrail_to_applied_guardrails_header, ) from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.llms.anthropic import claude_hosted_web_tool_call from litellm.types.utils import ( Choices, Delta, @@ -129,6 +134,12 @@ class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" +def _claude_hosted_web_tool_arguments(item: object) -> str | None: + search: Final = web_search_call(item) + hosted_call: Final = None if search is None else claude_hosted_web_tool_call(search) + return None if hosted_call is None else hosted_call[2] + + def _streamed_choice_index(choice: object) -> int: index: Final[object] = getattr(choice, "index", None) return index if isinstance(index, int) else 0 @@ -1691,7 +1702,7 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): def _redact_responses_api_output( self, - output_items: list, + output_items: list[object], sanitized_text: str | None, sanitized_messages: list[dict[str, object]] | None, ) -> bool: @@ -1703,7 +1714,10 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): if not replacement_text: return False applied = False - for item in output_items: + for index, item in enumerate(output_items): + if _claude_hosted_web_tool_arguments(item): + output_items[index] = with_web_search_call_arguments(item, "{}") # rebind-ok: in-place redaction + applied = True content = getattr(item, "content", None) or (item.get("content") if isinstance(item, dict) else None) if isinstance(content, list): for part in content: @@ -2064,8 +2078,9 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): output_items: Final = getattr(response, "output", None) if not isinstance(output_items, list): return [] + typed_output_items: Final[Sequence[object]] = output_items output_parts: Final[list[str]] = [] - for item in output_items: + for item in typed_output_items: get = item.get if isinstance(item, dict) else (lambda k: getattr(item, k, None)) for part in get("content") or []: pget = part.get if isinstance(part, dict) else (lambda k: getattr(part, k, None)) @@ -2079,6 +2094,8 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): direct = get("text") if isinstance(direct, str) and direct: output_parts.append(direct) + if hosted_arguments := _claude_hosted_web_tool_arguments(item): + output_parts.append(hosted_arguments) joined = " ".join(output_parts) return [{"role": "assistant", "content": joined}] if joined else [] diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index bb205c5e251..365bb0ebfef 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -7,7 +7,7 @@ with guardrail transformations, including tool calls. import json from collections.abc import Mapping -from typing import Any, Literal, Optional +from typing import Any, Final, Literal, Optional import pytest @@ -163,8 +163,8 @@ def _hosted_web_call_inputs(web_search_calls) -> list[tuple[str, str, Optional[s ] -_SEARCH_INPUT = {"query": "jane@example.com"} -_FETCH_INPUT = {"url": "https://example.com/?email=jane@example.com"} +_SEARCH_INPUT: Final = {"query": "jane@example.com"} +_FETCH_INPUT: Final = {"url": "https://example.com/?email=jane@example.com"} class QueryRedactingGuardrail(CustomGuardrail): diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py index 3506afbaa5c..740c3c90758 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py @@ -404,6 +404,33 @@ class TestCiscoAIDefenseChatMode: ) +_CLAUDE_HOSTED_WEB_ACTIONS = ( + {"type": "search", "query": "ssn 123-45-6789"}, + {"type": "open_page", "url": "https://example.com/?ssn=123-45-6789"}, +) + + +def _claude_hosted_web_call_response(action: dict): + from openai.types.responses import ResponseFunctionWebSearch + + from litellm.types.llms.openai import ResponsesAPIResponse + + return ResponsesAPIResponse( + id="resp_1", + created_at=0, + output=[ + ResponseFunctionWebSearch( + id="ws_srvtoolu_01", type="web_search_call", status="completed", action=action + ) + ], + parallel_tool_calls=False, + tool_choice=None, + tools=None, + top_p=None, + usage=None, + ) + + class TestCiscoAIDefenseResponsesAPIOutput: @staticmethod @@ -505,6 +532,23 @@ class TestCiscoAIDefenseResponsesAPIOutput: joined = " ".join(m.get("content", "") for m in (sent.get("messages") or [])) assert "4111-1111-1111-1111" in joined + @pytest.mark.parametrize("action", _CLAUDE_HOSTED_WEB_ACTIONS) + @pytest.mark.asyncio + async def test_post_call_scans_claude_hosted_web_call(self, action): + g = _make_guardrail(event_hook="post_call") + post_mock = AsyncMock(return_value=_safe_response()) + + with _patch_inspection_post(g, post_mock): + await g.async_post_call_success_hook( + data={"input": [{"role": "user", "content": "look it up"}]}, + user_api_key_dict=UserAPIKeyAuth(), + response=_claude_hosted_web_call_response(action), + ) + + sent = post_mock.call_args.kwargs["json"] + joined = " ".join(m.get("content", "") for m in (sent.get("messages") or [])) + assert "123-45-6789" in joined + @pytest.mark.asyncio async def test_post_call_responses_api_violation_is_blocked(self): g = _make_guardrail(event_hook="post_call") @@ -573,6 +617,24 @@ class TestCiscoAIDefenseResponsesAPIOutputRedaction: ) + @pytest.mark.parametrize("action", _CLAUDE_HOSTED_WEB_ACTIONS) + @pytest.mark.asyncio + async def test_redact_clears_claude_hosted_web_call(self, action): + g = _make_guardrail(event_hook="post_call", on_flagged_action="monitor") + cisco_resp = _redact_response(sanitized_text="[REDACTED]") + + with _patch_inspection_post(g, AsyncMock(return_value=cisco_resp)): + result = await g.async_post_call_success_hook( + data={"input": [{"role": "user", "content": "look it up"}]}, + user_api_key_dict=UserAPIKeyAuth(), + response=_claude_hosted_web_call_response(action), + ) + + call = result.output[0] + assert (call.id, call.action.type) == ("ws_srvtoolu_01", action["type"]) + assert "123-45-6789" not in call.action.model_dump_json() + + class TestCiscoAIDefenseResponsesAPIInputRedaction: @pytest.mark.parametrize(