diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index a72023eaedf..79788d35854 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -44,6 +44,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.redact_messages import should_redact_message_logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -315,7 +316,10 @@ def _to_dd_tool_calls(message: Mapping[str, object]) -> tuple[ToolCall, ...]: tool_id=_text_field(tool_call, "id"), type=_text_field(tool_call, "type", "function"), ) - for tool_call in (raw_tool_calls if isinstance(raw_tool_calls, list) else ()) + for tool_call in ( + *anthropic_server_tool_call_params(message), + *(raw_tool_calls if isinstance(raw_tool_calls, list) else ()), + ) if isinstance(tool_call, dict) for function in [_mapping_field(tool_call, "function")] ) diff --git a/litellm/integrations/otel/mappers/utils.py b/litellm/integrations/otel/mappers/utils.py index 80ef2dd553a..bfd50fea5d5 100644 --- a/litellm/integrations/otel/mappers/utils.py +++ b/litellm/integrations/otel/mappers/utils.py @@ -12,6 +12,8 @@ from typing import Final from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue from litellm.integrations.otel.model.payloads import LLMCallSpanData, ToolDefinition +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params @dataclass(frozen=True, slots=True) @@ -141,12 +143,13 @@ def output_messages(data: LLMCallSpanData) -> list: def message_tool_calls(message: object) -> tuple[MessageToolCall, ...]: + tool_call_values: Final = ( + *anthropic_server_tool_call_params(message), + *parts_of(message_field(message, "tool_calls")), + ) if not isinstance(message, dict): return () - tool_calls: Final = message.get("tool_calls") - if not isinstance(tool_calls, (list, tuple)) or not tool_calls: - return () - return tuple(tool_call for value in tool_calls if (tool_call := _message_tool_call(value)) is not None) + return tuple(tool_call for value in tool_call_values if (tool_call := _message_tool_call(value)) is not None) def _message_tool_call(value: object) -> MessageToolCall | None: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index fad4caa59a0..80ab0ca5a9f 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2004,12 +2004,39 @@ def _tool_call_ids_with_a_kept_server_result(message: object) -> frozenset[str]: return frozenset() return frozenset( tool_call_id - for tool_call in parts_of(message_field(message, "tool_calls")) + for tool_call in ( + *parts_of(message_field(message, "tool_calls")), + *parts_of(message_field(provider_specific_fields, "server_tool_calls")), + ) if isinstance(tool_call_id := message_field(tool_call, "id"), str) and tool_call_is_rebuilt_as_server_tool_use(tool_call_id, provider_specific_fields) ) +def _assistant_tool_calls_with_server_tool_calls( + message: ChatCompletionAssistantMessage, +) -> Sequence[ChatCompletionAssistantToolCall] | None: + from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params + + tool_calls: Final = message.get("tool_calls") + server_tool_calls: Final = anthropic_server_tool_call_params(message) + if not server_tool_calls: + return tool_calls + client_tool_call_ids: Final = frozenset(tool_call["id"] for tool_call in tool_calls or ()) + return ( + *(tool_call for tool_call in server_tool_calls if tool_call["id"] not in client_tool_call_ids), + *(tool_calls or ()), + ) + + +def _server_tool_call_ids(message: object) -> frozenset[str]: + from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params + + return frozenset( + tool_call["id"] for tool_call in anthropic_server_tool_call_params(message) if tool_call["id"] is not None + ) + + def _server_tool_answers_replaced_after(messages: Sequence[object], index: int) -> frozenset[str]: kept_server_result_ids: Final = _tool_call_ids_with_a_kept_server_result(messages[index]) if not kept_server_result_ids: @@ -2069,10 +2096,11 @@ _AnthropicToolInvokeItem: TypeAlias = AnthropicMessagesToolUseParam | dict[str, def convert_to_anthropic_tool_invoke( - tool_calls: list[ChatCompletionAssistantToolCall], + tool_calls: Sequence[ChatCompletionAssistantToolCall], web_search_results: Sequence[object] | None = None, tool_results: Sequence[object] | None = None, answered_tool_ids: Container[str] = frozenset(), + server_tool_call_ids: Container[str] = frozenset(), ) -> list[_AnthropicToolInvokeItem]: """ OpenAI tool invokes: @@ -2150,6 +2178,10 @@ def convert_to_anthropic_tool_invoke( anthropic_tool_invoke.append(server_tool_result) elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids: continue + elif tool_id in server_tool_call_ids and tool_id not in answered_tool_ids: + anthropic_tool_invoke.append( + {"type": "server_tool_use", "id": tool_id, "name": tool_name, "input": tool_input} + ) else: sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id) _anthropic_tool_use_param = AnthropicMessagesToolUseParam( @@ -3109,7 +3141,7 @@ def anthropic_messages_pt( # If so, we need to interleave thinking blocks with tool call groups # to preserve the original content block ordering. # Fixes: https://github.com/BerriAI/litellm/issues/23047 - assistant_tool_calls = assistant_content_block.get("tool_calls") + assistant_tool_calls = _assistant_tool_calls_with_server_tool_calls(assistant_content_block) _has_server_tool_calls = False if assistant_tool_calls is not None: for _tc in assistant_tool_calls: @@ -3142,6 +3174,7 @@ def anthropic_messages_pt( web_search_results=_web_search_results_tc, tool_results=_tool_results_tc, answered_tool_ids=client_answered_tool_ids, + server_tool_call_ids=_server_tool_call_ids(assistant_content_block), ) # Group tool invoke results into (server_tool_use, result) pairs @@ -3307,6 +3340,7 @@ def anthropic_messages_pt( web_search_results=_web_search_results, tool_results=_tool_results, answered_tool_ids=client_answered_tool_ids, + server_tool_call_ids=_server_tool_call_ids(assistant_content_block), ) # Prevent "tool_use ids must be unique" errors by filtering duplicates diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 31b0f06251d..a88fd052e27 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -26,6 +26,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls from litellm.llms.anthropic.messages.adapters.transformation import ( OPENAI_MAX_TOOL_NAME_LENGTH, LiteLLMAnthropicMessagesAdapter, @@ -1317,9 +1318,9 @@ class AnthropicMessagesHandler(BaseTranslation): if built_response is not None and hasattr(built_response, "choices") and built_response.choices: model_response: Final = cast(ModelResponse, built_response) first_choice: Final = cast(Choices, model_response.choices[0]) - tool_calls_list: Final = cast( - list[ChatCompletionMessageToolCall] | None, - first_choice.message.tool_calls, + tool_calls_list: Final = ( + *anthropic_server_tool_calls(first_choice.message), + *cast(list[ChatCompletionMessageToolCall], first_choice.message.tool_calls or []), ) string_so_far = first_choice.message.content pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ()) @@ -1327,7 +1328,7 @@ class AnthropicMessagesHandler(BaseTranslation): if string_so_far: guardrail_inputs["texts"] = [string_so_far] if tool_calls_list: - guardrail_inputs["tool_calls"] = tool_calls_list + guardrail_inputs["tool_calls"] = list(tool_calls_list) try: prepared_request_data = self._prepare_request_data( diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index e1436473426..e414e333684 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -54,6 +54,7 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallFunctionChunk, ) from litellm.types.responses.main import ( + HOSTED_WEB_TOOL_NAMES, OutputCodeInterpreterCall, build_code_interpreter_log_outputs, build_web_search_call, @@ -754,6 +755,8 @@ class ModelResponseIterator: self._server_tool_inputs: dict[str, object] = {} self.tool_results: list[dict[str, Any]] = [] self._current_server_tool_id: str | None = None + self._current_server_tool_name: str = "" + self._server_tool_calls: tuple[ChatCompletionToolCallChunk, ...] = () self._container_id: str | None = None @property @@ -824,10 +827,9 @@ class ModelResponseIterator: return text, tool_use, thinking_blocks, provider_specific_fields, reasoning_content self.content_blocks.append(content_block) if "partial_json" in content_block["delta"]: - # Only emit tool calls if we're in a tool_use or server_tool_use block # web_search_tool_result blocks also have input_json_delta but should not be treated as tool calls # See: https://github.com/BerriAI/litellm/issues/17254 - if self.current_content_block_type in ("tool_use", "server_tool_use"): + if self.current_content_block_type == "tool_use": tool_use = cast( ChatCompletionToolCallChunk, { @@ -926,7 +928,7 @@ class ModelResponseIterator: def _web_search_call_snapshot(self) -> dict[str, object]: return dict(self._web_search_calls) - def _complete_web_search_call(self, result: dict[str, object]) -> None: + def _complete_web_search_call(self, result: dict[str, object], tool_name: str) -> None: tool_use_id: Final = result.get("tool_use_id") if not isinstance(tool_use_id, str) or tool_use_id not in self._web_search_calls: return @@ -934,6 +936,7 @@ class ModelResponseIterator: tool_id=tool_use_id, tool_input=self._server_tool_inputs.get(tool_use_id, {}), result=result, + tool_name=tool_name, ) def _build_code_interpreter_results(self) -> list: @@ -1013,7 +1016,6 @@ class ModelResponseIterator: content_block_start["content_block"]["type"] == "tool_use" or content_block_start["content_block"]["type"] == "server_tool_use" ): - self.tool_index += 1 # Reverse-map the (sanitized) tool name back to the # caller's original. No-op when the map is empty. _stream_tool_name = content_block_start["content_block"]["name"] @@ -1022,35 +1024,39 @@ class ModelResponseIterator: # Use empty string for arguments in content_block_start - actual arguments # come in subsequent content_block_delta chunks and get accumulated. # Using str(input) here would prepend '{}' causing invalid JSON accumulation. - tool_use = ChatCompletionToolCallChunk( - id=content_block_start["content_block"]["id"], - type="function", - function=ChatCompletionToolCallFunctionChunk( - name=_stream_tool_name, - arguments="", - ), - index=self.tool_index, - ) + if content_block_start["content_block"]["type"] == "tool_use": + self.tool_index += 1 + tool_use = ChatCompletionToolCallChunk( + id=content_block_start["content_block"]["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=_stream_tool_name, + arguments="", + ), + index=self.tool_index, + ) # Track server tool use inputs for code_interpreter_results. # The initial input in content_block_start is typically {} # for streaming; the full input arrives via input_json_delta # and is assembled at content_block_stop. if content_block_start["content_block"]["type"] == "server_tool_use": self._current_server_tool_id = content_block_start["content_block"]["id"] + self._current_server_tool_name = _stream_tool_name tool_input: Final = content_block_start["content_block"].get("input", {}) self._server_tool_inputs[self._current_server_tool_id] = tool_input - if _stream_tool_name == "web_search": + if _stream_tool_name in HOSTED_WEB_TOOL_NAMES: self._web_search_calls[self._current_server_tool_id] = build_web_search_call( self._current_server_tool_id, tool_input, {"content": []}, status="in_progress", + tool_name=_stream_tool_name, ) provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot() # Include caller information if present (for programmatic tool calling) if "caller" in content_block_start["content_block"]: caller_data: Final = content_block_start["content_block"]["caller"] - if caller_data: + if caller_data and tool_use is not None: tool_use["caller"] = cast(dict[str, Any], caller_data) elif content_block_start["content_block"]["type"] == "redacted_thinking": ( @@ -1081,7 +1087,7 @@ class ModelResponseIterator: # The full content comes in content_block_start, not in deltas # See: https://github.com/BerriAI/litellm/issues/17737 self.web_search_results.append(content_block_start["content_block"]) - self._complete_web_search_call(content_block_start["content_block"]) + self._complete_web_search_call(content_block_start["content_block"], "web_search") provider_specific_fields["web_search_results"] = self.web_search_results provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot() elif content_type == "web_fetch_tool_result": @@ -1089,7 +1095,9 @@ class ModelResponseIterator: # The full content comes in content_block_start, not in deltas # Fixes: https://github.com/BerriAI/litellm/issues/18137 self.web_search_results.append(content_block_start["content_block"]) + self._complete_web_search_call(content_block_start["content_block"], "web_fetch") provider_specific_fields["web_search_results"] = self.web_search_results + provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot() else: self.tool_results.append(content_block_start["content_block"]) provider_specific_fields["tool_results"] = self.tool_results @@ -1104,7 +1112,7 @@ class ModelResponseIterator: # check if tool call content block - only for tool_use and server_tool_use blocks if self.current_content_block_type in ("tool_use", "server_tool_use"): is_empty: Final = self.check_empty_tool_call_args() - if is_empty: + if is_empty and self.current_content_block_type == "tool_use": tool_use = ChatCompletionToolCallChunk( id=None, type="function", @@ -1128,6 +1136,29 @@ class ModelResponseIterator: self._server_tool_inputs[self._current_server_tool_id] = json.loads(args) except (json.JSONDecodeError, TypeError): pass + server_tool_input: Final = self._server_tool_inputs.get(self._current_server_tool_id, {}) + if self._current_server_tool_id in self._web_search_calls: + self._web_search_calls[self._current_server_tool_id] = build_web_search_call( + self._current_server_tool_id, + server_tool_input, + {"content": []}, + status="in_progress", + tool_name=self._current_server_tool_name, + ) + provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot() + self._server_tool_calls = ( + *self._server_tool_calls, + ChatCompletionToolCallChunk( + id=self._current_server_tool_id, + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=self._current_server_tool_name, + arguments=json.dumps(server_tool_input), + ), + index=len(self._server_tool_calls), + ), + ) + provider_specific_fields["server_tool_calls"] = list(self._server_tool_calls) self._current_server_tool_id = None if self.is_response_format_tool: text = self._flush_response_format_buffer() diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 865247afc63..77ac29fe756 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -7,6 +7,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast import httpx +from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -62,6 +63,7 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_HOSTED_TOOLS, + ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AllAnthropicPassThroughMessageValues, AllAnthropicToolsValues, @@ -94,6 +96,7 @@ from litellm.types.llms.openai import ( OpenAIWebSearchOptions, ) from litellm.types.responses.main import ( + HOSTED_WEB_TOOL_NAMES, OutputCodeInterpreterCall, build_code_interpreter_log_outputs, build_web_search_call, @@ -251,6 +254,29 @@ def _validated_defer_loading(value: object) -> bool: raise ValueError("defer_loading must be a boolean") from exc +def _hosted_web_tool_use(block: object) -> tuple[str, str, object] | None: + match block: + case { + "type": "server_tool_use", + "name": str() as name, + "id": str() as call_id, + "input": object() as tool_input, + } if name in HOSTED_WEB_TOOL_NAMES: + return call_id, name, tool_input + case _: + return None + + +def _hosted_web_call(call_id: str, tool_name: str, tool_input: object, result: object) -> ResponseFunctionWebSearch: + return build_web_search_call( + tool_id=call_id, + tool_input=tool_input, + result=result, + status=None if result is not None else "in_progress", + tool_name=tool_name, + ) + + class _AnthropicUsageIteration(TypedDict, total=False): """One entry of the ``usage.iterations`` array on an Anthropic response.""" @@ -2850,6 +2876,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return usage + @staticmethod + def _is_server_tool_call(tool_call: ChatCompletionToolCallChunk) -> bool: + tool_call_id: Final = tool_call["id"] + return tool_call_id is not None and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) + def _build_code_by_id_map(self, tool_calls: list[ChatCompletionToolCallChunk]) -> dict[str, str]: code_by_id: Final[dict[str, str]] = {} for tc in tool_calls: @@ -2894,29 +2925,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self, web_search_results: Sequence[object], completion_response: Mapping[str, object], - ) -> list[object]: + ) -> list[ResponseFunctionWebSearch]: content: Final = completion_response.get("content") blocks: Final = content if isinstance(content, Sequence) else () - inputs: Final = { - call_id: tool_input - for block in blocks - if isinstance(block, Mapping) - and block.get("type") == "server_tool_use" - and block.get("name") == "web_search" - and isinstance((call_id := block.get("id")), str) - and isinstance((tool_input := block.get("input")), Mapping) + results_by_call: Final = { + (tool_use_id, result.get("type")): result + for result in web_search_results + if isinstance(result, dict) and isinstance((tool_use_id := result.get("tool_use_id")), str) } return [ - build_web_search_call( - tool_id=tool_use_id, - tool_input=inputs.get(tool_use_id, {}), - result=result, - ) - for result in web_search_results - if isinstance(result, dict) - and result.get("type") == "web_search_tool_result" - and isinstance((tool_use_id := result.get("tool_use_id")), str) - and tool_use_id in inputs + _hosted_web_call(call_id, tool_name, tool_input, results_by_call.get((call_id, f"{tool_name}_tool_result"))) + for call_id, tool_name, tool_input in filter(None, map(_hosted_web_tool_use, blocks)) ] def _build_provider_specific_fields( @@ -2940,10 +2959,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if web_search_results is not None: provider_specific_fields["web_search_results"] = web_search_results - provider_specific_fields["web_search_calls"] = self._build_web_search_calls( - web_search_results, - completion_response, - ) + + web_search_calls: Final = self._build_web_search_calls(web_search_results or (), completion_response) + if web_search_calls: + provider_specific_fields["web_search_calls"] = web_search_calls if tool_results is not None: provider_specific_fields["tool_results"] = tool_results @@ -2965,6 +2984,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if compaction_blocks is not None: provider_specific_fields["compaction_blocks"] = compaction_blocks + server_tool_calls: Final = [tc for tc in tool_calls if self._is_server_tool_call(tc)] + if server_tool_calls: + provider_specific_fields["server_tool_calls"] = server_tool_calls + refusal_fields: Final = refusal_provider_specific_fields( REFUSAL_STOP_FIELDS.validate_python(completion_response) ) @@ -3024,6 +3047,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ): text_content = prefix_prompt + text_content + client_tool_calls: Final = [tc for tc in tool_calls if not self._is_server_tool_call(tc)] provider_specific_fields: Final = self._build_provider_specific_fields( completion_response, citations, @@ -3036,7 +3060,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): json_mode_message, tool_calls_for_message, json_extra_content = self._resolve_json_mode_non_streaming( json_mode=json_mode, - tool_calls=tool_calls, + tool_calls=client_tool_calls, response_format_schema=response_format_schema, ) merged_text = text_content or "" diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 3b7fcdea0e4..1fccade3528 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -60,10 +60,10 @@ from litellm.types.llms.anthropic import ( AnthropicThinkingParam, ) from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantToolCall from litellm.types.proxy.auth.special_headers import SpecialHeaders from litellm.types.proxy.model_listing import ModelInfoResponse -from litellm.types.utils import LlmProviders +from litellm.types.utils import ChatCompletionMessageToolCall, LlmProviders _MessageT = TypeVar("_MessageT") @@ -116,6 +116,7 @@ _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)") _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) +_SERVER_TOOL_CALLS_ADAPTER: Final = TypeAdapter(tuple[ChatCompletionAssistantToolCall, ...]) _DECODED_JSON: Final = TypeAdapter(object) @@ -2074,6 +2075,21 @@ def _replayed_server_tool_use(block: object) -> _ReplayedServerToolUse | None: return None +def anthropic_server_tool_call_params(message: object) -> tuple[ChatCompletionAssistantToolCall, ...]: + fields: Final = validated_object_mapping(message_field(message, "provider_specific_fields")) + try: + return _SERVER_TOOL_CALLS_ADAPTER.validate_python(fields.get("server_tool_calls", ()) if fields else ()) + except ValidationError: + return () + + +def anthropic_server_tool_calls(message: object) -> tuple[ChatCompletionMessageToolCall, ...]: + return tuple( + ChatCompletionMessageToolCall(id=call["id"], type=call["type"], function=dict(call["function"])) + for call in anthropic_server_tool_call_params(message) + ) + + def tool_call_is_rebuilt_as_server_tool_use( tool_call_id: object, provider_specific_fields: object, answered_tool_ids: Container[str] = frozenset() ) -> bool: diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index ddda759384f..10b86537623 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -45,6 +45,25 @@ def _thought_signature(provider_specific_fields: object) -> str | None: return signature if isinstance(signature, str) else None +def _object_sequence(value: object) -> Sequence[object] | None: + return value if isinstance(value, (list, tuple)) else None + + +def _server_tool_blocks(message: object) -> tuple[Mapping[str, object], ...]: + server_tool_calls: Final = anthropic_server_tool_call_params(message) + if not server_tool_calls: + return () + fields: Final = _as_string_mapping(_optional_attr(message, "provider_specific_fields")) or {} + return tuple( + convert_to_anthropic_tool_invoke( + server_tool_calls, + web_search_results=_object_sequence(fields.get("web_search_results")), + tool_results=_object_sequence(fields.get("tool_results")), + server_tool_call_ids=frozenset(tool_call["id"] for tool_call in server_tool_calls), + ) + ) + + def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]: fields: Final = _as_string_mapping(provider_specific_fields) raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None @@ -130,11 +149,13 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, + convert_to_anthropic_tool_invoke, ) from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, ) from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, eager_input_streaming_flag, is_empty_unsigned_thinking_block, normalize_anthropic_tool_use_id, @@ -1428,6 +1449,7 @@ class LiteLLMAnthropicMessagesAdapter: ).model_dump() ) + new_content.extend(dict(block) for block in _server_tool_blocks(choice.message)) # Handle text content if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks): new_content.append( diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 3070bedd30d..d7c3aeca354 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -21,6 +21,7 @@ from collections.abc import Iterator, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Union, cast +from pydantic import TypeAdapter, ValidationError from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm @@ -29,6 +30,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i replayed_tool_search_tool_names, ) from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, StreamingScanKey, @@ -50,7 +52,12 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( stream_item_items, unappliable_request_rewrite, ) -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionAssistantToolCall, + ChatCompletionToolCallFunctionChunk, + ChatCompletionToolParam, +) from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( coerce_stream_holdback_value, ) @@ -436,9 +443,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation): text_task_mappings=text_task_mappings, tool_call_task_mappings=tool_call_task_mappings, ) + server_tool_call_tasks: Final = tuple(_server_tool_call_tasks(response.choices)) + tool_calls_with_server_calls: Final = [ + *tool_calls_to_check, + *(_server_tool_call_input(call) for _, call in server_tool_call_tasks), + ] # Step 2: Apply guardrail to all texts and tool calls in batch - if texts_to_check or tool_calls_to_check: + if texts_to_check or tool_calls_with_server_calls: # Use the real request_data if provided (proxy path), otherwise # create a standalone dict (SDK / direct-call path). if request_data is None: @@ -452,8 +464,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) if images_to_check: inputs["images"] = images_to_check - if tool_calls_to_check: - inputs["tool_calls"] = tool_calls_to_check + if tool_calls_with_server_calls: + inputs["tool_calls"] = tool_calls_with_server_calls # Include model information from the response if available if hasattr(response, "model") and response.model: inputs["model"] = response.model @@ -469,8 +481,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation): returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") guardrailed_tool_calls: Final[list[dict[str, object]]] = ( cast(list[dict[str, object]], returned_tool_calls) - if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check) - else tool_calls_to_check + if isinstance(returned_tool_calls, list) + and len(returned_tool_calls) == len(tool_calls_with_server_calls) + else tool_calls_with_server_calls ) # Step 3: Map guardrail responses back to original response structure @@ -488,6 +501,11 @@ class OpenAIChatCompletionsHandler(BaseTranslation): tool_calls=guardrailed_tool_calls, task_mappings=tool_call_task_mappings, ) + self._apply_guardrail_responses_to_output_server_tool_calls( + response=response, + guardrailed_tool_calls=guardrailed_tool_calls[len(tool_calls_to_check) :], + task_mappings=tuple(mapping for mapping, _ in server_tool_call_tasks), + ) verbose_proxy_logger.debug("OpenAI Chat Completions: Processed output response: %s", response) @@ -681,6 +699,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): pre_guardrail_tool_calls=pre_guardrail_tool_calls, guardrail_name=guardrail_to_apply.guardrail_name or "unknown", ) + self._write_ended_stream_server_tool_call_rewrites( + responses_so_far=responses_so_far, + guardrailed_response=model_response, + ) @staticmethod def _rebuild_ended_stream_per_choice( @@ -939,6 +961,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if choice.message.tool_calls and isinstance(choice.message.tool_calls, list): if len(choice.message.tool_calls) > 0: return True + if anthropic_server_tool_call_params(choice.message): + return True elif isinstance(response, ModelResponseStream): for streaming_choice in response.choices: if isinstance(streaming_choice, litellm.StreamingChoices): @@ -949,6 +973,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if streaming_choice.delta.tool_calls and isinstance(streaming_choice.delta.tool_calls, list): if len(streaming_choice.delta.tool_calls) > 0: return True + if anthropic_server_tool_call_params(streaming_choice.delta): + return True return False def _extract_output_text_images_and_tool_calls( @@ -1099,6 +1125,25 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if "name" in func_dict: existing_tool_call.function.name = func_dict["name"] + @staticmethod + def _apply_guardrail_responses_to_output_server_tool_calls( + response: "ModelResponse", + guardrailed_tool_calls: Sequence[object], + task_mappings: Sequence[tuple[int, int]], + ) -> None: + guardrailed_by_task: Final = MappingProxyType(dict(zip(task_mappings, guardrailed_tool_calls))) + for choice_idx, choice in enumerate(response.choices): + server_tool_calls = anthropic_server_tool_call_params(choice.message) + rewritten = tuple( + _rewritten_server_tool_call(server_tool_call, guardrailed_by_task.get((choice_idx, server_idx))) + for server_idx, server_tool_call in enumerate(server_tool_calls) + ) + if rewritten != server_tool_calls: + choice.message.provider_specific_fields = { + **(choice.message.provider_specific_fields or {}), + "server_tool_calls": list(rewritten), + } + @staticmethod def _string_choice_contents(response: "ModelResponse") -> tuple[str | None, ...]: return tuple( @@ -1211,6 +1256,32 @@ class OpenAIChatCompletionsHandler(BaseTranslation): for fragment in tail: fragment.function.arguments = "" + @staticmethod + def _write_ended_stream_server_tool_call_rewrites( + responses_so_far: Sequence["ModelResponseStream"], + guardrailed_response: "ModelResponse", + ) -> None: + guardrailed_by_id: Final = MappingProxyType( + { + (guardrailed_response.choices[choice_position].index, server_tool_call["id"]): server_tool_call + for (choice_position, _), server_tool_call in _server_tool_call_tasks(guardrailed_response.choices) + } + ) + if not guardrailed_by_id: + return + for response in responses_so_far: + for choice in response.choices: + streamed = anthropic_server_tool_call_params(choice.delta) + rewritten = tuple( + guardrailed_by_id.get((choice.index, server_tool_call["id"]), server_tool_call) + for server_tool_call in streamed + ) + if rewritten != streamed: + choice.delta.provider_specific_fields = { + **(choice.delta.provider_specific_fields or {}), + "server_tool_calls": list(rewritten), + } + async def _apply_guardrail_responses_to_output_streaming( self, responses: list["ModelResponseStream"], @@ -1400,7 +1471,42 @@ def _stream_chunk_choices(item: object) -> Sequence[object]: def _streamed_delta_tool_calls(delta: object) -> tuple[object, ...]: function_call: Final = stream_item_field(delta, "function_call") legacy: Final = () if function_call is None else (function_call,) - return stream_item_items(delta, "tool_calls") + legacy + return anthropic_server_tool_call_params(delta) + stream_item_items(delta, "tool_calls") + legacy + + +def _server_tool_call_tasks( + choices: Sequence[object], +) -> Iterator[tuple[tuple[int, int], ChatCompletionAssistantToolCall]]: + for choice_idx, choice in enumerate(choices): + if not isinstance(choice, Choices): + continue + for server_idx, server_tool_call in enumerate(anthropic_server_tool_call_params(choice.message)): + yield (choice_idx, server_idx), server_tool_call + + +def _server_tool_call_input(server_tool_call: ChatCompletionAssistantToolCall) -> dict[str, object]: + return { + "id": server_tool_call["id"], + "type": server_tool_call["type"], + "function": dict(server_tool_call["function"]), + } + + +_TOOL_CALL_FUNCTION_ADAPTER: Final = TypeAdapter(ChatCompletionToolCallFunctionChunk) + + +def _rewritten_server_tool_call( + server_tool_call: ChatCompletionAssistantToolCall, guardrailed: object +) -> ChatCompletionAssistantToolCall: + try: + function: Final = _TOOL_CALL_FUNCTION_ADAPTER.validate_python(stream_item_field(guardrailed, "function")) + except ValidationError: + return server_tool_call + return ChatCompletionAssistantToolCall( + id=server_tool_call["id"], + type=server_tool_call["type"], + function={**server_tool_call["function"], **function}, + ) def _released_choices(responses_so_far: Sequence[object]) -> Iterator[object]: diff --git a/litellm/proxy/db/spend_log_tool_index.py b/litellm/proxy/db/spend_log_tool_index.py index 591ad81a4e5..92181d546eb 100644 --- a/litellm/proxy/db/spend_log_tool_index.py +++ b/litellm/proxy/db/spend_log_tool_index.py @@ -63,11 +63,26 @@ def response_tool_call_names(completion_response: object) -> tuple[str, ...]: ) return tuple( stripped - for name in chain(called_names, _auto_executed_mcp_tool_names(completion_response)) + for name in chain( + _server_tool_call_names(completion_response), + called_names, + _auto_executed_mcp_tool_names(completion_response), + ) if isinstance(name, str) and (stripped := name.strip()) ) +def _server_tool_call_names(completion_response: object) -> tuple[str | None, ...]: + from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of + from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params + + server_tool_calls: Final = chain.from_iterable( + anthropic_server_tool_call_params(message_field(choice, "message")) + for choice in parts_of(message_field(completion_response, "choices")) + ) + return tuple(server_tool_call["function"].get("name") for server_tool_call in server_tool_calls) + + class _ExecutedTool(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index fa92a0c90e0..541e49c6751 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -19,6 +19,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params, anthropic_server_tool_calls from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import ( add_guardrail_to_applied_guardrails_header, @@ -515,7 +516,7 @@ class ToolPermissionGuardrail(CustomGuardrail): for choice_index, choice in enumerate(response.choices): if isinstance(choice, Choices): - for tool in choice.message.tool_calls or []: + for tool in (*(choice.message.tool_calls or ()), *anthropic_server_tool_calls(choice.message)): tool_calls.append(tool) legacy_tool_call = self._legacy_function_call_to_tool_call( getattr(choice.message, "function_call", None), choice_index @@ -784,6 +785,23 @@ class ToolPermissionGuardrail(CustomGuardrail): choice.message.tool_calls = filtered_tool_calls if filtered_tool_calls else None + if (server_tool_calls := anthropic_server_tool_call_params(choice.message)) and any( + server_tool_call["id"] in error_results for server_tool_call in server_tool_calls + ): + error_messages.extend( + error_results[server_tool_call["id"]].content + for server_tool_call in server_tool_calls + if server_tool_call["id"] in error_results + ) + choice.message.provider_specific_fields = { + **(choice.message.provider_specific_fields or {}), + "server_tool_calls": [ + server_tool_call + for server_tool_call in server_tool_calls + if server_tool_call["id"] not in error_results + ], + } + legacy_tool_call = self._legacy_function_call_to_tool_call( getattr(choice.message, "function_call", None), choice_index ) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 892da541935..3e053dd87a9 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -9,6 +9,7 @@ import litellm from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend +from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls from litellm.main import stream_chunk_builder from litellm.responses.litellm_completion_transformation.compaction_items import compaction_output_item from litellm.responses.litellm_completion_transformation.custom_tools import ( @@ -486,13 +487,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): try: message: Final = litellm_complete_object.choices[0].message - tool_calls = getattr(message, "tool_calls", None) + server_tool_calls: Final = anthropic_server_tool_calls(message) + tool_calls: Final = [*server_tool_calls, *(getattr(message, "tool_calls", None) or ())] except Exception: - tool_calls = None - - if not tool_calls or not isinstance(tool_calls, list): return + if not tool_calls: + return + + server_tool_call_ids: Final = frozenset(tool_call.id for tool_call in server_tool_calls) + for tc in tool_calls: call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None) if not call_id_raw: @@ -518,6 +522,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if (search_execution := self._tool_search_execution(call_id, fn_name)) is not None: self._queue_tool_search_call_done(call_id, output_index, fn_args, search_execution) continue + if call_id in server_tool_call_ids: + continue # Track if this is a new tool call that wasn't streamed is_new_tool_call = call_id not in self._tool_item_id_by_call_id @@ -592,10 +598,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): } ), ) + finished_events: Final = ( + () + if item.status == "in_progress" + else ((ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),) + ) for event_type, event_class in ( (ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent), (ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent), - (ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent), + *finished_events, ): self._sequence_number += 1 event = event_class(type=event_type, output_index=output_index, item_id=item.id) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index ce8841c9190..ad03186c7a5 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -31,6 +31,7 @@ from openai.types.chat.chat_completion_named_tool_choice_param import ( from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch from openai.types.responses.response_compaction_item import ResponseCompactionItem from openai.types.responses.response_create_params import ResponseInputParam +from openai.types.responses.response_function_web_search import ActionOpenPage, ActionSearch from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam from openai.types.responses.tool_param import FunctionToolParam @@ -56,7 +57,12 @@ from litellm.litellm_core_utils.hidden_params import ( set_hidden_params, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + anthropic_server_tool_calls, + supports_anthropic_cache_control, + validated_object_mapping, +) from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control @@ -66,11 +72,12 @@ from litellm.responses.litellm_completion_transformation.session_handler import ResponsesSessionHandler, StoredTurns, ) -from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex +from litellm.types.llms.anthropic import ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, AnthropicToolSearchToolRegex from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, + ChatCompletionAssistantToolCall, ChatCompletionAudioObject, ChatCompletionCachedContent, ChatCompletionFileObject, @@ -114,6 +121,7 @@ from litellm.types.responses.main import ( OutputImageGenerationCall, OutputText, OutputToolSearchCall, + web_search_call_tool_id, ) from litellm.types.utils import ( ChatCompletionAnnotation, @@ -873,6 +881,29 @@ def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, obj return {key: {**_STR_KEY_DICT_ADAPTER.validate_python(metadata), CALLER_SESSION_ID_METADATA_KEY: chain_session}} +def _pending_claude_server_tool_call(search: ResponseFunctionWebSearch) -> ChatCompletionAssistantToolCall | None: + tool_id: Final = web_search_call_tool_id(search.id) + if search.status != "in_progress" or tool_id is None: + return None + if not tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX): + return None + match search.action: + case ActionSearch(query=str() as query): + return _server_tool_call(tool_id, "web_search", json.dumps({"query": query})) + case ActionOpenPage(url=str() as url): + return _server_tool_call(tool_id, "web_fetch", json.dumps({"url": url})) + case _: + return None + + +def _server_tool_call(tool_id: str, name: str, arguments: str) -> ChatCompletionAssistantToolCall: + return ChatCompletionAssistantToolCall( + id=tool_id, + type="function", + function=ChatCompletionToolCallFunctionChunk(name=name, arguments=arguments), + ) + + class LiteLLMCompletionResponsesConfig: @staticmethod def get_supported_openai_params(model: str) -> list: @@ -1778,10 +1809,22 @@ class LiteLLMCompletionResponsesConfig: message_cache_control: Final = ( None if isinstance(content, list) else new_cache_control or previous_cache_control ) + new_server_tool_calls: Final = anthropic_server_tool_call_params(new_message) + server_tool_call_fields: Final = ( + { + "provider_specific_fields": { + **(validated_object_mapping(last_message.get("provider_specific_fields")) or {}), + "server_tool_calls": [*anthropic_server_tool_call_params(last_message), *new_server_tool_calls], + } + } + if new_server_tool_calls + else {} + ) merged: Final = { **{key: value for key, value in last_message.items() if key != "cache_control"}, "content": content, **({} if message_cache_control is None else {"cache_control": message_cache_control}), + **server_tool_call_fields, } return cast( # cast-ok: preserves the assistant fields and content blocks ChatCompletionAssistantMessage, merged @@ -2223,10 +2266,15 @@ class LiteLLMCompletionResponsesConfig: content="Hosted web search: " + replay.model_dump_json(exclude_none=True), ) ] + hosted_search_text: Final = "Hosted web search: " + search.model_dump_json(exclude_none=True) + pending_server_tool_call: Final = _pending_claude_server_tool_call(search) + if pending_server_tool_call is None: + return [GenericChatCompletionMessage(role="assistant", content=hosted_search_text)] return [ - GenericChatCompletionMessage( + ChatCompletionResponseMessage( role="assistant", - content="Hosted web search: " + search.model_dump_json(exclude_none=True), + content=hosted_search_text, + provider_specific_fields={"server_tool_calls": [pending_server_tool_call]}, ) ] if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(input_item): @@ -2956,13 +3004,12 @@ class LiteLLMCompletionResponsesConfig: all_chat_completion_tools: Final[list[ChatCompletionMessageToolCall]] = [] for choice in chat_completion_response.choices: if isinstance(choice, Choices): - if choice.message.tool_calls: - all_chat_completion_tools.extend(choice.message.tool_calls) - for tool_call in choice.message.tool_calls: - TOOL_CALLS_CACHE.set_cache( - key=tool_call.id, - value=tool_call, - ) + for tool_call in (*anthropic_server_tool_calls(choice.message), *(choice.message.tool_calls or ())): + all_chat_completion_tools.append(tool_call) + TOOL_CALLS_CACHE.set_cache( + key=tool_call.id, + value=tool_call, + ) request_tools: Final = responses_api_request.get("tools") if responses_api_request is not None else None custom_tool_names: Final = extract_custom_tool_names(request_tools) @@ -3379,7 +3426,22 @@ class LiteLLMCompletionResponsesConfig: for item in responses_output ] - return responses_output + server_tool_call_ids: Final = LiteLLMCompletionResponsesConfig._server_tool_call_ids(choices) + return [ + item + for item in responses_output + if not ( + getattr(item, "type", None) == "function_call" + and getattr(item, "call_id", None) in server_tool_call_ids + ) + ] + + @staticmethod + def _server_tool_call_ids(choices: Sequence[Choices]) -> frozenset[str]: + server_tool_calls: Final = chain.from_iterable( + anthropic_server_tool_calls(choice.message) for choice in choices + ) + return frozenset(tool_call.id for tool_call in server_tool_calls) @staticmethod def _extract_tool_result_output_items( diff --git a/litellm/tool_loop.py b/litellm/tool_loop.py index d0b2850bb5b..554fdff4fce 100644 --- a/litellm/tool_loop.py +++ b/litellm/tool_loop.py @@ -120,9 +120,12 @@ def _replays_provider_specific_fields( client_tool_calls: tuple[ChatCompletionMessageToolCall, ...], ) -> bool: from litellm.completion_extras.litellm_responses_transformation.transformation import carries_tool_search_items + from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params - return client_tool_calls != tool_calls or carries_tool_search_items( - getattr(message, "provider_specific_fields", None) + return ( + client_tool_calls != tool_calls + or bool(anthropic_server_tool_call_params(message)) + or carries_tool_search_items(getattr(message, "provider_specific_fields", None)) ) diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 72dc63a581f..e9111eb8c99 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -3,7 +3,12 @@ from collections.abc import Mapping, Sequence from typing import Final, Literal, Optional, Union from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall -from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch +from openai.types.responses.response_function_web_search import ( + ActionOpenPage, + ActionSearch, + ActionSearchSource, + ResponseFunctionWebSearch, +) from pydantic import ConfigDict, PrivateAttr from typing_extensions import TypedDict @@ -42,14 +47,11 @@ class OutputFunctionToolCall(BaseLiteLLMOpenAIResponseObject): phase: Phase = None -def build_web_search_call( - tool_id: str, - tool_input: object, - result: object, - status: Literal["in_progress", "searching", "completed", "failed"] | None = None, -) -> ResponseFunctionWebSearch: +HOSTED_WEB_TOOL_NAMES: Final = frozenset({"web_search", "web_fetch"}) + + +def _web_search_action(tool_input: object, content: object) -> ActionSearch: query: Final = tool_input.get("query", "") if isinstance(tool_input, Mapping) else "" - content: Final = result.get("content") if isinstance(result, Mapping) else None result_items: Final = content if isinstance(content, Sequence) and not isinstance(content, (str, bytes)) else () sources: Final = [ ActionSearchSource(type="url", url=url) @@ -58,20 +60,62 @@ def build_web_search_call( and item.get("type") == "web_search_result" and isinstance((url := item.get("url")), str) ] - failed: Final = isinstance(content, Mapping) and content.get("type") == "web_search_tool_result_error" - return ResponseFunctionWebSearch( - id=f"ws_{tool_id}", - type="web_search_call", - status=status or ("failed" if failed else "completed"), - action={ - "type": "search", - "query": query if isinstance(query, str) else "", - "queries": [query] if isinstance(query, str) and query else [], - "sources": sources, - }, + return ActionSearch( + type="search", + query=query if isinstance(query, str) else "", + queries=[query] if isinstance(query, str) and query else [], + sources=sources, ) +def _web_fetch_action(tool_input: object) -> ActionOpenPage: + match tool_input: + case {"url": str() as url}: + return ActionOpenPage(type="open_page", url=url) + case _: + return ActionOpenPage(type="open_page", url=None) + + +def _result_content(result: object) -> object: + match result: + case {"content": object() as content}: + return content + case _: + return None + + +def _is_tool_result_error(content: object, tool_name: str) -> bool: + match content: + case {"type": str() as content_type}: + return content_type == f"{tool_name}_tool_result_error" + case _: + return False + + +WEB_SEARCH_CALL_ID_PREFIX: Final = "ws_" + + +def build_web_search_call( + tool_id: str, + tool_input: object, + result: object, + status: Literal["in_progress", "searching", "completed", "failed"] | None = None, + tool_name: str = "web_search", +) -> ResponseFunctionWebSearch: + content: Final = _result_content(result) + return ResponseFunctionWebSearch( + id=f"{WEB_SEARCH_CALL_ID_PREFIX}{tool_id}", + type="web_search_call", + status=status or ("failed" if _is_tool_result_error(content, tool_name) else "completed"), + action=_web_fetch_action(tool_input) if tool_name == "web_fetch" else _web_search_action(tool_input, content), + ) + + +def web_search_call_tool_id(call_id: str) -> str | None: + tool_id: Final = call_id.removeprefix(WEB_SEARCH_CALL_ID_PREFIX) + return None if tool_id == call_id else tool_id + + class OutputImageGenerationCall(BaseLiteLLMOpenAIResponseObject): """An image generation call output""" diff --git a/tests/unit/integrations/datadog/test_datadog_llm_obs.py b/tests/unit/integrations/datadog/test_datadog_llm_obs.py index a7826e11483..dcc1567f5f8 100644 --- a/tests/unit/integrations/datadog/test_datadog_llm_obs.py +++ b/tests/unit/integrations/datadog/test_datadog_llm_obs.py @@ -110,6 +110,41 @@ def test_output_tool_calls_use_the_datadog_tool_call_schema(logger: DataDogLLMOb assert "function" not in message["tool_calls"][0] + +def test_claude_server_tool_calls_are_reported_before_client_tool_calls(logger: DataDogLLMObsLogger) -> None: + payload = build( + logger, + response_message={ + "role": "assistant", + "content": None, + "tool_calls": [ASSISTANT_TOOL_CALL], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "paris weather"}'}, + } + ] + }, + }, + ) + + assert payload["meta"]["output"]["messages"][0]["tool_calls"] == [ + { + "name": "web_search", + "arguments": {"query": "paris weather"}, + "tool_id": "srvtoolu_01Search", + "type": "function", + }, + { + "name": "get_weather", + "arguments": {"city": "Paris", "unit": "c"}, + "tool_id": "call_abc123", + "type": "function", + }, + ] + def test_tool_call_identifiers_that_are_not_strings_are_blanked_not_stringified(logger: DataDogLLMObsLogger) -> None: payload = build( logger, diff --git a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py index 57764bf902b..611e828cf84 100644 --- a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py @@ -174,6 +174,45 @@ def test_openinference_output_tool_calls_preserve_calls_in_attributes_and_value( ] +def test_openinference_output_lists_claude_server_tool_calls_before_client_tool_calls(): + data: Final = _llm_call( + choices_out=( + { + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_paris", + "type": "function", + "function": {"name": "lookup_weather", "arguments": '{"city": "Paris"}'}, + } + ], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "paris weather"}'}, + } + ] + }, + }, + }, + ) + ) + attrs: Final = OpenInferenceMapper().map(data) + assert {key: value for key, value in attrs.items() if ".tool_calls." in key} == { + "llm.output_messages.0.message.tool_calls.0.tool_call.id": "srvtoolu_01Search", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.name": "web_search", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": '{"query": "paris weather"}', + "llm.output_messages.0.message.tool_calls.1.tool_call.id": "call_paris", + "llm.output_messages.0.message.tool_calls.1.tool_call.function.name": "lookup_weather", + "llm.output_messages.0.message.tool_calls.1.tool_call.function.arguments": '{"city": "Paris"}', + } + + def test_openinference_responses_tool_calls_are_emitted_as_output_attributes(): data: Final = LLMCallSpanData.from_standard_logging_payload( _responses_payload( diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index 5d3fc523c0c..3225101cdfc 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -15,7 +15,9 @@ import litellm from litellm._uuid import uuid from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of from litellm.llms.anthropic.chat.handler import ModelResponseIterator, _envelope_verdict, make_call, make_sync_call +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params from litellm.llms.base_llm.base_utils import response_format_schema, unwrap_response_format_envelope from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.llms.openai import ( @@ -23,7 +25,7 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallFunctionChunk, ) from litellm.types.responses.main import OutputCodeInterpreterCall -from litellm.types.utils import ChatCompletionDeltaToolCall, ModelResponseStream +from litellm.types.utils import ChatCompletionDeltaToolCall, ChatCompletionMessageToolCall, Choices, ModelResponseStream from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE @@ -1780,19 +1782,113 @@ def test_web_search_tool_result_no_extra_tool_calls(): for tc in parsed.choices[0].delta.tool_calls: tool_calls_emitted.append(tc) - # Should have exactly 2 tool calls: - # 1. From content_block_start (server_tool_use) with id and name - # 2. From content_block_delta with the actual query - assert len(tool_calls_emitted) == 2, f"Expected 2 tool calls, got {len(tool_calls_emitted)}" + assert tool_calls_emitted == [] - # First tool call should have the id and name - assert tool_calls_emitted[0]["id"] == "srvtoolu_01ABC123" - assert tool_calls_emitted[0]["function"]["name"] == "web_search" - # Second tool call should have the query arguments - assert tool_calls_emitted[1]["function"]["arguments"] == '{"query": "test"}' +def _server_tool_input_delta(fragment: str) -> dict[str, object]: + return {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": fragment}} - # The {} chunks from web_search_tool_result should NOT have been emitted as tool calls + +def _server_search_then_answer(*partial_json: str) -> tuple[dict[str, object], ...]: + return ( + { + "type": "message_start", + "message": {"id": "msg_01", "model": "claude-opus-5-5", "usage": {"input_tokens": 9}}, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_01Web", "name": "web_search", "input": {}}, + }, + *(_server_tool_input_delta(fragment) for fragment in partial_json), + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_01Web", "content": []}, + }, + {"type": "content_block_stop", "index": 1}, + {"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "Paris is in France."}}, + {"type": "content_block_stop", "index": 2}, + ) + + +_SERVER_SEARCH_THEN_ANSWER: Final = _server_search_then_answer('{"query": "Paris"}') +_CLIENT_TOOL_CALL: Final[tuple[dict[str, object], ...]] = ( + { + "type": "content_block_start", + "index": 3, + "content_block": {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {}}, + }, + { + "type": "content_block_delta", + "index": 3, + "delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'}, + }, + {"type": "content_block_stop", "index": 3}, +) + + +def _message_delta(stop_reason: str) -> dict[str, object]: + return {"type": "message_delta", "delta": {"stop_reason": stop_reason}, "usage": {"output_tokens": 20}} + + +_PARIS_SEARCH: Final = ("srvtoolu_01Web", "web_search", '{"query": "Paris"}') + + +@pytest.mark.parametrize( + ("events", "expected_tool_calls", "expected_finish_reason", "expected_server_tool_call"), + [ + pytest.param( + (*_SERVER_SEARCH_THEN_ANSWER, _message_delta("end_turn")), [], "stop", _PARIS_SEARCH, id="server_tool_only" + ), + pytest.param( + (*_SERVER_SEARCH_THEN_ANSWER, *_CLIENT_TOOL_CALL, _message_delta("tool_use")), + [("toolu_01Weather", "get_weather", '{"city": "Paris"}')], + "tool_calls", + _PARIS_SEARCH, + id="server_tool_then_client_tool", + ), + pytest.param( + (*_server_search_then_answer('{"que', 'ry": "Pa', 'ris"}'), _message_delta("end_turn")), + [], + "stop", + _PARIS_SEARCH, + id="server_tool_input_split_across_deltas", + ), + pytest.param( + (*_server_search_then_answer(), _message_delta("end_turn")), + [], + "stop", + ("srvtoolu_01Web", "web_search", "{}"), + id="server_tool_with_empty_input", + ), + ], +) +def test_streamed_server_tool_use_is_not_returned_as_a_client_tool_call( + events: tuple[dict[str, object], ...], + expected_tool_calls: list[tuple[str, str, str]], + expected_finish_reason: str, + expected_server_tool_call: tuple[str, str, str], +): + choice: Final = _assemble_stream(events).choices[0] + assert isinstance(choice, Choices) + + assert (choice.message.tool_calls is None) == (not expected_tool_calls) + assert [ + (call.id, call.function.name, call.function.arguments) + for call in choice.message.tool_calls or () + if isinstance(call, ChatCompletionMessageToolCall) + ] == expected_tool_calls + assert choice.finish_reason == expected_finish_reason + assert choice.message.content == "Paris is in France." + assert [ + (call["id"], call["function"].get("name"), call["function"].get("arguments")) + for call in anthropic_server_tool_call_params(choice.message) + ] == [expected_server_tool_call] + web_search_results: Final = message_field(choice.message.provider_specific_fields, "web_search_results") + assert [message_field(result, "tool_use_id") for result in parts_of(web_search_results)] == ["srvtoolu_01Web"] def test_current_content_block_type_tracking(): diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 4d5d6cc17a9..255162b3cd9 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,7 +1,8 @@ import copy import json +from collections.abc import Mapping from itertools import chain -from typing import Final +from typing import Final, cast from unittest.mock import MagicMock, patch import httpx @@ -22,8 +23,13 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of from litellm.llms.anthropic.chat import ModelResponseIterator -from litellm.llms.anthropic.common_utils import process_anthropic_headers +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + process_anthropic_headers, + validated_object_mapping, +) from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema from litellm.llms.anthropic.messages.native.transformation import ( @@ -38,8 +44,8 @@ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation im VertexAIAnthropicConfig, ) from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES -from litellm.types.llms.openai import ChatCompletionToolCallFunctionChunk -from litellm.types.utils import ChatCompletionToolCallChunk, ModelResponse, ServerToolUse, Usage +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallFunctionChunk +from litellm.types.utils import ChatCompletionToolCallChunk, Choices, Message, ModelResponse, ServerToolUse, Usage from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE @@ -3509,21 +3515,17 @@ def test_code_execution_tool_results_extraction(): prefix_prompt=None, ) - # Verify tool calls are present - assert transformed_response.choices[0].message.tool_calls is not None - assert len(transformed_response.choices[0].message.tool_calls) == 2 + assert transformed_response.choices[0].message.tool_calls is None - # Verify first tool call - assert transformed_response.choices[0].message.tool_calls[0].id == "srvtoolu_01ABC" - assert transformed_response.choices[0].message.tool_calls[0].function.name == "bash_code_execution" - - # Verify second tool call - assert transformed_response.choices[0].message.tool_calls[1].id == "srvtoolu_01DEF" - assert transformed_response.choices[0].message.tool_calls[1].function.name == "text_editor_code_execution" - - # Verify tool results are in provider_specific_fields provider_fields = transformed_response.choices[0].message.provider_specific_fields assert provider_fields is not None + server_tool_calls: Final = anthropic_server_tool_call_params(transformed_response.choices[0].message) + assert [(call["id"], call["function"].get("name")) for call in server_tool_calls] == [ + ("srvtoolu_01ABC", "bash_code_execution"), + ("srvtoolu_01DEF", "text_editor_code_execution"), + ] + + # Verify tool results are in provider_specific_fields assert "tool_results" in provider_fields assert provider_fields["tool_results"] is not None assert len(provider_fields["tool_results"]) == 2 @@ -3633,7 +3635,7 @@ def _tool_search_turn(tool_references: list[str], last_block: dict[str, object]) ] -def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str): +def _parsed_choice(content: list[dict[str, object]], stop_reason: str) -> Choices: anthropic_response: Final = { "id": "msg_01XYZ", "type": "message", @@ -3648,7 +3650,7 @@ def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str raw_response.json.return_value = anthropic_response raw_response.status_code = 200 raw_response.headers = {} - return ( + choice: Final = ( AnthropicConfig() .transform_parsed_response( completion_response=anthropic_response, @@ -3658,14 +3660,19 @@ def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str prefix_prompt=None, ) .choices[0] - .message ) + assert isinstance(choice, Choices) + return choice + + +def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str) -> Message: + return _parsed_choice(content, stop_reason).message _WEATHER_CALL: Final = {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}} -def _web_search_turn(last_block: dict[str, object]) -> list[dict[str, object]]: +def _web_search_turn(last_block: Mapping[str, object]) -> list[dict[str, object]]: return [ {"type": "server_tool_use", "id": "srvtoolu_01Web", "name": "web_search", "input": {"query": "Paris facts"}}, { @@ -3681,7 +3688,7 @@ def _web_search_turn(last_block: dict[str, object]) -> list[dict[str, object]]: } ], }, - last_block, + dict(last_block), ] @@ -3728,8 +3735,114 @@ def test_server_tool_result_round_trips_right_after_its_call( assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"] -def _block_ids(message: dict[str, object], block_type: str, id_key: str) -> list[object]: - return sorted(block[id_key] for block in message["content"] if block["type"] == block_type) +_FETCH_STILL_WAITING_ON_ITS_RESULT: Final = { + "type": "server_tool_use", + "id": "srvtoolu_01Fetch", + "name": "web_fetch", + "input": {"url": "https://example.com/"}, +} + + +@pytest.mark.parametrize("modify_params", [False, True]) +def test_server_call_still_waiting_on_its_result_is_replayed_as_a_server_call( + modify_params: bool, monkeypatch: pytest.MonkeyPatch +): + content: Final = [_FETCH_STILL_WAITING_ON_ITS_RESULT, _WEATHER_CALL] + message: Final = _parsed_assistant_message(content, "tool_use") + monkeypatch.setattr(litellm, "modify_params", modify_params) + + replayed: Final = _replayed( + {"role": "user", "content": "Fetch example.com, then tell me the weather in Paris"}, + message.model_dump(), + {"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C"}, + ) + + assert replayed[1]["content"] == content + assert [turn["role"] for turn in replayed] == ["user", "assistant", "user"] + assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"] + + +_PARIS_TEXT: Final = "Paris is the capital of France." +_PARIS_ANSWER: Final[Mapping[str, str]] = {"type": "text", "text": _PARIS_TEXT} + + +@pytest.mark.parametrize( + ("content", "stop_reason", "expected_tool_call_ids", "expected_finish_reason"), + [ + pytest.param(_web_search_turn(_PARIS_ANSWER), "end_turn", [], "stop", id="web_search_then_answer"), + pytest.param( + _web_search_turn(_WEATHER_CALL), "tool_use", ["toolu_01Weather"], "tool_calls", id="web_search_then_tool" + ), + pytest.param([_WEATHER_CALL], "tool_use", ["toolu_01Weather"], "tool_calls", id="client_tool_only"), + ], +) +def test_server_tool_use_is_not_returned_as_a_client_tool_call( + content: list[dict[str, object]], + stop_reason: str, + expected_tool_call_ids: list[str], + expected_finish_reason: str, +): + choice: Final = _parsed_choice(content, stop_reason) + + assert (choice.message.tool_calls is None) == (not expected_tool_call_ids) + assert [call.id for call in choice.message.tool_calls or ()] == expected_tool_call_ids + assert choice.finish_reason == expected_finish_reason + assert [ + (call["id"], call["function"].get("name"), call["function"].get("arguments")) + for call in anthropic_server_tool_call_params(choice.message) + ] == [ + (block["id"], block["name"], json.dumps(block["input"])) + for block in content + if block["type"] == "server_tool_use" + ] + if not expected_tool_call_ids: + assert choice.message.content == _PARIS_TEXT + + +def _replayed(*messages: object): + return anthropic_messages_pt( + messages=cast(list[AllMessageValues], list(messages)), # cast-ok: replays history the way clients send it back + model="claude-opus-5-5", + llm_provider="anthropic", + ) + + +def test_web_search_answer_round_trips_without_any_client_tool_call(): + message: Final = _parsed_assistant_message(_web_search_turn(_PARIS_ANSWER), "end_turn") + + replayed: Final = _replayed( + {"role": "user", "content": "Tell me about Paris"}, + message.model_dump(), + {"role": "user", "content": "And Lyon?"}, + ) + + assert _block_ids(replayed[1], "server_tool_use", "id") == ["srvtoolu_01Web"] + assert _block_ids(replayed[1], "web_search_tool_result", "tool_use_id") == ["srvtoolu_01Web"] + assert [ + message_field(block, "text") + for block in parts_of(replayed[1]["content"]) + if message_field(block, "type") == "text" + ] == [_PARIS_TEXT] + + +def _block_ids(message: Mapping[str, object], block_type: str, id_key: str) -> list[str]: + return sorted( + block_id + for block in parts_of(message["content"]) + if message_field(block, "type") == block_type and isinstance(block_id := message_field(block, id_key), str) + ) + + +def _as_returned_before_server_tool_calls_moved(message: Message) -> Mapping[str, object]: + dumped: Final[Mapping[str, object]] = message.model_dump() + provider_fields: Final = validated_object_mapping(dumped.get("provider_specific_fields")) or {} + return { + **dumped, + "tool_calls": [*anthropic_server_tool_call_params(message), *parts_of(dumped.get("tool_calls"))], + "provider_specific_fields": { + key: value for key, value in provider_fields.items() if key != "server_tool_calls" + }, + } _SEARCH_THAT_FOUND_NOTHING: Final = _tool_search_turn( @@ -3764,21 +3877,33 @@ def test_replayed_search_that_found_nothing_answers_every_tool_use_and_ends_on_a server_tool_use_ids: list[str], monkeypatch: pytest.MonkeyPatch, ): - from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt - message: Final = _parsed_assistant_message(_SEARCH_THAT_FOUND_NOTHING, "end_turn") monkeypatch.setattr(litellm, "modify_params", modify_params) - replayed: Final = anthropic_messages_pt( - messages=[{"role": "user", "content": "Find a tool and use it"}, message.model_dump(), *follow_ups], - model="claude-opus-5-5", - llm_provider="anthropic", + replayed: Final = _replayed( + {"role": "user", "content": "Find a tool and use it"}, + _as_returned_before_server_tool_calls_moved(message), + *follow_ups, ) assert [turn["role"] for turn in replayed] == ["user", "assistant", "user"] assert _block_ids(replayed[2], "tool_result", "tool_use_id") == _block_ids(replayed[1], "tool_use", "id") assert _block_ids(replayed[1], "server_tool_use", "id") == server_tool_use_ids +def test_replayed_search_that_found_nothing_keeps_its_result_when_the_call_never_reached_the_client( + monkeypatch: pytest.MonkeyPatch, +): + message: Final = _parsed_assistant_message(_SEARCH_THAT_FOUND_NOTHING, "end_turn") + monkeypatch.setattr(litellm, "modify_params", True) + + replayed: Final = _replayed({"role": "user", "content": "Find a tool and use it"}, message.model_dump()) + + assert [turn["role"] for turn in replayed] == ["user", "assistant"] + assert _block_ids(replayed[1], "server_tool_use", "id") == ["srvtoolu_01Search"] + assert _block_ids(replayed[1], "tool_search_tool_result", "tool_use_id") == ["srvtoolu_01Search"] + assert _block_ids(replayed[1], "tool_use", "id") == [] + + _CODE_EXECUTION_BLOCKS: Final = [ { "type": "server_tool_use", diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index c30644e7fdc..54ffec73ad0 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -80,6 +80,59 @@ def test_native_compaction_response_roundtrip(text: str | None, count: int, expe assert result["stop_reason"] == expected_stop +_SERVER_AND_CLIENT_TOOL_TURN: Final[tuple[dict[str, object], ...]] = ( + { + "type": "server_tool_use", + "id": "srvtoolu_01Fetch", + "name": "web_fetch", + "input": {"url": "https://example.com/"}, + }, + { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_01Fetch", + "content": {"type": "web_fetch_result", "url": "https://example.com/"}, + }, + {"type": "server_tool_use", "id": "srvtoolu_02Search", "name": "web_search", "input": {"query": "Paris"}}, + {"type": "text", "text": "The page is titled Example."}, + {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}}, +) + + +def test_claude_server_tool_blocks_survive_the_round_trip_through_chat_completions() -> None: + from unittest.mock import MagicMock + + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + anthropic_response: Final = { + "id": "msg_01Fetch", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": list(_SERVER_AND_CLIENT_TOOL_TURN), + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 100, "output_tokens": 50}, + } + raw_response: Final = MagicMock(spec=httpx.Response) + raw_response.json.return_value = anthropic_response + raw_response.status_code = 200 + raw_response.headers = {} + chat_response: Final = AnthropicConfig().transform_parsed_response( + completion_response=anthropic_response, + raw_response=raw_response, + model_response=ModelResponse(), + json_mode=False, + prefix_prompt=None, + ) + + result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(chat_response) + + assert [(block["type"], block.get("id") or block.get("tool_use_id")) for block in result["content"]] == [ + (block["type"], block.get("id") or block.get("tool_use_id")) for block in _SERVER_AND_CLIENT_TOOL_TURN + ] + assert result["stop_reason"] == "tool_use" + + def test_translate_chat_refusal_to_anthropic_response(): response = ModelResponse( id="chatcmpl-refusal", diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 5d71cbe9afc..fcc3b1e2b3c 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -144,6 +144,42 @@ class MockMisalignedToolCallGuardrail(CustomGuardrail): ) +def _server_tool_call(call_id: str, name: str, arguments: Mapping[str, object]) -> dict[str, object]: + return {"id": call_id, "type": "function", "function": {"name": name, "arguments": json.dumps(arguments)}} + + +def _tool_call_ids_names_and_arguments(tool_calls) -> list[tuple[str, str, str]]: + return [ + (tool_call["id"], tool_call["function"]["name"], tool_call["function"]["arguments"]) + for tool_call in tool_calls + ] + + +class QueryRedactingGuardrail(CustomGuardrail): + """Returns copied tool calls with any ``query`` argument redacted, leaving other calls as they came.""" + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + def redacted(tool_call): + arguments = json.loads(tool_call["function"]["arguments"]) + if "query" not in arguments: + return tool_call + return { + **tool_call, + "function": {**tool_call["function"], "arguments": json.dumps({**arguments, "query": "[REDACTED]"})}, + } + + return GenericGuardrailAPIInputs( + texts=inputs.get("texts", []), + tool_calls=[redacted(tool_call) for tool_call in inputs.get("tool_calls", [])], + ) + + class TestOpenAIChatCompletionsHandlerToolsInput: """Test input processing with tools (function definitions)""" @@ -906,6 +942,92 @@ class TestOpenAIChatCompletionsHandlerToolCallsOutput: assert tool_calls[0].function.arguments == first_args assert tool_calls[1].function.arguments == second_args + @pytest.mark.asyncio + async def test_output_guardrail_sees_a_response_that_only_carries_server_tool_calls(self): + handler = OpenAIChatCompletionsHandler() + guardrail = TestOpenAIChatCompletionsHandlerStreamingOutput._recording_guardrail() + response = ModelResponse( + model="claude-sonnet-4-5", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + content=None, + role="assistant", + provider_specific_fields={ + "server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})] + }, + ), + ) + ], + ) + + await handler.process_output_response(response, guardrail) + + assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "x"}') + ] + + @pytest.mark.asyncio + async def test_output_guardrail_rewrite_of_a_server_tool_call_lands_in_server_tool_calls_only(self): + handler = OpenAIChatCompletionsHandler() + weather_args = json.dumps({"city": "Paris", "query": "forecast"}) + response = ModelResponse( + model="claude-sonnet-4-5", + choices=[ + Choices( + finish_reason="tool_calls", + index=0, + message=Message( + content="Looking it up", + role="assistant", + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_1", + type="function", + function=Function(name="get_weather", arguments=json.dumps({"city": "Paris"})), + ) + ], + provider_specific_fields={ + "server_tool_calls": [ + _server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"}) + ] + }, + ), + ), + Choices( + finish_reason="tool_calls", + index=1, + message=Message( + content=None, + role="assistant", + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_2", + type="function", + function=Function(name="get_weather", arguments=weather_args), + ) + ], + ), + ), + ], + ) + + await handler.process_output_response(response, QueryRedactingGuardrail(guardrail_name="redact")) + + first, second = response.choices + assert _tool_call_ids_names_and_arguments(first.message.provider_specific_fields["server_tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}') + ] + assert [(call.id, call.function.arguments) for call in first.message.tool_calls] == [ + ("call_1", '{"city": "Paris"}') + ] + assert [(call.id, call.function.arguments) for call in second.message.tool_calls] == [ + ("call_2", '{"city": "Paris", "query": "[REDACTED]"}') + ] + assert not (second.message.provider_specific_fields or {}).get("server_tool_calls") + class MockPassThroughGuardrail(CustomGuardrail): """Mock guardrail that passes through without blocking - for testing streaming fallback behavior""" @@ -1462,6 +1584,61 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert chunks[0].choices[0].delta.content == "hello [MASKED]" assert chunks[1].choices[0].delta.content in (None, "") + @staticmethod + def _ended_server_tool_call_stream_chunks() -> list: + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + def chunk(delta: Delta, finish_reason: Optional[str] = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-123", + created=1234567890, + model="claude-sonnet-4-5", + object="chat.completion.chunk", + choices=[StreamingChoices(index=0, delta=delta, finish_reason=finish_reason)], + ) + + search = {**_server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"}), "index": 0} + return [ + chunk(Delta(content="Searching")), + chunk(Delta(content=None, provider_specific_fields={"server_tool_calls": [search]})), + chunk(Delta(content=None), finish_reason="stop"), + ] + + @pytest.mark.asyncio + async def test_ended_stream_guardrail_sees_streamed_server_tool_calls(self): + handler = OpenAIChatCompletionsHandler() + guardrail = self._recording_guardrail() + + await handler.process_output_streaming_response( + responses_so_far=self._ended_server_tool_call_stream_chunks(), + guardrail_to_apply=guardrail, + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "jane@example.com"}') + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_writes_server_tool_call_arguments_back_into_chunks(self): + handler = OpenAIChatCompletionsHandler() + chunks = self._ended_server_tool_call_stream_chunks() + + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=QueryRedactingGuardrail(guardrail_name="redact"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + delta = chunks[1].choices[0].delta + assert _tool_call_ids_names_and_arguments(delta.provider_specific_fields["server_tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}') + ] + assert delta.tool_calls is None + assert chunks[0].choices[0].delta.content == "Searching" + class TestUndecoratedGuardrailIsRecorded: """LIT-5983 regression: the handler calls apply_guardrail bare, so a custom guardrail @@ -2323,6 +2500,29 @@ class TestStreamingScanKey: assert len(ended_key.tool_calls) == 1 and "run_shell" in ended_key.tool_calls[0] assert ended_key.tool_calls_in_flight is False + def test_server_tool_calls_hold_the_window_and_enter_the_key_once_the_stream_has_ended(self): + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + handler = OpenAIChatCompletionsHandler() + server_chunk = ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta( + content=None, + provider_specific_fields={ + "server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})] + }, + ), + ) + ] + ) + open_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk]) + ended_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk, self._chunk(None, "stop")]) + assert open_key.tool_calls_in_flight is True + assert len(ended_key.tool_calls) == 1 and "web_search" in ended_key.tool_calls[0] + assert ended_key != open_key + def test_text_after_the_first_choice_finishes_still_changes_the_key(self): handler = OpenAIChatCompletionsHandler() first_done = [self._chunk("a", index=0), self._chunk("b", finish_reason="stop", index=0)] diff --git a/tests/unit/proxy/db/test_spend_log_tool_index.py b/tests/unit/proxy/db/test_spend_log_tool_index.py index af3268a7cc9..a3f400bbc15 100644 --- a/tests/unit/proxy/db/test_spend_log_tool_index.py +++ b/tests/unit/proxy/db/test_spend_log_tool_index.py @@ -23,6 +23,7 @@ from litellm.proxy.db.spend_log_tool_index import ( ) from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import ModelResponse from tests.unit.proxy.db.fake_prisma_engine import engine_call @@ -204,6 +205,36 @@ class TestResponseToolCallNames: def test_blank_names_are_dropped(self): assert response_tool_call_names(_response_with_tool_calls(" ", "real_tool")) == ("real_tool",) + @pytest.mark.parametrize("as_object", [False, True], ids=["dict", "model_response"]) + def test_server_tool_calls_of_every_choice_are_named_before_client_tool_calls(self, as_object: bool): + def tool_call(call_id: str, name: str) -> dict[str, object]: + return {"id": call_id, "type": "function", "function": {"name": name, "arguments": "{}"}} + + response: dict[str, object] = { + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [tool_call("call_1", "get_weather")], + "provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_01", "web_search")]}, + }, + }, + { + "index": 1, + "message": { + "role": "assistant", + "content": "done", + "provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_02", "web_fetch")]}, + }, + }, + ] + } + completion_response = ModelResponse(**response) if as_object else response + + assert response_tool_call_names(completion_response) == ("web_search", "web_fetch", "get_weather") + def test_responses_api_output_function_calls(self): # Regression: /v1/responses carries invocations in output[] items of # type function_call, not in choices; they must reach the rollup. diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py index 2ee4c8c72c0..1fac7e7a4c9 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py @@ -27,11 +27,21 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( from litellm.types.utils import ( ChatCompletionMessageToolCall, Choices, + Delta, ModelResponse, ModelResponseStream, + StreamingChoices, ) +_SERVER_WEB_SEARCH_CALL: Final = { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "weather in Paris"}'}, +} +_ALLOWED_BASH_CALL: Final = {"id": "toolu_01Bash", "type": "function", "function": {"name": "Bash", "arguments": "{}"}} + + class TestToolPermissionGuardrail: """Test class for Tool Permission Guardrail functionality""" @@ -952,6 +962,71 @@ class TestToolPermissionGuardrail: assert "Permission denied" in choice.message.content + def _server_search_response(self) -> ModelResponse: + return ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message={ + "content": "Searching.", + "tool_calls": [_ALLOWED_BASH_CALL], + "provider_specific_fields": {"server_tool_calls": [_SERVER_WEB_SEARCH_CALL]}, + }, + ) + ] + ) + + @pytest.mark.asyncio + async def test_denied_server_tool_call_is_blocked(self): + with patch.object(self.guardrail, "should_run_guardrail", return_value=True): + with pytest.raises(GuardrailRaisedException): + await self.guardrail.async_post_call_success_hook( + data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response() + ) + + @pytest.mark.asyncio + async def test_denied_server_tool_call_in_a_chat_stream_is_blocked(self): + async def _stream(): + yield ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta( + content="Searching.", + provider_specific_fields={"server_tool_calls": [{**_SERVER_WEB_SEARCH_CALL, "index": 0}]}, + ), + finish_reason="stop", + ) + ] + ) + + with patch.object(self.guardrail, "should_run_guardrail", return_value=True): + with pytest.raises(GuardrailRaisedException): + async for _ in self.guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_stream(), request_data={} + ): + pass + + @pytest.mark.asyncio + async def test_rewrite_mode_removes_the_denied_server_tool_call(self): + rewriting: Final = ToolPermissionGuardrail( + guardrail_name="test-tool-permission", + rules=self.test_rules, + default_action="deny", + on_disallowed_action="rewrite", + ) + + with patch.object(rewriting, "should_run_guardrail", return_value=True): + result: Final = await rewriting.async_post_call_success_hook( + data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response() + ) + + message: Final = result.choices[0].message + assert message.provider_specific_fields["server_tool_calls"] == [] + assert [tool_call.id for tool_call in message.tool_calls] == [_ALLOWED_BASH_CALL["id"]] + assert "Permission denied" in message.content + + class TestToolPermissionGuardrailIntegration: """Integration tests for Tool Permission Guardrail""" diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 4250c78c669..aabf2e1d7cc 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -33,7 +33,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.types.llms.base import HiddenParams -from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse from litellm.types.responses.main import OutputToolSearchCall, build_web_search_call from litellm.types.utils import ( ChatCompletionMessageToolCall, @@ -4993,6 +4993,159 @@ def test_function_call_tool_id_falls_back_to_unique_id_for_degenerate_call_id(): assert convert(openai)["id"] == "call_tokyo" +_WEATHER_TOOL_USE: Final = { + "type": "tool_use", + "id": "toolu_01Weather", + "name": "get_weather", + "input": {"city": "Paris"}, +} +_FETCHED_URL: Final = "https://example.com/" +_BLOCKED_URL: Final = "https://blocked.invalid/" +_ANTHROPIC_WEB_FETCH_TURN: Final[tuple[dict[str, object], ...]] = ( + {"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch", "input": {"url": _FETCHED_URL}}, + { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_01Fetch", + "content": { + "type": "web_fetch_result", + "url": _FETCHED_URL, + "content": {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Example"}}, + }, + }, + {"type": "server_tool_use", "id": "srvtoolu_02Fetch", "name": "web_fetch", "input": {"url": _BLOCKED_URL}}, + { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_02Fetch", + "content": {"type": "web_fetch_tool_result_error", "error_code": "url_not_allowed"}, + }, + {"type": "text", "text": "The page is titled Example."}, + _WEATHER_TOOL_USE, +) + + +def _anthropic_turn_on_responses(content: Sequence[Mapping[str, object]]) -> ResponsesAPIResponse: + from unittest.mock import MagicMock + + import httpx + + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + anthropic_response: Final = { + "id": "msg_01Fetch", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": list(content), + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 100, "output_tokens": 50}, + } + raw_response: Final = MagicMock(spec=httpx.Response) + raw_response.json.return_value = anthropic_response + raw_response.status_code = 200 + raw_response.headers = {} + chat_response: Final = AnthropicConfig().transform_parsed_response( + completion_response=anthropic_response, + raw_response=raw_response, + model_response=ModelResponse(), + json_mode=False, + prefix_prompt=None, + ) + return LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Use the web and check the weather in Paris.", + responses_api_request={ + "tools": [ + {"type": "web_search_20250305", "name": "web_search"}, + {"type": "web_fetch_20250910", "name": "web_fetch"}, + {"type": "function", "name": "get_weather", "parameters": {"type": "object"}}, + ] + }, + chat_completion_response=chat_response, + ) + + +def test_anthropic_web_fetch_is_a_hosted_open_page_call_on_responses() -> None: + response: Final = _anthropic_turn_on_responses(_ANTHROPIC_WEB_FETCH_TURN) + + assert [ + (item.id, item.status, item.action.model_dump()) for item in response.output if item.type == "web_search_call" + ] == [ + ("ws_srvtoolu_01Fetch", "completed", {"type": "open_page", "url": _FETCHED_URL}), + ("ws_srvtoolu_02Fetch", "failed", {"type": "open_page", "url": _BLOCKED_URL}), + ] + assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"] + + +@pytest.mark.parametrize( + ("server_tool_use", "expected_action"), + [ + ( + { + "type": "server_tool_use", + "id": "srvtoolu_01Pending", + "name": "web_fetch", + "input": {"url": _FETCHED_URL}, + }, + {"type": "open_page", "url": _FETCHED_URL}, + ), + ( + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}}, + {"type": "search", "query": "python"}, + ), + ], +) +def test_anthropic_server_web_call_without_a_result_is_an_in_progress_hosted_call_on_responses( + server_tool_use: Mapping[str, object], expected_action: Mapping[str, object] +) -> None: + response: Final = _anthropic_turn_on_responses((server_tool_use, _WEATHER_TOOL_USE)) + + web_calls: Final = [item for item in response.output if item.type == "web_search_call"] + assert [(item.id, item.status) for item in web_calls] == [("ws_srvtoolu_01Pending", "in_progress")] + assert expected_action.items() <= web_calls[0].action.model_dump().items() + assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"] + + +def test_anthropic_code_execution_is_never_a_client_function_call_on_responses() -> None: + response: Final = _anthropic_turn_on_responses( + ( + { + "type": "server_tool_use", + "id": "srvtoolu_01Bash", + "name": "bash_code_execution", + "input": {"command": "ls"}, + }, + { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01Bash", + "content": { + "type": "bash_code_execution_result", + "stdout": "notes.txt\n", + "stderr": "", + "return_code": 0, + }, + }, + { + "type": "server_tool_use", + "id": "srvtoolu_02Edit", + "name": "text_editor_code_execution", + "input": {"command": "view", "path": "notes.txt"}, + }, + { + "type": "server_tool_use", + "id": "srvtoolu_03Bash", + "name": "bash_code_execution", + "input": {"command": "pwd"}, + }, + _WEATHER_TOOL_USE, + ) + ) + + assert [(item.type, getattr(item, "call_id", item.id)) for item in response.output if item.type != "message"] == [ + ("code_interpreter_call", "srvtoolu_01Bash"), + ("function_call", "toolu_01Weather"), + ] + + class TestHostedWebSearchReplay: def test_emitted_hosted_search_output_round_trips_with_client_tool_result(self) -> None: search_result: Final = { @@ -5063,6 +5216,75 @@ class TestHostedWebSearchReplay: assert messages[2]["content"] == "Paris is sunny." assert history == original + @pytest.mark.parametrize( + "leading_text", ((), ({"type": "text", "text": "Let me look that up."},)), ids=("alone", "after_text") + ) + @pytest.mark.parametrize( + ("server_tool_use", "expected_input"), + ( + ( + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_fetch", "input": {"url": _FETCHED_URL}}, + {"url": _FETCHED_URL}, + ), + ( + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}}, + {"query": "python"}, + ), + ), + ids=("web_fetch", "web_search"), + ) + def test_pending_claude_web_call_replays_to_anthropic_as_its_own_server_tool_use( + self, + leading_text: tuple[Mapping[str, object], ...], + server_tool_use: Mapping[str, object], + expected_input: Mapping[str, object], + ) -> None: + response: Final = _anthropic_turn_on_responses((*leading_text, server_tool_use, _WEATHER_TOOL_USE)) + history: Final = [ + {"role": "user", "content": "Use the web and check the weather in Paris."}, + *(item.model_dump(exclude_none=True) for item in response.output), + {"type": "function_call_output", "call_id": "toolu_01Weather", "output": "Paris is sunny."}, + ] + + anthropic_turns: Final = anthropic_messages_pt( + messages=LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=history, responses_api_request={} + ), + model="claude-sonnet-5-5", + llm_provider="anthropic", + ) + + assert [turn["role"] for turn in anthropic_turns] == ["user", "assistant", "user"] + assert [block for block in anthropic_turns[1]["content"] if block["type"] != "text"] == [ + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": server_tool_use["name"], "input": expected_input}, + {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}}, + ] + assert [(block["type"], block["tool_use_id"]) for block in anthropic_turns[2]["content"]] == [ + ("tool_result", "toolu_01Weather") + ] + + @pytest.mark.parametrize( + ("call_id", "status"), + (("ws_srvtoolu_01Fetch", "completed"), ("ws_68a1b2c3d4", "in_progress"), ("srvtoolu_01Fetch", "in_progress")), + ids=("completed-claude-call", "pending-non-claude-call", "pending-unprefixed-id"), + ) + def test_web_call_that_is_not_a_pending_claude_call_replays_as_text_only( + self, call_id: str, status: Literal["completed", "in_progress"] + ) -> None: + search: Final = ResponseFunctionWebSearch( + id=call_id, type="web_search_call", status=status, action=ActionOpenPage(type="open_page", url=_FETCHED_URL) + ) + + messages: Final = ( + LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=search.model_dump(exclude_none=True) + ) + ) + + assert messages == [ + {"role": "assistant", "content": "Hosted web search: " + search.model_dump_json(exclude_none=True)} + ] + @pytest.mark.parametrize( "action", ( diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 86773dc7ddf..e40b3febd3f 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -356,6 +356,198 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to ) +_STREAMED_WEB_FETCH_CALL: Final[tuple[dict[str, object], ...]] = ( + { + "type": "message_start", + "message": { + "id": "msg_01Fetch", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch"}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com/"}'}, + }, + {"type": "content_block_stop", "index": 0}, +) +_STREAMED_WEB_FETCH_RESULT: Final[tuple[dict[str, object], ...]] = ( + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_01Fetch", + "content": {"type": "web_fetch_result", "url": "https://example.com/"}, + }, + }, + {"type": "content_block_stop", "index": 1}, +) + + +def _streamed_text_then_weather_call(first_index: int) -> tuple[dict[str, object], ...]: + return ( + {"type": "content_block_start", "index": first_index, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": first_index, "delta": {"type": "text_delta", "text": "Fetched."}}, + {"type": "content_block_stop", "index": first_index}, + { + "type": "content_block_start", + "index": first_index + 1, + "content_block": {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {}}, + }, + { + "type": "content_block_delta", + "index": first_index + 1, + "delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'}, + }, + {"type": "content_block_stop", "index": first_index + 1}, + { + "type": "message_delta", + "delta": {"stop_reason": "tool_use", "stop_sequence": None}, + "usage": {"output_tokens": 20}, + }, + ) + + +def _anthropic_web_fetch_stream_events(fetch_finished: bool) -> tuple[dict[str, object], ...]: + if fetch_finished: + return (*_STREAMED_WEB_FETCH_CALL, *_STREAMED_WEB_FETCH_RESULT, *_streamed_text_then_weather_call(2)) + return (*_STREAMED_WEB_FETCH_CALL, *_streamed_text_then_weather_call(1)) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize( + ("fetch_finished", "expected_status", "expected_search_events"), + [ + pytest.param( + True, + "completed", + [ + ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, + ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + ], + id="fetch_finished_in_this_turn", + ), + pytest.param( + False, + "in_progress", + [ + ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, + ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + ], + id="fetch_still_waiting_on_its_result", + ), + ], +) +@pytest.mark.asyncio +async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call( + sync_mode: bool, + fetch_finished: bool, + expected_status: str, + expected_search_events: list[ResponsesAPIStreamEvents], +) -> None: + from litellm.llms.anthropic.chat.handler import ModelResponseIterator + + parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False) + chunks: Final = [parser.chunk_parser(event) for event in _anthropic_web_fetch_stream_events(fetch_finished)] + iterator: Final = LiteLLMCompletionStreamingIterator( + model="claude-sonnet-5-5", + litellm_custom_stream_wrapper=_FakeStreamWrapper(chunks), + request_input="Fetch example.com and check the weather in Paris.", + responses_api_request={ + "tools": [ + {"type": "web_fetch_20250910", "name": "web_fetch"}, + {"type": "function", "name": "get_weather", "parameters": {"type": "object"}}, + ] + }, + custom_llm_provider="anthropic", + ) + + events: Final = await _collect_events(iterator, sync_mode) + + completed: Final = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + assert [ + (item.id, item.status, item.action.model_dump()) + for item in completed.response.output + if item.type == "web_search_call" + ] == [("ws_srvtoolu_01Fetch", expected_status, {"type": "open_page", "url": "https://example.com/"})] + assert [ + event.type + for event in events + if getattr(event, "item_id", getattr(getattr(event, "item", None), "id", None)) == "ws_srvtoolu_01Fetch" + ] == expected_search_events + assert [item.name for item in completed.response.output if item.type == "function_call"] == ["get_weather"] + assert not any( + event.item.name == "web_fetch" + for event in events + if getattr(event, "type", None) == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + and event.item.type == "function_call" + ) + + +_STREAMED_PENDING_CODE_EXECUTION: Final[tuple[dict[str, object], ...]] = ( + _STREAMED_WEB_FETCH_CALL[0], + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_01Bash", "name": "bash_code_execution"}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"command": "ls"}'}, + }, + {"type": "content_block_stop", "index": 0}, + *_streamed_text_then_weather_call(1), +) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_streamed_anthropic_code_execution_is_never_a_client_function_call(sync_mode: bool) -> None: + from litellm.llms.anthropic.chat.handler import ModelResponseIterator + + parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False) + iterator: Final = LiteLLMCompletionStreamingIterator( + model="claude-sonnet-5-5", + litellm_custom_stream_wrapper=_FakeStreamWrapper( + [parser.chunk_parser(event) for event in _STREAMED_PENDING_CODE_EXECUTION] + ), + request_input="List the files, then check the weather in Paris.", + responses_api_request={ + "tools": [ + {"type": "code_execution_20250825", "name": "code_execution"}, + {"type": "function", "name": "get_weather", "parameters": {"type": "object"}}, + ] + }, + custom_llm_provider="anthropic", + ) + + events: Final = await _collect_events(iterator, sync_mode) + + item_events: Final = [event for event in events if getattr(event, "item", None) is not None] + assert {event.item.call_id for event in item_events if event.item.type == "function_call"} == {"toolu_01Weather"} + completed: Final = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + assert [item.call_id for item in completed.response.output if item.type == "function_call"] == ["toolu_01Weather"] + + def test_tool_calls_present_only_in_final_response_are_emitted_before_completed(): iterator = LiteLLMCompletionStreamingIterator( model="test-model", diff --git a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts index 78372fe56d9..a6693794df6 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts @@ -131,6 +131,52 @@ describe("ToolsSection utils", () => { expect(result[1].callData).toBeUndefined(); }); + it("should mark Claude server tools as called from provider_specific_fields.server_tool_calls", () => { + const log: Partial = { + request_id: "test-server-tools", + proxy_server_request: { + tools: [ + { type: "web_search_20250305", name: "web_search", max_uses: 3 }, + { type: "function", function: { name: "get_weather", description: "Get weather" } }, + { type: "function", function: { name: "send_email", description: "Send email" } }, + ], + }, + response: { + choices: [ + { + message: { + tool_calls: [ + { + id: "toolu_01Weather", + type: "function", + function: { name: "get_weather", arguments: '{"location": "Paris"}' }, + }, + ], + provider_specific_fields: { + server_tool_calls: [ + { + id: "srvtoolu_01Search", + type: "function", + function: { name: "web_search", arguments: '{"query": "paris weather"}' }, + }, + ], + }, + }, + }, + ], + }, + }; + + const result = parseToolsFromLog(log as LogEntry); + + expect(result.map((tool) => [tool.name, tool.called, tool.callData?.id])).toEqual([ + ["web_search", true, "srvtoolu_01Search"], + ["get_weather", true, "toolu_01Weather"], + ["send_email", false, undefined], + ]); + expect(result[0].callData?.arguments).toEqual({ query: "paris weather" }); + }); + it("should handle string format request and response", () => { const log: Partial = { request_id: "test-5", diff --git a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts index 5e76a09c155..b1b2e6feb99 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts @@ -42,6 +42,24 @@ function extractToolsFromRequest(log: LogEntry): ToolDefinition[] { return []; } +interface ChatMessageToolCallFields { + tool_calls?: unknown; + provider_specific_fields?: { server_tool_calls?: unknown } | null; +} + +function toolCallList(value: unknown): ToolCall[] { + return Array.isArray(value) ? value : []; +} + +function extractChatMessageToolCalls(message: ChatMessageToolCallFields | undefined): ToolCall[] | null { + const serverToolCalls = message?.provider_specific_fields?.server_tool_calls; + const clientToolCalls = message?.tool_calls; + if (!Array.isArray(serverToolCalls) && !Array.isArray(clientToolCalls)) { + return null; + } + return [...toolCallList(serverToolCalls), ...toolCallList(clientToolCalls)]; +} + /** * Extract tool calls from response data */ @@ -53,10 +71,9 @@ function extractToolCallsFromResponse(log: LogEntry): ToolCall[] { // OpenAI format: response.choices[0].message.tool_calls const choices = responseData.choices; if (Array.isArray(choices) && choices.length > 0) { - const firstChoice = choices[0]; - const message = firstChoice.message; - if (message && Array.isArray(message.tool_calls)) { - return message.tool_calls; + const chatToolCalls = extractChatMessageToolCalls(choices[0].message); + if (chatToolCalls) { + return chatToolCalls; } }