diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index a72023eaedf..79788d35854 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -44,6 +44,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.redact_messages import should_redact_message_logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -315,7 +316,10 @@ def _to_dd_tool_calls(message: Mapping[str, object]) -> tuple[ToolCall, ...]: tool_id=_text_field(tool_call, "id"), type=_text_field(tool_call, "type", "function"), ) - for tool_call in (raw_tool_calls if isinstance(raw_tool_calls, list) else ()) + for tool_call in ( + *anthropic_server_tool_call_params(message), + *(raw_tool_calls if isinstance(raw_tool_calls, list) else ()), + ) if isinstance(tool_call, dict) for function in [_mapping_field(tool_call, "function")] ) diff --git a/litellm/integrations/otel/mappers/utils.py b/litellm/integrations/otel/mappers/utils.py index 80ef2dd553a..bfd50fea5d5 100644 --- a/litellm/integrations/otel/mappers/utils.py +++ b/litellm/integrations/otel/mappers/utils.py @@ -12,6 +12,8 @@ from typing import Final from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue from litellm.integrations.otel.model.payloads import LLMCallSpanData, ToolDefinition +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params @dataclass(frozen=True, slots=True) @@ -141,12 +143,13 @@ def output_messages(data: LLMCallSpanData) -> list: def message_tool_calls(message: object) -> tuple[MessageToolCall, ...]: + tool_call_values: Final = ( + *anthropic_server_tool_call_params(message), + *parts_of(message_field(message, "tool_calls")), + ) if not isinstance(message, dict): return () - tool_calls: Final = message.get("tool_calls") - if not isinstance(tool_calls, (list, tuple)) or not tool_calls: - return () - return tuple(tool_call for value in tool_calls if (tool_call := _message_tool_call(value)) is not None) + return tuple(tool_call for value in tool_call_values if (tool_call := _message_tool_call(value)) is not None) def _message_tool_call(value: object) -> MessageToolCall | None: diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index bb02b9aab85..e414e333684 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -827,7 +827,6 @@ class ModelResponseIterator: return text, tool_use, thinking_blocks, provider_specific_fields, reasoning_content self.content_blocks.append(content_block) if "partial_json" in content_block["delta"]: - # Only emit tool calls if we're in a client tool_use block, server_tool_use runs on Anthropic's side # web_search_tool_result blocks also have input_json_delta but should not be treated as tool calls # See: https://github.com/BerriAI/litellm/issues/17254 if self.current_content_block_type == "tool_use": diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 3070bedd30d..d7c3aeca354 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -21,6 +21,7 @@ from collections.abc import Iterator, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Union, cast +from pydantic import TypeAdapter, ValidationError from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm @@ -29,6 +30,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i replayed_tool_search_tool_names, ) from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, StreamingScanKey, @@ -50,7 +52,12 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( stream_item_items, unappliable_request_rewrite, ) -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionAssistantToolCall, + ChatCompletionToolCallFunctionChunk, + ChatCompletionToolParam, +) from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( coerce_stream_holdback_value, ) @@ -436,9 +443,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation): text_task_mappings=text_task_mappings, tool_call_task_mappings=tool_call_task_mappings, ) + server_tool_call_tasks: Final = tuple(_server_tool_call_tasks(response.choices)) + tool_calls_with_server_calls: Final = [ + *tool_calls_to_check, + *(_server_tool_call_input(call) for _, call in server_tool_call_tasks), + ] # Step 2: Apply guardrail to all texts and tool calls in batch - if texts_to_check or tool_calls_to_check: + if texts_to_check or tool_calls_with_server_calls: # Use the real request_data if provided (proxy path), otherwise # create a standalone dict (SDK / direct-call path). if request_data is None: @@ -452,8 +464,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) if images_to_check: inputs["images"] = images_to_check - if tool_calls_to_check: - inputs["tool_calls"] = tool_calls_to_check + if tool_calls_with_server_calls: + inputs["tool_calls"] = tool_calls_with_server_calls # Include model information from the response if available if hasattr(response, "model") and response.model: inputs["model"] = response.model @@ -469,8 +481,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation): returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") guardrailed_tool_calls: Final[list[dict[str, object]]] = ( cast(list[dict[str, object]], returned_tool_calls) - if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check) - else tool_calls_to_check + if isinstance(returned_tool_calls, list) + and len(returned_tool_calls) == len(tool_calls_with_server_calls) + else tool_calls_with_server_calls ) # Step 3: Map guardrail responses back to original response structure @@ -488,6 +501,11 @@ class OpenAIChatCompletionsHandler(BaseTranslation): tool_calls=guardrailed_tool_calls, task_mappings=tool_call_task_mappings, ) + self._apply_guardrail_responses_to_output_server_tool_calls( + response=response, + guardrailed_tool_calls=guardrailed_tool_calls[len(tool_calls_to_check) :], + task_mappings=tuple(mapping for mapping, _ in server_tool_call_tasks), + ) verbose_proxy_logger.debug("OpenAI Chat Completions: Processed output response: %s", response) @@ -681,6 +699,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): pre_guardrail_tool_calls=pre_guardrail_tool_calls, guardrail_name=guardrail_to_apply.guardrail_name or "unknown", ) + self._write_ended_stream_server_tool_call_rewrites( + responses_so_far=responses_so_far, + guardrailed_response=model_response, + ) @staticmethod def _rebuild_ended_stream_per_choice( @@ -939,6 +961,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if choice.message.tool_calls and isinstance(choice.message.tool_calls, list): if len(choice.message.tool_calls) > 0: return True + if anthropic_server_tool_call_params(choice.message): + return True elif isinstance(response, ModelResponseStream): for streaming_choice in response.choices: if isinstance(streaming_choice, litellm.StreamingChoices): @@ -949,6 +973,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if streaming_choice.delta.tool_calls and isinstance(streaming_choice.delta.tool_calls, list): if len(streaming_choice.delta.tool_calls) > 0: return True + if anthropic_server_tool_call_params(streaming_choice.delta): + return True return False def _extract_output_text_images_and_tool_calls( @@ -1099,6 +1125,25 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if "name" in func_dict: existing_tool_call.function.name = func_dict["name"] + @staticmethod + def _apply_guardrail_responses_to_output_server_tool_calls( + response: "ModelResponse", + guardrailed_tool_calls: Sequence[object], + task_mappings: Sequence[tuple[int, int]], + ) -> None: + guardrailed_by_task: Final = MappingProxyType(dict(zip(task_mappings, guardrailed_tool_calls))) + for choice_idx, choice in enumerate(response.choices): + server_tool_calls = anthropic_server_tool_call_params(choice.message) + rewritten = tuple( + _rewritten_server_tool_call(server_tool_call, guardrailed_by_task.get((choice_idx, server_idx))) + for server_idx, server_tool_call in enumerate(server_tool_calls) + ) + if rewritten != server_tool_calls: + choice.message.provider_specific_fields = { + **(choice.message.provider_specific_fields or {}), + "server_tool_calls": list(rewritten), + } + @staticmethod def _string_choice_contents(response: "ModelResponse") -> tuple[str | None, ...]: return tuple( @@ -1211,6 +1256,32 @@ class OpenAIChatCompletionsHandler(BaseTranslation): for fragment in tail: fragment.function.arguments = "" + @staticmethod + def _write_ended_stream_server_tool_call_rewrites( + responses_so_far: Sequence["ModelResponseStream"], + guardrailed_response: "ModelResponse", + ) -> None: + guardrailed_by_id: Final = MappingProxyType( + { + (guardrailed_response.choices[choice_position].index, server_tool_call["id"]): server_tool_call + for (choice_position, _), server_tool_call in _server_tool_call_tasks(guardrailed_response.choices) + } + ) + if not guardrailed_by_id: + return + for response in responses_so_far: + for choice in response.choices: + streamed = anthropic_server_tool_call_params(choice.delta) + rewritten = tuple( + guardrailed_by_id.get((choice.index, server_tool_call["id"]), server_tool_call) + for server_tool_call in streamed + ) + if rewritten != streamed: + choice.delta.provider_specific_fields = { + **(choice.delta.provider_specific_fields or {}), + "server_tool_calls": list(rewritten), + } + async def _apply_guardrail_responses_to_output_streaming( self, responses: list["ModelResponseStream"], @@ -1400,7 +1471,42 @@ def _stream_chunk_choices(item: object) -> Sequence[object]: def _streamed_delta_tool_calls(delta: object) -> tuple[object, ...]: function_call: Final = stream_item_field(delta, "function_call") legacy: Final = () if function_call is None else (function_call,) - return stream_item_items(delta, "tool_calls") + legacy + return anthropic_server_tool_call_params(delta) + stream_item_items(delta, "tool_calls") + legacy + + +def _server_tool_call_tasks( + choices: Sequence[object], +) -> Iterator[tuple[tuple[int, int], ChatCompletionAssistantToolCall]]: + for choice_idx, choice in enumerate(choices): + if not isinstance(choice, Choices): + continue + for server_idx, server_tool_call in enumerate(anthropic_server_tool_call_params(choice.message)): + yield (choice_idx, server_idx), server_tool_call + + +def _server_tool_call_input(server_tool_call: ChatCompletionAssistantToolCall) -> dict[str, object]: + return { + "id": server_tool_call["id"], + "type": server_tool_call["type"], + "function": dict(server_tool_call["function"]), + } + + +_TOOL_CALL_FUNCTION_ADAPTER: Final = TypeAdapter(ChatCompletionToolCallFunctionChunk) + + +def _rewritten_server_tool_call( + server_tool_call: ChatCompletionAssistantToolCall, guardrailed: object +) -> ChatCompletionAssistantToolCall: + try: + function: Final = _TOOL_CALL_FUNCTION_ADAPTER.validate_python(stream_item_field(guardrailed, "function")) + except ValidationError: + return server_tool_call + return ChatCompletionAssistantToolCall( + id=server_tool_call["id"], + type=server_tool_call["type"], + function={**server_tool_call["function"], **function}, + ) def _released_choices(responses_so_far: Sequence[object]) -> Iterator[object]: diff --git a/litellm/proxy/db/spend_log_tool_index.py b/litellm/proxy/db/spend_log_tool_index.py index 591ad81a4e5..92181d546eb 100644 --- a/litellm/proxy/db/spend_log_tool_index.py +++ b/litellm/proxy/db/spend_log_tool_index.py @@ -63,11 +63,26 @@ def response_tool_call_names(completion_response: object) -> tuple[str, ...]: ) return tuple( stripped - for name in chain(called_names, _auto_executed_mcp_tool_names(completion_response)) + for name in chain( + _server_tool_call_names(completion_response), + called_names, + _auto_executed_mcp_tool_names(completion_response), + ) if isinstance(name, str) and (stripped := name.strip()) ) +def _server_tool_call_names(completion_response: object) -> tuple[str | None, ...]: + from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of + from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params + + server_tool_calls: Final = chain.from_iterable( + anthropic_server_tool_call_params(message_field(choice, "message")) + for choice in parts_of(message_field(completion_response, "choices")) + ) + return tuple(server_tool_call["function"].get("name") for server_tool_call in server_tool_calls) + + class _ExecutedTool(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 559ca6b7c23..ad03186c7a5 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -31,6 +31,7 @@ from openai.types.chat.chat_completion_named_tool_choice_param import ( from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch from openai.types.responses.response_compaction_item import ResponseCompactionItem from openai.types.responses.response_create_params import ResponseInputParam +from openai.types.responses.response_function_web_search import ActionOpenPage, ActionSearch from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam from openai.types.responses.tool_param import FunctionToolParam @@ -56,7 +57,12 @@ from litellm.litellm_core_utils.hidden_params import ( set_hidden_params, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls, supports_anthropic_cache_control +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + anthropic_server_tool_calls, + supports_anthropic_cache_control, + validated_object_mapping, +) from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control @@ -66,11 +72,12 @@ from litellm.responses.litellm_completion_transformation.session_handler import ResponsesSessionHandler, StoredTurns, ) -from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex +from litellm.types.llms.anthropic import ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, AnthropicToolSearchToolRegex from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, + ChatCompletionAssistantToolCall, ChatCompletionAudioObject, ChatCompletionCachedContent, ChatCompletionFileObject, @@ -114,6 +121,7 @@ from litellm.types.responses.main import ( OutputImageGenerationCall, OutputText, OutputToolSearchCall, + web_search_call_tool_id, ) from litellm.types.utils import ( ChatCompletionAnnotation, @@ -873,6 +881,29 @@ def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, obj return {key: {**_STR_KEY_DICT_ADAPTER.validate_python(metadata), CALLER_SESSION_ID_METADATA_KEY: chain_session}} +def _pending_claude_server_tool_call(search: ResponseFunctionWebSearch) -> ChatCompletionAssistantToolCall | None: + tool_id: Final = web_search_call_tool_id(search.id) + if search.status != "in_progress" or tool_id is None: + return None + if not tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX): + return None + match search.action: + case ActionSearch(query=str() as query): + return _server_tool_call(tool_id, "web_search", json.dumps({"query": query})) + case ActionOpenPage(url=str() as url): + return _server_tool_call(tool_id, "web_fetch", json.dumps({"url": url})) + case _: + return None + + +def _server_tool_call(tool_id: str, name: str, arguments: str) -> ChatCompletionAssistantToolCall: + return ChatCompletionAssistantToolCall( + id=tool_id, + type="function", + function=ChatCompletionToolCallFunctionChunk(name=name, arguments=arguments), + ) + + class LiteLLMCompletionResponsesConfig: @staticmethod def get_supported_openai_params(model: str) -> list: @@ -1778,10 +1809,22 @@ class LiteLLMCompletionResponsesConfig: message_cache_control: Final = ( None if isinstance(content, list) else new_cache_control or previous_cache_control ) + new_server_tool_calls: Final = anthropic_server_tool_call_params(new_message) + server_tool_call_fields: Final = ( + { + "provider_specific_fields": { + **(validated_object_mapping(last_message.get("provider_specific_fields")) or {}), + "server_tool_calls": [*anthropic_server_tool_call_params(last_message), *new_server_tool_calls], + } + } + if new_server_tool_calls + else {} + ) merged: Final = { **{key: value for key, value in last_message.items() if key != "cache_control"}, "content": content, **({} if message_cache_control is None else {"cache_control": message_cache_control}), + **server_tool_call_fields, } return cast( # cast-ok: preserves the assistant fields and content blocks ChatCompletionAssistantMessage, merged @@ -2223,10 +2266,15 @@ class LiteLLMCompletionResponsesConfig: content="Hosted web search: " + replay.model_dump_json(exclude_none=True), ) ] + hosted_search_text: Final = "Hosted web search: " + search.model_dump_json(exclude_none=True) + pending_server_tool_call: Final = _pending_claude_server_tool_call(search) + if pending_server_tool_call is None: + return [GenericChatCompletionMessage(role="assistant", content=hosted_search_text)] return [ - GenericChatCompletionMessage( + ChatCompletionResponseMessage( role="assistant", - content="Hosted web search: " + search.model_dump_json(exclude_none=True), + content=hosted_search_text, + provider_specific_fields={"server_tool_calls": [pending_server_tool_call]}, ) ] if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(input_item): diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 7b1fb0130d8..e9111eb8c99 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -92,6 +92,9 @@ def _is_tool_result_error(content: object, tool_name: str) -> bool: return False +WEB_SEARCH_CALL_ID_PREFIX: Final = "ws_" + + def build_web_search_call( tool_id: str, tool_input: object, @@ -101,13 +104,18 @@ def build_web_search_call( ) -> ResponseFunctionWebSearch: content: Final = _result_content(result) return ResponseFunctionWebSearch( - id=f"ws_{tool_id}", + id=f"{WEB_SEARCH_CALL_ID_PREFIX}{tool_id}", type="web_search_call", status=status or ("failed" if _is_tool_result_error(content, tool_name) else "completed"), action=_web_fetch_action(tool_input) if tool_name == "web_fetch" else _web_search_action(tool_input, content), ) +def web_search_call_tool_id(call_id: str) -> str | None: + tool_id: Final = call_id.removeprefix(WEB_SEARCH_CALL_ID_PREFIX) + return None if tool_id == call_id else tool_id + + class OutputImageGenerationCall(BaseLiteLLMOpenAIResponseObject): """An image generation call output""" diff --git a/tests/unit/integrations/datadog/test_datadog_llm_obs.py b/tests/unit/integrations/datadog/test_datadog_llm_obs.py index a7826e11483..dcc1567f5f8 100644 --- a/tests/unit/integrations/datadog/test_datadog_llm_obs.py +++ b/tests/unit/integrations/datadog/test_datadog_llm_obs.py @@ -110,6 +110,41 @@ def test_output_tool_calls_use_the_datadog_tool_call_schema(logger: DataDogLLMOb assert "function" not in message["tool_calls"][0] + +def test_claude_server_tool_calls_are_reported_before_client_tool_calls(logger: DataDogLLMObsLogger) -> None: + payload = build( + logger, + response_message={ + "role": "assistant", + "content": None, + "tool_calls": [ASSISTANT_TOOL_CALL], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "paris weather"}'}, + } + ] + }, + }, + ) + + assert payload["meta"]["output"]["messages"][0]["tool_calls"] == [ + { + "name": "web_search", + "arguments": {"query": "paris weather"}, + "tool_id": "srvtoolu_01Search", + "type": "function", + }, + { + "name": "get_weather", + "arguments": {"city": "Paris", "unit": "c"}, + "tool_id": "call_abc123", + "type": "function", + }, + ] + def test_tool_call_identifiers_that_are_not_strings_are_blanked_not_stringified(logger: DataDogLLMObsLogger) -> None: payload = build( logger, diff --git a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py index 57764bf902b..611e828cf84 100644 --- a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py @@ -174,6 +174,45 @@ def test_openinference_output_tool_calls_preserve_calls_in_attributes_and_value( ] +def test_openinference_output_lists_claude_server_tool_calls_before_client_tool_calls(): + data: Final = _llm_call( + choices_out=( + { + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_paris", + "type": "function", + "function": {"name": "lookup_weather", "arguments": '{"city": "Paris"}'}, + } + ], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "paris weather"}'}, + } + ] + }, + }, + }, + ) + ) + attrs: Final = OpenInferenceMapper().map(data) + assert {key: value for key, value in attrs.items() if ".tool_calls." in key} == { + "llm.output_messages.0.message.tool_calls.0.tool_call.id": "srvtoolu_01Search", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.name": "web_search", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": '{"query": "paris weather"}', + "llm.output_messages.0.message.tool_calls.1.tool_call.id": "call_paris", + "llm.output_messages.0.message.tool_calls.1.tool_call.function.name": "lookup_weather", + "llm.output_messages.0.message.tool_calls.1.tool_call.function.arguments": '{"city": "Paris"}', + } + + def test_openinference_responses_tool_calls_are_emitted_as_output_attributes(): data: Final = LLMCallSpanData.from_standard_logging_payload( _responses_payload( diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 5d71cbe9afc..fcc3b1e2b3c 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -144,6 +144,42 @@ class MockMisalignedToolCallGuardrail(CustomGuardrail): ) +def _server_tool_call(call_id: str, name: str, arguments: Mapping[str, object]) -> dict[str, object]: + return {"id": call_id, "type": "function", "function": {"name": name, "arguments": json.dumps(arguments)}} + + +def _tool_call_ids_names_and_arguments(tool_calls) -> list[tuple[str, str, str]]: + return [ + (tool_call["id"], tool_call["function"]["name"], tool_call["function"]["arguments"]) + for tool_call in tool_calls + ] + + +class QueryRedactingGuardrail(CustomGuardrail): + """Returns copied tool calls with any ``query`` argument redacted, leaving other calls as they came.""" + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + def redacted(tool_call): + arguments = json.loads(tool_call["function"]["arguments"]) + if "query" not in arguments: + return tool_call + return { + **tool_call, + "function": {**tool_call["function"], "arguments": json.dumps({**arguments, "query": "[REDACTED]"})}, + } + + return GenericGuardrailAPIInputs( + texts=inputs.get("texts", []), + tool_calls=[redacted(tool_call) for tool_call in inputs.get("tool_calls", [])], + ) + + class TestOpenAIChatCompletionsHandlerToolsInput: """Test input processing with tools (function definitions)""" @@ -906,6 +942,92 @@ class TestOpenAIChatCompletionsHandlerToolCallsOutput: assert tool_calls[0].function.arguments == first_args assert tool_calls[1].function.arguments == second_args + @pytest.mark.asyncio + async def test_output_guardrail_sees_a_response_that_only_carries_server_tool_calls(self): + handler = OpenAIChatCompletionsHandler() + guardrail = TestOpenAIChatCompletionsHandlerStreamingOutput._recording_guardrail() + response = ModelResponse( + model="claude-sonnet-4-5", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + content=None, + role="assistant", + provider_specific_fields={ + "server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})] + }, + ), + ) + ], + ) + + await handler.process_output_response(response, guardrail) + + assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "x"}') + ] + + @pytest.mark.asyncio + async def test_output_guardrail_rewrite_of_a_server_tool_call_lands_in_server_tool_calls_only(self): + handler = OpenAIChatCompletionsHandler() + weather_args = json.dumps({"city": "Paris", "query": "forecast"}) + response = ModelResponse( + model="claude-sonnet-4-5", + choices=[ + Choices( + finish_reason="tool_calls", + index=0, + message=Message( + content="Looking it up", + role="assistant", + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_1", + type="function", + function=Function(name="get_weather", arguments=json.dumps({"city": "Paris"})), + ) + ], + provider_specific_fields={ + "server_tool_calls": [ + _server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"}) + ] + }, + ), + ), + Choices( + finish_reason="tool_calls", + index=1, + message=Message( + content=None, + role="assistant", + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_2", + type="function", + function=Function(name="get_weather", arguments=weather_args), + ) + ], + ), + ), + ], + ) + + await handler.process_output_response(response, QueryRedactingGuardrail(guardrail_name="redact")) + + first, second = response.choices + assert _tool_call_ids_names_and_arguments(first.message.provider_specific_fields["server_tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}') + ] + assert [(call.id, call.function.arguments) for call in first.message.tool_calls] == [ + ("call_1", '{"city": "Paris"}') + ] + assert [(call.id, call.function.arguments) for call in second.message.tool_calls] == [ + ("call_2", '{"city": "Paris", "query": "[REDACTED]"}') + ] + assert not (second.message.provider_specific_fields or {}).get("server_tool_calls") + class MockPassThroughGuardrail(CustomGuardrail): """Mock guardrail that passes through without blocking - for testing streaming fallback behavior""" @@ -1462,6 +1584,61 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert chunks[0].choices[0].delta.content == "hello [MASKED]" assert chunks[1].choices[0].delta.content in (None, "") + @staticmethod + def _ended_server_tool_call_stream_chunks() -> list: + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + def chunk(delta: Delta, finish_reason: Optional[str] = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-123", + created=1234567890, + model="claude-sonnet-4-5", + object="chat.completion.chunk", + choices=[StreamingChoices(index=0, delta=delta, finish_reason=finish_reason)], + ) + + search = {**_server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"}), "index": 0} + return [ + chunk(Delta(content="Searching")), + chunk(Delta(content=None, provider_specific_fields={"server_tool_calls": [search]})), + chunk(Delta(content=None), finish_reason="stop"), + ] + + @pytest.mark.asyncio + async def test_ended_stream_guardrail_sees_streamed_server_tool_calls(self): + handler = OpenAIChatCompletionsHandler() + guardrail = self._recording_guardrail() + + await handler.process_output_streaming_response( + responses_so_far=self._ended_server_tool_call_stream_chunks(), + guardrail_to_apply=guardrail, + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "jane@example.com"}') + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_writes_server_tool_call_arguments_back_into_chunks(self): + handler = OpenAIChatCompletionsHandler() + chunks = self._ended_server_tool_call_stream_chunks() + + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=QueryRedactingGuardrail(guardrail_name="redact"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + delta = chunks[1].choices[0].delta + assert _tool_call_ids_names_and_arguments(delta.provider_specific_fields["server_tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}') + ] + assert delta.tool_calls is None + assert chunks[0].choices[0].delta.content == "Searching" + class TestUndecoratedGuardrailIsRecorded: """LIT-5983 regression: the handler calls apply_guardrail bare, so a custom guardrail @@ -2323,6 +2500,29 @@ class TestStreamingScanKey: assert len(ended_key.tool_calls) == 1 and "run_shell" in ended_key.tool_calls[0] assert ended_key.tool_calls_in_flight is False + def test_server_tool_calls_hold_the_window_and_enter_the_key_once_the_stream_has_ended(self): + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + handler = OpenAIChatCompletionsHandler() + server_chunk = ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta( + content=None, + provider_specific_fields={ + "server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})] + }, + ), + ) + ] + ) + open_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk]) + ended_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk, self._chunk(None, "stop")]) + assert open_key.tool_calls_in_flight is True + assert len(ended_key.tool_calls) == 1 and "web_search" in ended_key.tool_calls[0] + assert ended_key != open_key + def test_text_after_the_first_choice_finishes_still_changes_the_key(self): handler = OpenAIChatCompletionsHandler() first_done = [self._chunk("a", index=0), self._chunk("b", finish_reason="stop", index=0)] diff --git a/tests/unit/proxy/db/test_spend_log_tool_index.py b/tests/unit/proxy/db/test_spend_log_tool_index.py index af3268a7cc9..a3f400bbc15 100644 --- a/tests/unit/proxy/db/test_spend_log_tool_index.py +++ b/tests/unit/proxy/db/test_spend_log_tool_index.py @@ -23,6 +23,7 @@ from litellm.proxy.db.spend_log_tool_index import ( ) from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import ModelResponse from tests.unit.proxy.db.fake_prisma_engine import engine_call @@ -204,6 +205,36 @@ class TestResponseToolCallNames: def test_blank_names_are_dropped(self): assert response_tool_call_names(_response_with_tool_calls(" ", "real_tool")) == ("real_tool",) + @pytest.mark.parametrize("as_object", [False, True], ids=["dict", "model_response"]) + def test_server_tool_calls_of_every_choice_are_named_before_client_tool_calls(self, as_object: bool): + def tool_call(call_id: str, name: str) -> dict[str, object]: + return {"id": call_id, "type": "function", "function": {"name": name, "arguments": "{}"}} + + response: dict[str, object] = { + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [tool_call("call_1", "get_weather")], + "provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_01", "web_search")]}, + }, + }, + { + "index": 1, + "message": { + "role": "assistant", + "content": "done", + "provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_02", "web_fetch")]}, + }, + }, + ] + } + completion_response = ModelResponse(**response) if as_object else response + + assert response_tool_call_names(completion_response) == ("web_search", "web_fetch", "get_weather") + def test_responses_api_output_function_calls(self): # Regression: /v1/responses carries invocations in output[] items of # type function_call, not in choices; they must reach the rollup. diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 20c7d3076c1..aabf2e1d7cc 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -5216,6 +5216,75 @@ class TestHostedWebSearchReplay: assert messages[2]["content"] == "Paris is sunny." assert history == original + @pytest.mark.parametrize( + "leading_text", ((), ({"type": "text", "text": "Let me look that up."},)), ids=("alone", "after_text") + ) + @pytest.mark.parametrize( + ("server_tool_use", "expected_input"), + ( + ( + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_fetch", "input": {"url": _FETCHED_URL}}, + {"url": _FETCHED_URL}, + ), + ( + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}}, + {"query": "python"}, + ), + ), + ids=("web_fetch", "web_search"), + ) + def test_pending_claude_web_call_replays_to_anthropic_as_its_own_server_tool_use( + self, + leading_text: tuple[Mapping[str, object], ...], + server_tool_use: Mapping[str, object], + expected_input: Mapping[str, object], + ) -> None: + response: Final = _anthropic_turn_on_responses((*leading_text, server_tool_use, _WEATHER_TOOL_USE)) + history: Final = [ + {"role": "user", "content": "Use the web and check the weather in Paris."}, + *(item.model_dump(exclude_none=True) for item in response.output), + {"type": "function_call_output", "call_id": "toolu_01Weather", "output": "Paris is sunny."}, + ] + + anthropic_turns: Final = anthropic_messages_pt( + messages=LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=history, responses_api_request={} + ), + model="claude-sonnet-5-5", + llm_provider="anthropic", + ) + + assert [turn["role"] for turn in anthropic_turns] == ["user", "assistant", "user"] + assert [block for block in anthropic_turns[1]["content"] if block["type"] != "text"] == [ + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": server_tool_use["name"], "input": expected_input}, + {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}}, + ] + assert [(block["type"], block["tool_use_id"]) for block in anthropic_turns[2]["content"]] == [ + ("tool_result", "toolu_01Weather") + ] + + @pytest.mark.parametrize( + ("call_id", "status"), + (("ws_srvtoolu_01Fetch", "completed"), ("ws_68a1b2c3d4", "in_progress"), ("srvtoolu_01Fetch", "in_progress")), + ids=("completed-claude-call", "pending-non-claude-call", "pending-unprefixed-id"), + ) + def test_web_call_that_is_not_a_pending_claude_call_replays_as_text_only( + self, call_id: str, status: Literal["completed", "in_progress"] + ) -> None: + search: Final = ResponseFunctionWebSearch( + id=call_id, type="web_search_call", status=status, action=ActionOpenPage(type="open_page", url=_FETCHED_URL) + ) + + messages: Final = ( + LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=search.model_dump(exclude_none=True) + ) + ) + + assert messages == [ + {"role": "assistant", "content": "Hosted web search: " + search.model_dump_json(exclude_none=True)} + ] + @pytest.mark.parametrize( "action", ( diff --git a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts index 78372fe56d9..a6693794df6 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts @@ -131,6 +131,52 @@ describe("ToolsSection utils", () => { expect(result[1].callData).toBeUndefined(); }); + it("should mark Claude server tools as called from provider_specific_fields.server_tool_calls", () => { + const log: Partial = { + request_id: "test-server-tools", + proxy_server_request: { + tools: [ + { type: "web_search_20250305", name: "web_search", max_uses: 3 }, + { type: "function", function: { name: "get_weather", description: "Get weather" } }, + { type: "function", function: { name: "send_email", description: "Send email" } }, + ], + }, + response: { + choices: [ + { + message: { + tool_calls: [ + { + id: "toolu_01Weather", + type: "function", + function: { name: "get_weather", arguments: '{"location": "Paris"}' }, + }, + ], + provider_specific_fields: { + server_tool_calls: [ + { + id: "srvtoolu_01Search", + type: "function", + function: { name: "web_search", arguments: '{"query": "paris weather"}' }, + }, + ], + }, + }, + }, + ], + }, + }; + + const result = parseToolsFromLog(log as LogEntry); + + expect(result.map((tool) => [tool.name, tool.called, tool.callData?.id])).toEqual([ + ["web_search", true, "srvtoolu_01Search"], + ["get_weather", true, "toolu_01Weather"], + ["send_email", false, undefined], + ]); + expect(result[0].callData?.arguments).toEqual({ query: "paris weather" }); + }); + it("should handle string format request and response", () => { const log: Partial = { request_id: "test-5", diff --git a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts index 5e76a09c155..b1b2e6feb99 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts @@ -42,6 +42,24 @@ function extractToolsFromRequest(log: LogEntry): ToolDefinition[] { return []; } +interface ChatMessageToolCallFields { + tool_calls?: unknown; + provider_specific_fields?: { server_tool_calls?: unknown } | null; +} + +function toolCallList(value: unknown): ToolCall[] { + return Array.isArray(value) ? value : []; +} + +function extractChatMessageToolCalls(message: ChatMessageToolCallFields | undefined): ToolCall[] | null { + const serverToolCalls = message?.provider_specific_fields?.server_tool_calls; + const clientToolCalls = message?.tool_calls; + if (!Array.isArray(serverToolCalls) && !Array.isArray(clientToolCalls)) { + return null; + } + return [...toolCallList(serverToolCalls), ...toolCallList(clientToolCalls)]; +} + /** * Extract tool calls from response data */ @@ -53,10 +71,9 @@ function extractToolCallsFromResponse(log: LogEntry): ToolCall[] { // OpenAI format: response.choices[0].message.tool_calls const choices = responseData.choices; if (Array.isArray(choices) && choices.length > 0) { - const firstChoice = choices[0]; - const message = firstChoice.message; - if (message && Array.isArray(message.tool_calls)) { - return message.tool_calls; + const chatToolCalls = extractChatMessageToolCalls(choices[0].message); + if (chatToolCalls) { + return chatToolCalls; } }